def run_tests(options): """ Executes the entire test suite. """ if options.get('coverage'): prefix = 'coverage run --branch --source=geonode \ --omit="*/management/*,*/__init__*,*/views*,*/signals*,*/tasks*,*/test*,*/wsgi*,*/middleware*,*/search_indexes*,\ */migrations*,*/context_processors*,geonode/qgis_server/*,geonode/upload/*,geonode/monitoring/*"' else: prefix = 'python' local = options.get('local', 'false') # travis uses default to false if not integration_tests and not integration_csw_tests and not integration_bdd_tests: call_task('test', options={'prefix': prefix}) else: if integration_tests: call_task('test_integration') # only start if using Geoserver backend _backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND']) if _backend == 'geonode.geoserver' and 'geonode.geoserver' in INSTALLED_APPS: call_task('test_integration', options={'name': 'geonode.upload.tests.integration'}) elif integration_csw_tests: call_task('test_integration', options={'name': 'geonode.tests.csw'}) if integration_bdd_tests: call_task('test_bdd', options={'local': local}) sh('flake8 geonode')
def deb(options): """ Creates debian packages. Example uses: paver deb paver deb -k 12345 paver deb -k 12345 -p geonode/testing """ key = options.get("key", None) ppa = options.get("ppa", None) version, simple_version = versions() info("Creating package for GeoNode version %s" % version) with pushd("package"): # Get rid of any uncommitted changes to debian/changelog info("Getting rid of any uncommitted changes in debian/changelog") sh("git checkout debian/changelog") # Workaround for git-dch bug # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580 # path('.git').makedirs() # Link the parent git repository folder in to the current one # needed by git-dch sh("ln -s ../.git .git") # Install requirements # sh('sudo apt-get -y install debhelper devscripts git-buildpackage') sh( ( "git-dch --spawn-editor=snapshot --git-author --new-version=%s" " --id-length=6 --ignore-branch --release" % (simple_version) ) ) deb_changelog = path("debian") / "changelog" for line in fileinput.input([deb_changelog], inplace=True): print line.replace("urgency=low", "urgency=high"), ## Revert workaround for git-dhc bug sh("rm -rf .git") if key is None and ppa is None: # A local installable package sh("debuild -uc -us -A") elif key is None and ppa is not None: # A sources package, signed by daemon sh("debuild -S") elif key is not None and ppa is None: # A signed installable package sh("debuild -k%s -A" % key) elif key is not None and ppa is not None: # A signed, source package sh("debuild -k%s -S" % key) if ppa is not None: sh("dput ppa:%s geonode_%s_source.changes" % (ppa, simple_version))
def start_django(): """ Start the GeoNode Django application """ bind = options.get("bind", "") foreground = "" if options.get("foreground", False) else "&" sh("python manage.py runserver %s %s" % (bind, foreground))
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" download_dir = path("downloaded") if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path("geoserver") geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL) jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) grab(options.get("geoserver", GEOSERVER_URL), geoserver_bin, "geoserver binary") grab(options.get("jetty", JETTY_RUNNER_URL), jetty_runner, "jetty runner") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir / "geoserver" if not webapp_dir: webapp_dir.makedirs() print "extracting geoserver" z = zipfile.ZipFile(geoserver_bin, "r") z.extractall(webapp_dir) _install_data_dir()
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" download_dir = path('downloaded') if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path('geoserver') geoserver_bin = download_dir / os.path.basename(dev_config['GEOSERVER_URL']) jetty_runner = download_dir / os.path.basename(dev_config['JETTY_RUNNER_URL']) grab(options.get('geoserver', dev_config['GEOSERVER_URL']), geoserver_bin, "geoserver binary") grab(options.get('jetty', dev_config['JETTY_RUNNER_URL']), jetty_runner, "jetty runner") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir / 'geoserver' if not webapp_dir: webapp_dir.makedirs() print 'extracting geoserver' z = zipfile.ZipFile(geoserver_bin, "r") z.extractall(webapp_dir) _install_data_dir()
def manifest(): """Guarantee the existence of a basic MANIFEST.in. manifest doc: http://docs.python.org/distutils/sourcedist.html#manifest `options.paved.dist.manifest.include`: set of files (or globs) to include with the `include` directive. `options.paved.dist.manifest.recursive_include`: set of files (or globs) to include with the `recursive-include` directive. `options.paved.dist.manifest.prune`: set of files (or globs) to exclude with the `prune` directive. `options.paved.dist.manifest.include_sphinx_docroot`: True -> sphinx docroot is added as `graft` `options.paved.dist.manifest.include_sphinx_docroot`: True -> sphinx builddir is added as `prune` """ prune = options.paved.dist.manifest.prune graft = set() if options.paved.dist.manifest.include_sphinx_docroot: docroot = options.get("docroot", "docs") graft.update([docroot]) if options.paved.dist.manifest.exclude_sphinx_builddir: builddir = docroot + "/" + options.get("builddir", ".build") prune.update([builddir]) with open(options.paved.cwd / "MANIFEST.in", "w") as fo: for item in graft: fo.write("graft %s\n" % item) for item in options.paved.dist.manifest.include: fo.write("include %s\n" % item) for item in options.paved.dist.manifest.recursive_include: fo.write("recursive-include %s\n" % item) for item in prune: fo.write("prune %s\n" % item)
def start_django(): """ Start the GeoNode Django application """ bind = options.get('bind', '') foreground = '' if options.get('foreground', False) else '&' sh('python manage.py runserver %s %s' % (bind, foreground))
def start_django(): """ Start the GeoNode Django application """ settings = options.get('settings', '') if settings: settings = 'DJANGO_SETTINGS_MODULE=%s' % settings bind = options.get('bind', '0.0.0.0:8000') foreground = '' if options.get('foreground', False) else '&' sh('%s python -W ignore manage.py runserver %s %s' % (settings, bind, foreground)) if ASYNC_SIGNALS: celery_queues = [ "default", "geonode", "cleanup", "update", "email", # Those queues are directly managed by messages.consumer # "broadcast", # "email.events", # "all.geoserver", # "geoserver.events", # "geoserver.data", # "geoserver.catalog", # "notifications.events", # "geonode.layer.viewer" ] sh('%s celery -A geonode worker -Q %s -B -E -l INFO %s' % (settings, ",".join(celery_queues),foreground)) sh('%s python -W ignore manage.py runmessaging %s' % (settings, foreground))
def run_tests(options): """ Executes the entire test suite. """ if options.get('coverage'): prefix = 'coverage run --branch --source=geonode --omit="*/management/*,geonode/contrib/*,*/test*,*/wsgi*,*/middleware*"' else: prefix = 'python' local = options.get('local', 'false') # travis uses default to false if not integration_tests: sh('%s manage.py test geonode.tests.smoke %s %s' % (prefix, _keepdb, _parallel)) call_task('test', options={'prefix': prefix}) else: call_task('test_integration') call_task('test_integration', options={'name': 'geonode.tests.csw'}) # only start if using Geoserver backend if 'geonode.geoserver' in INSTALLED_APPS and OGC_SERVER['default']['BACKEND'] == 'geonode.geoserver': call_task('test_integration', options={'name': 'geonode.upload.tests.integration', 'settings': 'geonode.upload.tests.test_settings'}) call_task('test_bdd', options={'local': local}) sh('flake8 geonode')
def sloccount(): '''Print "Source Lines of Code" and export to file. Export is hudson_ plugin_ compatible: sloccount.sc requirements: - sloccount_ should be installed. - tee and pipes are used options.paved.pycheck.sloccount.param .. _sloccount: http://www.dwheeler.com/sloccount/ .. _hudson: http://hudson-ci.org/ .. _plugin: http://wiki.hudson-ci.org/display/HUDSON/SLOCCount+Plugin ''' # filter out subpackages setup = options.get('setup') packages = options.get('packages') if setup else None if packages: dirs = [x for x in packages if '.' not in x] else: dirs = ['.'] # sloccount has strange behaviour with directories, # can cause exception in hudson sloccount plugin. # Better to call it with file list ls=[] for d in dirs: ls += list(path(d).walkfiles()) #ls=list(set(ls)) files=' '.join(ls) param=options.paved.pycheck.sloccount.param sh('sloccount {param} {files} | tee sloccount.sc'.format(param=param, files=files))
def deb(options): """ Creates debian packages. Example uses: paver deb paver deb -k 12345 paver deb -k 12345 -p geonode/testing """ key = options.get('key', None) ppa = options.get('ppa', None) version, simple_version = versions() info('Creating package for GeoNode version %s' % version) # Get rid of any uncommitted changes to debian/changelog info('Getting rid of any uncommitted changes in debian/changelog') sh('git checkout package/debian/changelog') # Workaround for git-dch bug # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580 sh('ln -s %s %s' % (os.path.realpath('.git'), os.path.realpath('package'))) with pushd('package'): # Install requirements # sh('sudo apt-get -y install debhelper devscripts git-buildpackage') sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s' ' --id-length=6 --ignore-branch --release' % (simple_version))) #In case you publish from Ubuntu Xenial (git-dch is removed from upstream) # use the following line instead: #sh(('gbp dch --spawn-editor=snapshot --git-author --new-version=%s' # ' --id-length=6 --ignore-branch --release' % (simple_version))) deb_changelog = path('debian') / 'changelog' for line in fileinput.input([deb_changelog], inplace=True): print line.replace("urgency=medium", "urgency=high"), # Revert workaround for git-dhc bug sh('rm -rf .git') if key is None and ppa is None: # A local installable package sh('debuild -uc -us -A') elif key is None and ppa is not None: # A sources package, signed by daemon sh('debuild -S') elif key is not None and ppa is None: # A signed installable package sh('debuild -k%s -A' % key) elif key is not None and ppa is not None: # A signed, source package sh('debuild -k%s -S' % key) if ppa is not None: sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
def start_messaging(): """ Start the GeoNode messaging server """ settings = options.get('settings', '') if settings: settings = 'DJANGO_SETTINGS_MODULE=%s' % settings foreground = '' if options.get('foreground', False) else '&' sh('%s python -W ignore manage.py runmessaging %s' % (settings, foreground))
def create(options): """create database objects and superuser account""" args = '' username = options.get('username', None) password = options.get('password', None) email = options.get('email', None) if all([username, password, email]): args = '%s %s %s' % (username, password, email) sh('python GeoHealthCheck/models.py create %s' % args)
def deploy(): """deploy install""" venv = options.get('venv', None) if venv is None: raise ValueError('path to virtual environment required') venv = os.path.abspath(venv) loc = [ 'import sys', 'sys.path.insert(0, "%s/app")' % venv, 'activate = "%s/bin/activate_this.py"' % venv, 'execfile(activate_this, {__file__: activate})', 'from app import APP as application' ] with open(options.app.build / 'vocab.wsgi', 'w') as fileobj: fileobj.write('\n'.join(loc)) loc = [ 'WSGIDaemonProcess vocab user=user1 group=group1 threads=5', 'WSGIScriptAlias /vocab %s' % options.app.home, '<Directory %s>' % path(venv), ' WSGIProcessGroup vocab', ' WSGIApplicationGroup %{GLOBAL}', ' Order deny,allow', ' Allow from all', '</Directory>'] with open(options.app.build / 'osgeo.vocab.conf', 'w') as fileobj: fileobj.write('\n'.join(loc))
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" fast = options.get('fast', False) download_dir = path('downloaded') if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path('geoserver') geoserver_bin = path('geoserver_ext/target/geoserver.war') if not geoserver_bin.exists(): geoserver_bin = download_dir / os.path.basename(ROGUE_GEOSERVER_URL) grab(ROGUE_GEOSERVER_URL, geoserver_bin, "geoserver binary") jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner") data_dir = download_dir / os.path.basename(DATA_DIR_URL) grab(DATA_DIR_URL, data_dir, "data dir") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir if not webapp_dir: webapp_dir.makedirs() print 'extracting geoserver' z = zipfile.ZipFile(geoserver_bin, "r") z.extractall(webapp_dir) # Set the geonode auth config to dev port 8000 sh("perl -pi.back -e 's/localhost/localhost:8000/g;' geoserver/data/security/auth/geonodeAuthProvider/config.xml")
def start_django(): """ Start the GeoNode Django application """ bind = options.get('bind', '') #Se comento la linea original para correr la app en una IP especifica sh('python manage.py runserver %s &' % bind)
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" fast = options.get('fast', False) download_dir = path('downloaded') if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path('geoserver') geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL) jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) data_dir = download_dir / os.path.basename(DATA_DIR_URL) grab(GEOSERVER_URL, geoserver_bin, "geoserver binary") grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner") grab(DATA_DIR_URL, data_dir, "data dir") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir / 'geoserver' if not webapp_dir: webapp_dir.makedirs() print 'extracting geoserver' with zipfile.ZipFile(geoserver_bin, "r") as z: z.extractall(webapp_dir) _install_data_dir()
def upload(): """upload package zipfile to server""" user = options.get("user", False) if not user: raise ValueError("OSGeo userid required") password = getpass.getpass("Enter your password: "******"": raise ValueError("password required") call_task("package") zipf = get_package_filename() url = "http://%s:%s@%s:%d/%s" % (user, password, options.upload.host, options.upload.port, options.upload.endpoint) info("Uploading to http://%s/%s" % (options.upload.host, options.upload.endpoint)) server = xmlrpclib.ServerProxy(url, verbose=False) try: with open(zipf) as zfile: plugin_id, version_id = server.plugin.upload(xmlrpclib.Binary(zfile.read())) info("Plugin ID: %s", plugin_id) info("Version ID: %s", version_id) except xmlrpclib.Fault as err: error("ERROR: fault error") error("Fault code: %d", err.faultCode) error("Fault string: %s", err.faultString) except xmlrpclib.ProtocolError as err: error("Error: Protocol error") error("%s : %s", err.errcode, err.errmsg) if err.errcode == 403: error("Invalid name and password")
def upload(): """upload package zipfile to server""" user = options.get('user', False) if not user: raise ValueError('OSGeo userid required') password = getpass.getpass('Enter your password: '******'': raise ValueError('password required') call_task('package') zipf = get_package_filename() url = 'http://%s:%s@%s:%d/%s' % (user, password, options.upload.host, options.upload.port, options.upload.endpoint) info('Uploading to http://%s/%s' % (options.upload.host, options.upload.endpoint)) server = xmlrpclib.ServerProxy(url, verbose=False) try: with open(zipf) as zfile: plugin_id, version_id = \ server.plugin.upload(xmlrpclib.Binary(zfile.read())) info('Plugin ID: %s', plugin_id) info('Version ID: %s', version_id) except xmlrpclib.Fault, err: error('ERROR: fault error') error('Fault code: %d', err.faultCode) error('Fault string: %s', err.faultString)
def test_integration(options): """ Run GeoNode's Integration test suite against the external apps """ _reset() # Start GeoServer call_task('start_geoserver') info("GeoNode is now available, running the tests now.") name = options.get('name', 'geonode.tests.integration') success = False try: if name == 'geonode.tests.csw': call_task('sync') call_task('start') sh('sleep 30') call_task('setup_data') sh(('python manage.py test %s' ' --noinput --liveserver=0.0.0.0:8000' % name)) except BuildFailure as e: info('Tests failed! %s' % str(e)) else: success = True finally: # don't use call task here - it won't run since it already has stop() _reset() if not success: sys.exit(1)
def _adjust_options(): """ Set default sphinx and gh_pages options. """ if options.get('_github_tools_options_adjusted') is None: options.setdefault('sphinx', Bunch()) options.setdefault('gh_pages', Bunch()) options.sphinx.docroot = docroot \ = path( options.sphinx.get('docroot', 'docs')) options.sphinx._buildir = buildir = \ docroot / options.sphinx.get('builddir', 'build') options.sphinx._sourcedir = \ docroot / options.sphinx.get('sourcedir', 'source') options.sphinx._doctrees = buildir / "doctrees" options.sphinx._htmldir = htmldir = \ buildir / 'html' gh_pages_root = options.gh_pages.get('root', None) if gh_pages_root is None: options.gh_pages.root = htmldir else: options.gh_pages.root = path(gh_pages_root) gh_pages_htmlroot = options.gh_pages.get('htmlroot', None) if gh_pages_htmlroot is None: options.gh_pages.htmlroot = options.gh_pages.root else: options.gh_pages.htmlroot = path(gh_pages_htmlroot) options.gh_pages.setdefault('remote_name', 'origin') options.gh_pages.setdefault('master_branch', 'master') options._github_tools_options_adjusted = True
def generate_csw_connections_file(): """generate a CSW connections file from a flat file of CSW URLs""" filename = options.get("filename", False) if not filename: raise ValueError("path to file of CSW URLs required") conns = etree.Element("qgsCSWConnections") conns.attrib["version"] = "1.0" with open(filename) as connsfh: for line in connsfh: url = line.strip() if not url: # blank line continue try: csw = CatalogueServiceWeb(url) title = unicode(csw.identification.title) etree.SubElement(conns, "csw", name=title, url=url) except Exception as err: error("ERROR on CSW %s: %s", url, err) with open("%s.xml" % filename, "w") as connsxmlfh: connsxmlfh.write(etree.tostring(conns, encoding="utf-8"))
def start_geoserver(options): """ Start GeoServer with GeoNode extensions """ from geonode.settings import OGC_SERVER GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION'] url = "http://localhost:8080/geoserver/" if GEOSERVER_BASE_URL != url: print 'your GEOSERVER_BASE_URL does not match %s' % url sys.exit(1) download_dir = path('downloaded').abspath() jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) data_dir = path('geoserver/data').abspath() web_app = path('geoserver/geoserver').abspath() log_file = path('geoserver/jetty.log').abspath() config = path('scripts/misc/jetty-runner.xml').abspath() # @todo - we should not have set workdir to the datadir but a bug in geoserver # prevents geonode security from initializing correctly otherwise with pushd(data_dir): javapath = "java" loggernullpath = "/dev/null" try: sh(('java -version')) except: if not options.get('java_path', None): print "Paver cannot find java in the Windows Environment. Please provide the --java_path flag with your full path to java.exe e.g. --java_path=C:/path/to/java/bin/java.exe" sys.exit(1) # if there are spaces javapath = 'START /B "" "' + options['java_path'] + '"' # cmd log file needs to exist in windows # using folder from .gitignore open("../../downloaded/null.txt", 'w+').close() loggernullpath = "../../downloaded/null.txt" sh(( '%(javapath)s -Xmx512m -XX:MaxPermSize=256m' ' -DGEOSERVER_DATA_DIR=%(data_dir)s' # workaround for JAI sealed jar issue and jetty classloader ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true' ' -jar %(jetty_runner)s' ' --log %(log_file)s' ' %(config)s' ' > %(loggernullpath)s &' % locals() )) info('Starting GeoServer on %s' % url) # wait for GeoServer to start started = waitfor(url) info('The logs are available at %s' % log_file) if not started: # If applications did not start in time we will give the user a chance # to inspect them and stop them manually. info(('GeoServer never started properly or timed out.' 'It may still be running in the background.')) sys.exit(1)
def generate_csw_connections_file(): """generate a CSW connections file from a flat file of CSW URLs""" filename = options.get('filename', False) if not filename: raise ValueError('path to file of CSW URLs required') conns = etree.Element('qgsCSWConnections') conns.attrib['version'] = '1.0' with open(filename) as connsfh: for line in connsfh: url = line.strip() if not url: # blank line continue try: csw = CatalogueServiceWeb(url) title = unicode(csw.identification.title) conn = etree.SubElement(conns, 'csw', name=title, url=url) except Exception as err: error('ERROR on CSW %s: %s', url, err) with open('%s.xml' % filename, 'w') as connsxmlfh: connsxmlfh.write(etree.tostring(conns, encoding='utf-8'))
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" # only start if using Geoserver backend _backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND']) if _backend == 'geonode.qgis_server' or 'geonode.geoserver' not in INSTALLED_APPS: return download_dir = path('downloaded') if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path('geoserver') geoserver_bin = download_dir / \ os.path.basename(dev_config['GEOSERVER_URL']) jetty_runner = download_dir / \ os.path.basename(dev_config['JETTY_RUNNER_URL']) if _django_11 and (integration_tests or integration_csw_tests or integration_bdd_tests): """Will make use of the docker container for the Integration Tests""" pass else: grab( options.get( 'geoserver', dev_config['GEOSERVER_URL']), geoserver_bin, "geoserver binary") grab( options.get( 'jetty', dev_config['JETTY_RUNNER_URL']), jetty_runner, "jetty runner") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir / 'geoserver' if not webapp_dir: webapp_dir.makedirs() print 'extracting geoserver' z = zipfile.ZipFile(geoserver_bin, "r") z.extractall(webapp_dir) _install_data_dir()
def test(options): """ Run GeoNode's Unit Test Suite """ prefix = options.get('prefix', 'python') sh("%s manage.py test %s.tests --noinput -v 2" % (prefix, '.tests '.join(GEONODE_APPS)))
def updategeoip(options): """ Update geoip db """ settings = options.get('settings', '') if settings: settings = 'DJANGO_SETTINGS_MODULE=%s' % settings sh("%s python -W ignore manage.py updategeoip -o" % settings)
def setup_data(): """ Import sample data (from gisdata package) into GeoNode """ import gisdata ctype = options.get('type', None) data_dir = gisdata.GOOD_DATA if ctype in ['vector', 'raster', 'time']: data_dir = os.path.join(gisdata.GOOD_DATA, ctype) settings = options.get('settings', '') if settings: settings = 'DJANGO_SETTINGS_MODULE=%s' % settings sh("%s python -W ignore manage.py importlayers %s -v2" % (settings, data_dir))
def add_language_catalogue(options): """adds new language profile""" lang = options.get('lang', None) if lang is None: raise RuntimeError('missing lang argument') sh('pybabel init -i %s -d %s -l %s' % ( options.base.pot, options.base.translations, lang))
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" # only start if using Geoserver backend if 'geonode.geoserver' not in INSTALLED_APPS or OGC_SERVER['default']['BACKEND'] == 'geonode.qgis_server': return download_dir = path('downloaded') if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path('geoserver') geoserver_bin = download_dir / \ os.path.basename(dev_config['GEOSERVER_URL']) jetty_runner = download_dir / \ os.path.basename(dev_config['JETTY_RUNNER_URL']) grab( options.get( 'geoserver', dev_config['GEOSERVER_URL']), geoserver_bin, "geoserver binary") grab( options.get( 'jetty', dev_config['JETTY_RUNNER_URL']), jetty_runner, "jetty runner") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir / 'geoserver' if not webapp_dir: webapp_dir.makedirs() print 'extracting geoserver' z = zipfile.ZipFile(geoserver_bin, "r") z.extractall(webapp_dir) _install_data_dir()
def test(options): """ Run GeoNode's Unit Test Suite """ if on_travis: if core_tests: _apps = tuple(GEONODE_CORE_APPS) if internal_apps_tests: _apps = tuple(GEONODE_INTERNAL_APPS) else: _apps = tuple(GEONODE_APPS) sh("%s manage.py test %s.tests --noinput %s %s" % (options.get('prefix'), '.tests '.join(_apps), _keepdb, _parallel))
def sync(options): """ Run the migrate and migrate management commands to create and migrate a DB """ settings = options.get('settings', '') if settings: settings = 'DJANGO_SETTINGS_MODULE=%s' % settings sh("%s python -W ignore manage.py makemigrations --noinput" % settings) sh("%s python -W ignore manage.py migrate --noinput" % settings) sh("%s python -W ignore manage.py loaddata sample_admin.json" % settings) sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json" % settings) sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json" % settings) sh("%s python -W ignore manage.py set_all_layers_alternate" % settings)
def setup_data(): """ Import sample data (from gisdata package) into GeoNode """ import gisdata ctype = options.get('type', None) data_dir = gisdata.GOOD_DATA if ctype in ['vector', 'raster', 'time']: data_dir = os.path.join(gisdata.GOOD_DATA, ctype) sh("python manage.py importlayers %s -v2" % data_dir)
def run_tests(options): """ Executes the entire test suite. """ if options.get('coverage'): prefix = 'coverage run --branch --source=geonode \ --omit="*/__init__*,*/test*,*/wsgi*,*/version*,*/migrations*,\ */search_indexes*,*/management/*,*/context_processors*,*/qgis_server/*"' else: prefix = 'python' local = options.get('local', 'false') # travis uses default to false if not integration_tests and not integration_csw_tests and not integration_bdd_tests: call_task('test', options={'prefix': prefix}) else: if integration_tests: call_task('test_integration', options={'prefix': prefix}) # only start if using Geoserver backend _backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND']) if _backend == 'geonode.geoserver' and 'geonode.geoserver' in INSTALLED_APPS: call_task( 'test_integration', options={'name': 'geonode.geoserver.integration.tests'}) elif integration_csw_tests: call_task('test_integration', options={ 'prefix': prefix, 'name': 'geonode.tests.csw' }) if integration_bdd_tests: call_task('test_bdd', options={'local': local}) sh('flake8 geonode')
def stop_qgis_server(): """ Stop QGIS Server Backend. """ # only start if using QGIS Server backend if 'geonode.qgis_server' not in INSTALLED_APPS or OGC_SERVER['default'][ 'BACKEND'] == 'geonode.geoserver': return port = options.get('qgis_server_port', '9000') sh('docker-compose -f docker-compose-qgis-server.yml down', env={ 'GEONODE_PROJECT_PATH': os.getcwd(), 'QGIS_SERVER_PORT': port })
def sloccount(): '''Print "Source Lines of Code" and export to file. Export is hudson_ plugin_ compatible: sloccount.sc requirements: - sloccount_ should be installed. - tee and pipes are used options.paved.pycheck.sloccount.param .. _sloccount: http://www.dwheeler.com/sloccount/ .. _hudson: http://hudson-ci.org/ .. _plugin: http://wiki.hudson-ci.org/display/HUDSON/SLOCCount+Plugin ''' # filter out subpackages setup = options.get('setup') packages = options.get('packages') if setup else None if packages: dirs = [x for x in packages if '.' not in x] else: dirs = ['.'] # sloccount has strange behaviour with directories, # can cause exception in hudson sloccount plugin. # Better to call it with file list ls = [] for d in dirs: ls += list(path(d).walkfiles()) # ls=list(set(ls)) files = ' '.join(ls) param = options.paved.pycheck.sloccount.param sh('sloccount {param} {files} | tee sloccount.sc'.format(param=param, files=files))
def run_tests(options): """ Executes the entire test suite. """ if options.get('coverage'): prefix = 'coverage run --branch --source=geonode --omit="*/management/*,geonode/contrib/*,*/test*,*/wsgi*,*/middleware*"' else: prefix = 'python' local = options.get('local', 'false') # travis uses default to false sh('%s manage.py test geonode.tests.smoke' % prefix) call_task('test', options={'prefix': prefix}) call_task('test_integration') call_task('test_integration', options={'name': 'geonode.tests.csw'}) # only start if using Geoserver backend if 'geonode.geoserver' in INSTALLED_APPS: call_task('test_integration', options={ 'name': 'geonode.upload.tests.integration', 'settings': 'geonode.upload.tests.test_settings' }) call_task('test_bdd', options={'local': local}) sh('flake8 geonode')
def manifest(): """Guarantee the existence of a basic MANIFEST.in. manifest doc: http://docs.python.org/distutils/sourcedist.html#manifest `options.paved.dist.manifest.include`: set of files (or globs) to include with the `include` directive. `options.paved.dist.manifest.recursive_include`: set of files (or globs) to include with the `recursive-include` directive. `options.paved.dist.manifest.prune`: set of files (or globs) to exclude with the `prune` directive. `options.paved.dist.manifest.include_sphinx_docroot`: True -> sphinx docroot is added as `graft` `options.paved.dist.manifest.include_sphinx_docroot`: True -> sphinx builddir is added as `prune` """ prune = options.paved.dist.manifest.prune graft = set() if options.paved.dist.manifest.include_sphinx_docroot: docroot = options.get('docroot', 'docs') graft.update([docroot]) if options.paved.dist.manifest.exclude_sphinx_builddir: builddir = docroot + '/' + options.get("builddir", ".build") prune.update([builddir]) with open(options.paved.cwd / 'MANIFEST.in', 'w') as fo: for item in graft: fo.write('graft %s\n' % item) for item in options.paved.dist.manifest.include: fo.write('include %s\n' % item) for item in options.paved.dist.manifest.recursive_include: fo.write('recursive-include %s\n' % item) for item in prune: fo.write('prune %s\n' % item)
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" from geonode.settings import INSTALLED_APPS # only start if using Geoserver backend if 'geonode.geoserver' not in INSTALLED_APPS: return download_dir = path('downloaded') if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path('geoserver') geoserver_bin = download_dir / \ os.path.basename(dev_config['GEOSERVER_URL']) jetty_runner = download_dir / \ os.path.basename(dev_config['JETTY_RUNNER_URL']) grab(options.get('geoserver', dev_config['GEOSERVER_URL']), geoserver_bin, "geoserver binary") grab(options.get('jetty', dev_config['JETTY_RUNNER_URL']), jetty_runner, "jetty runner") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir / 'geoserver' if not webapp_dir: webapp_dir.makedirs() print 'extracting geoserver' z = zipfile.ZipFile(geoserver_bin, "r") z.extractall(webapp_dir) _install_data_dir()
def start_qgis_server(): """Start QGIS Server instance with GeoNode related plugins.""" # only start if using QGIS Serrver backend if 'geonode.qgis_server' not in INSTALLED_APPS or OGC_SERVER['default'][ 'BACKEND'] == 'geonode.geoserver': return info('Starting up QGIS Server...') port = options.get('qgis_server_port', '9000') sh('docker-compose -f docker-compose-qgis-server.yml up -d qgis-server', env={ 'GEONODE_PROJECT_PATH': os.getcwd(), 'QGIS_SERVER_PORT': port }) info('QGIS Server is up.')
def stop_qgis_server(): """ Stop QGIS Server Backend. """ from geonode.settings import INSTALLED_APPS # only start if using QGIS Server backend if 'geonode.qgis_server' not in INSTALLED_APPS: return port = options.get('qgis_server_port', '9000') sh('docker-compose -f docker-compose-qgis-server.yml down', env={ 'GEONODE_PROJECT_PATH': os.getcwd(), 'QGIS_SERVER_PORT': port })
def sync(options): """ Run the migrate and migrate management commands to create and migrate a DB """ # for app in dev_config['MIGRATE_APPS']: # try: # sh("python manage.py migrate {app} --noinput".format(app=app)) # except: # pass settings = options.get('settings', '') if settings: settings = 'DJANGO_SETTINGS_MODULE=%s' % settings sh("%s python manage.py makemigrations --noinput" % settings) sh("%s python manage.py migrate --noinput" % settings) sh("%s python manage.py loaddata fixtures/sample_admin.json" % settings) sh("%s python manage.py loaddata fixtures/default_oauth_apps.json" % settings) sh("%s python manage.py loaddata fixtures/initial_data.json" % settings)
def test_bdd(options): """ Run GeoNode's BDD Test Suite """ local = str2bool(options.get('local', 'false')) if local: call_task('reset_hard') else: call_task('reset') call_task('setup') call_task('sync') sh('sleep 30') info("GeoNode is now available, running the bdd tests now.") sh('py.test') if local: call_task('reset_hard')
def singletest(options): """ Run GeoNode's Unit Test Suite """ GEONODE_APPS = [ 'geonode.maps.tests:MapsTest.test_map_remove', 'geonode.maps.tests:MapsTest.test_rating_map_remove', 'geonode.social.tests:SimpleTest.test_layer_activity', 'geonode.documents.tests:DocumentsTest.test_ajax_document_permissions', 'geonode.documents.tests:DocumentsTest.test_create_document_url', 'geonode.documents.tests:DocumentsTest.test_create_document_with_no_rel', 'geonode.documents.tests:DocumentsTest.test_create_document_with_rel', ] apps_to_test = ' '.join(GEONODE_APPS) prefix = options.get('prefix', 'python') sh("%s manage.py test %s --noinput --failfast" % (prefix, apps_to_test))
def test(options): """ Run GeoNode's Unit Test Suite """ if on_travis: if core_tests: _apps = GEONODE_CORE_APPS if internal_apps_tests: _apps = GEONODE_INTERNAL_APPS else: _apps = GEONODE_APPS _apps_to_test = [] for _app in _apps: if _app and len(_app) > 0 and 'geonode' in _app: _apps_to_test.append(_app) sh("%s manage.py test geonode.tests.smoke %s.tests --noinput %s %s" % (options.get('prefix'), '.tests '.join(_apps_to_test), _keepdb, _parallel))
def test_integration(options): """ Run GeoNode's Integration test suite against the external apps """ _reset() # Start GeoServer call_task('start_geoserver') info("GeoNode is now available, running the tests now.") name = options.get('name', 'geonode.tests.integration') success = False try: if name == 'geonode.tests.csw': call_task('start') sh('sleep 30') call_task('setup_data') sh(('python manage.py test %s' ' --noinput --liveserver=localhost:8000' % name)) except BuildFailure, e: info('Tests failed! %s' % str(e))
def test_bdd(): """ Run GeoNode's BDD Test Suite """ call_task('stop_geoserver') sh('sleep 30') local = str2bool(options.get('local', 'false')) if local: call_task('reset_hard') call_task('setup') call_task('sync') # Start GeoServer call_task('start_geoserver') sh('sleep 30') info("GeoNode is now available, running the bdd tests now.") sh('py.test') if local: call_task('stop_geoserver') call_task('reset_hard')
def upload(): """upload package zipfile to server""" user = options.get('user', False) if not user: raise ValueError('OSGeo userid required') password = getpass.getpass('Enter your password: '******'': raise ValueError('password required') call_task('package') zipf = get_package_filename() url = 'http://%s:%s@%s:%d/%s' % (user, password, options.upload.host, options.upload.port, options.upload.endpoint) info('Uploading to http://%s/%s' % (options.upload.host, options.upload.endpoint)) server = xmlrpc.client.ServerProxy(url, verbose=False) try: with open(zipf) as zfile: plugin_id, version_id = \ server.plugin.upload(xmlrpc.client.Binary(zfile.read())) info('Plugin ID: %s', plugin_id) info('Version ID: %s', version_id) except xmlrpc.client.Fault as err: error('ERROR: fault error') error('Fault code: %d', err.faultCode) error('Fault string: %s', err.faultString) except xmlrpc.client.ProtocolError as err: error('Error: Protocol error') error("%s : %s", err.errcode, err.errmsg) if err.errcode == 403: error('Invalid name and password')
def generate_csw_connections_file(): """generate a CSW connections file from a flat file of CSW URLs""" filename = options.get('filename', False) if not filename: raise ValueError('path to file of CSW URLs required') conns = etree.Element('qgsCSWConnections') conns.attrib['version'] = '1.0' with open(filename) as connsfh: for line in connsfh: url = line.strip() if not url: # blank line continue try: csw = CatalogueServiceWeb(url) title = unicode(csw.identification.title) conn = etree.SubElement(conns, 'csw', name=title, url=url) except Exception, err: error('ERROR on CSW %s: %s', url, err)
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" fast = options.get('fast', False) download_dir = path('downloaded') download_dir.makedirs() geoserver_dir = path('geoserver') geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL) geoserver_ext = download_dir / os.path.basename(GEONODE_GEOSERVER_EXT_URL) if not geoserver_bin.exists(): print "Downloading geoserver binary distribution" grab(GEOSERVER_URL, geoserver_bin) elif not zipfile.is_zipfile(geoserver_bin): print "Downloading geoserver binary distribution (corrupt file)" grab(GEOSERVER_URL, geoserver_bin) if not geoserver_ext.exists(): print "Downloading geonode geoserver extensions" grab(GEONODE_GEOSERVER_EXT_URL, geoserver_ext) elif not zipfile.is_zipfile(geoserver_ext): print "Re-downloading geonode geoserver extensions (corrupt file)" grab(GEONODE_GEOSERVER_EXT_URL, geoserver_ext) if not geoserver_dir.exists(): with zipfile.ZipFile(geoserver_bin, "r") as z: z.extractall(download_dir) #FIXME(Ariel): Avoid hardcoding. g = download_dir / "geoserver-2.2.3" libs_dir = g / "webapps/geoserver/WEB-INF/lib" with zipfile.ZipFile(geoserver_ext, "r") as z: z.extractall(libs_dir) # Move geoserver out of downloaded geoserver_dir.remove() shutil.move(g, geoserver_dir)
def deb(options): """ Creates debian packages. Example uses: paver deb paver deb -k 12345 paver deb -k 12345 -p geonode/testing """ key = options.get('key', None) ppa = options.get('ppa', None) version, simple_version = versions() info('Creating package for GeoNode version %s' % version) # Get rid of any uncommitted changes to debian/changelog info('Getting rid of any uncommitted changes in debian/changelog') sh('git checkout package/debian/changelog') # Workaround for git-dch bug # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580 sh('rm -rf %s/.git' % (os.path.realpath('package'))) sh('ln -s %s %s' % (os.path.realpath('.git'), os.path.realpath('package'))) with pushd('package'): # Install requirements # sh('sudo apt-get -y install debhelper devscripts git-buildpackage') # sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s' # ' --id-length=6 --ignore-branch --release' % (simple_version))) # In case you publish from Ubuntu Xenial (git-dch is removed from upstream) # use the following line instead: # sh(('gbp dch --spawn-editor=snapshot --git-author --new-version=%s' # ' --id-length=6 --ignore-branch --release' % (simple_version))) distribution = "xenial" sh(('gbp dch --distribution=%s --force-distribution --spawn-editor=snapshot --git-author --new-version=%s' ' --id-length=6 --ignore-branch --release' % (distribution, simple_version))) deb_changelog = path('debian') / 'changelog' for idx, line in enumerate( fileinput.input([deb_changelog], inplace=True)): if idx == 0: print "geonode (%s) %s; urgency=high" % (simple_version, distribution), else: print line.replace("urgency=medium", "urgency=high"), # Revert workaround for git-dhc bug sh('rm -rf .git') if key is None and ppa is None: # A local installable package sh('debuild -uc -us -A') elif key is None and ppa is not None: # A sources package, signed by daemon sh('debuild -S') elif key is not None and ppa is None: # A signed installable package sh('debuild -k%s -A' % key) elif key is not None and ppa is not None: # A signed, source package sh('debuild -k%s -S' % key) if ppa is not None: sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
def test_integration(options): """ Run GeoNode's Integration test suite against the external apps """ if 'geonode.qgis_server' not in INSTALLED_APPS or OGC_SERVER['default'][ 'BACKEND'] == 'geonode.geoserver': call_task('stop_geoserver') _reset() # Start GeoServer call_task('start_geoserver') else: call_task('stop_qgis_server') _reset() # Start QGis Server call_task('start_qgis_server') sh('sleep 30') name = options.get('name', 'geonode.tests.integration') settings = options.get('settings', '') if not settings and name == 'geonode.upload.tests.integration': settings = 'geonode.upload.tests.test_settings' success = False try: if name == 'geonode.tests.csw': call_task('sync', options={'settings': settings}) call_task('start', options={'settings': settings}) sh('sleep 30') call_task('setup_data', options={'settings': settings}) settings = 'DJANGO_SETTINGS_MODULE=%s' % settings if settings else '' if name == 'geonode.upload.tests.integration': sh("%s python -W ignore manage.py makemigrations --noinput" % settings) sh("%s python -W ignore manage.py migrate --noinput" % settings) sh("%s python -W ignore manage.py loaddata sample_admin.json" % settings) sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json" % settings) sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json" % settings) call_task('start_geoserver') bind = options.get('bind', '0.0.0.0:8000') foreground = '' if options.get('foreground', False) else '&' sh('%s python -W ignore manage.py runmessaging %s' % (settings, foreground)) sh('%s python -W ignore manage.py runserver %s %s' % (settings, bind, foreground)) sh('sleep 30') settings = 'REUSE_DB=1 %s' % settings sh(('%s python manage.py test %s' ' --noinput --liveserver=0.0.0.0:8000' % (settings, name))) except BuildFailure as e: info('Tests failed! %s' % str(e)) else: success = True finally: # don't use call task here - it won't run since it already has stop() call_task('stop_geoserver') _reset() if not success: sys.exit(1)
def start_geoserver(options): """ Start GeoServer with GeoNode extensions """ from geonode.settings import OGC_SERVER GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION'] url = "http://localhost:8080/geoserver/" if GEOSERVER_BASE_URL != url: print 'your GEOSERVER_BASE_URL does not match %s' % url sys.exit(1) download_dir = path('downloaded').abspath() jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) data_dir = path('geoserver/data').abspath() web_app = path('geoserver/geoserver').abspath() log_file = path('geoserver/jetty.log').abspath() config = path('scripts/jetty-runner.xml').abspath() # @todo - we should not have set workdir to the datadir but a bug # in geoserver # prevents geonode security from initializing correctly otherwise with pushd(data_dir): javapath = "java" loggernullpath = os.devnull # checking if our loggernullpath exists and if not, reset it to # something manageable if loggernullpath == "nul": try: open("../../downloaded/null.txt", 'w+').close() except IOError as e: print "Chances are that you have Geoserver currently running.\ Youcan either stop all servers with paver stop or start only\ the django application with paver start_django." sys.exit(1) loggernullpath = "../../downloaded/null.txt" try: sh(('java -version')) except BaseException: print "Java was not found in your path. \ Trying some other options: " javapath_opt = None if os.environ.get('JAVA_HOME', None): print "Using the JAVA_HOME environment variable" javapath_opt = os.path.join( os.path.abspath(os.environ['JAVA_HOME']), "bin", "java.exe") elif options.get('java_path'): javapath_opt = options.get('java_path') else: print "Paver cannot find java in the Windows Environment. \ Please provide the --java_path flag with your full path to \ java.exe e.g. --java_path=C:/path/to/java/bin/java.exe" sys.exit(1) # if there are spaces javapath = 'START /B "" "' + javapath_opt + '"' sh(( '%(javapath)s -Xmx512m -XX:MaxPermSize=256m' ' -DGEOSERVER_DATA_DIR=%(data_dir)s' # workaround for JAI sealed jar issue and jetty classloader ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true' ' -jar %(jetty_runner)s' ' --log %(log_file)s' ' %(config)s' ' > %(loggernullpath)s &' % locals())) info('Starting GeoServer on %s' % url) # wait for GeoServer to start started = waitfor(url) info('The logs are available at %s' % log_file) if not started: # If applications did not start in time we will give the user a chance # to inspect them and stop them manually. info(('GeoServer never started properly or timed out.' 'It may still be running in the background.')) sys.exit(1)
def start_django(): """ Start the GeoNode Django application """ bind = options.get('bind', '') sh('python manage.py runserver %s &' % bind)
def test(options): """ Run GeoNode's Unit Test Suite """ sh("%s manage.py test %s.tests --noinput" % (options.get('prefix'), '.tests '.join(GEONODE_APPS)))
def start_geoserver(options): """ Start GeoServer with GeoNode extensions """ # only start if using Geoserver backend if 'geonode.geoserver' not in INSTALLED_APPS or OGC_SERVER['default'][ 'BACKEND'] == 'geonode.qgis_server': return GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION'] url = GEOSERVER_BASE_URL if urlparse(GEOSERVER_BASE_URL).hostname != 'localhost': print "Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'" if not GEOSERVER_BASE_URL.endswith('/'): print "Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'" sys.exit(1) download_dir = path('downloaded').abspath() jetty_runner = download_dir / \ os.path.basename(dev_config['JETTY_RUNNER_URL']) data_dir = path('geoserver/data').abspath() geofence_dir = path('geoserver/data/geofence').abspath() web_app = path('geoserver/geoserver').abspath() log_file = path('geoserver/jetty.log').abspath() config = path('scripts/misc/jetty-runner.xml').abspath() jetty_port = urlparse(GEOSERVER_BASE_URL).port import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) socket_free = True try: s.bind(("127.0.0.1", jetty_port)) except socket.error as e: socket_free = False if e.errno == 98: info('Port %s is already in use' % jetty_port) else: info( 'Something else raised the socket.error exception while checking port %s' % jetty_port) print(e) finally: s.close() if socket_free: # @todo - we should not have set workdir to the datadir but a bug in geoserver # prevents geonode security from initializing correctly otherwise with pushd(data_dir): javapath = "java" loggernullpath = os.devnull # checking if our loggernullpath exists and if not, reset it to # something manageable if loggernullpath == "nul": try: open("../../downloaded/null.txt", 'w+').close() except IOError as e: print "Chances are that you have Geoserver currently running. You \ can either stop all servers with paver stop or start only \ the django application with paver start_django." sys.exit(1) loggernullpath = "../../downloaded/null.txt" try: sh(('java -version')) except BaseException: print "Java was not found in your path. Trying some other options: " javapath_opt = None if os.environ.get('JAVA_HOME', None): print "Using the JAVA_HOME environment variable" javapath_opt = os.path.join( os.path.abspath(os.environ['JAVA_HOME']), "bin", "java.exe") elif options.get('java_path'): javapath_opt = options.get('java_path') else: print "Paver cannot find java in the Windows Environment. \ Please provide the --java_path flag with your full path to \ java.exe e.g. --java_path=C:/path/to/java/bin/java.exe" sys.exit(1) # if there are spaces javapath = 'START /B "" "' + javapath_opt + '"' sh(( '%(javapath)s -Xms512m -Xmx2048m -server -XX:+UseConcMarkSweepGC -XX:MaxPermSize=512m' ' -DGEOSERVER_DATA_DIR=%(data_dir)s' ' -Dgeofence.dir=%(geofence_dir)s' # ' -Dgeofence-ovr=geofence-datasource-ovr.properties' # workaround for JAI sealed jar issue and jetty classloader # ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true' ' -jar %(jetty_runner)s' ' --port %(jetty_port)i' ' --log %(log_file)s' ' %(config)s' ' > %(loggernullpath)s &' % locals())) info('Starting GeoServer on %s' % url) # wait for GeoServer to start started = waitfor(url) info('The logs are available at %s' % log_file) if not started: # If applications did not start in time we will give the user a chance # to inspect them and stop them manually. info(('GeoServer never started properly or timed out.' 'It may still be running in the background.')) sys.exit(1)
def start_geoserver(options): """ Start GeoServer with GeoNode extensions """ from geonode.settings import OGC_SERVER GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION'] url = GEOSERVER_BASE_URL if urlparse(GEOSERVER_BASE_URL).hostname != 'localhost': print "Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'" if not GEOSERVER_BASE_URL.endswith('/'): print "Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'" sys.exit(1) download_dir = path('downloaded').abspath() jetty_runner = download_dir / os.path.basename( dev_config['JETTY_RUNNER_URL']) data_dir = path('geoserver/data').abspath() web_app = path('geoserver/geoserver').abspath() log_file = path('geoserver/jetty.log').abspath() config = path('scripts/misc/jetty-runner.xml').abspath() jetty_port = urlparse(GEOSERVER_BASE_URL).port # @todo - we should not have set workdir to the datadir but a bug in geoserver # prevents geonode security from initializing correctly otherwise with pushd(data_dir): javapath = "java" loggernullpath = os.devnull # checking if our loggernullpath exists and if not, reset it to something manageable if loggernullpath == "nul": try: open("../../downloaded/null.txt", 'w+').close() except IOError, e: print "Chances are that you have Geoserver currently running. You \ can either stop all servers with paver stop or start only \ the django application with paver start_django." sys.exit(1) loggernullpath = "../../downloaded/null.txt" try: sh(('java -version')) except: print "Java was not found in your path. Trying some other options: " javapath_opt = None if os.environ.get('JAVA_HOME', None): print "Using the JAVA_HOME environment variable" javapath_opt = os.path.join( os.path.abspath(os.environ['JAVA_HOME']), "bin", "java.exe") elif options.get('java_path'): javapath_opt = options.get('java_path') else: print "Paver cannot find java in the Windows Environment. \ Please provide the --java_path flag with your full path to \ java.exe e.g. --java_path=C:/path/to/java/bin/java.exe" sys.exit(1) # if there are spaces javapath = 'START /B "" "' + javapath_opt + '"' sh(( '%(javapath)s -Xmx512m -XX:MaxPermSize=256m' ' -DGEOSERVER_DATA_DIR=%(data_dir)s' # workaround for JAI sealed jar issue and jetty classloader ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true' ' -jar %(jetty_runner)s' ' --port %(jetty_port)i' ' --log %(log_file)s' ' %(config)s' ' > %(loggernullpath)s &' % locals()))
def test_integration(options): """ Run GeoNode's Integration test suite against the external apps """ _backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND']) if _backend == 'geonode.geoserver' or 'geonode.qgis_server' not in INSTALLED_APPS: call_task('stop_geoserver') _reset() else: call_task('stop_qgis_server') _reset() name = options.get('name', 'geonode.tests.integration') settings = options.get('settings', '') if name == 'geonode.geoserver.integration.tests': if _django_11: sh("cp geonode/upload/tests/test_settings.py geonode/") settings = 'geonode.test_settings' else: settings = 'geonode.upload.tests.test_settings' success = False try: call_task('setup', options={'settings': settings}) if name == 'geonode.tests.csw': call_task('sync', options={'settings': settings}) call_task('start', options={'settings': settings}) call_task('setup_data', options={'settings': settings}) if name == 'geonode.geoserver.integration.tests': sh("DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py makemigrations --noinput" % settings) sh("DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py migrate --noinput" % settings) sh("DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py loaddata sample_admin.json" % settings) sh("DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json" % settings) sh("DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json" % settings) call_task('start_geoserver') bind = options.get('bind', '0.0.0.0:8000') foreground = '' if options.get('foreground', False) else '&' sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runmessaging %s' % (settings, foreground)) sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runserver %s %s' % (settings, bind, foreground)) sh('sleep 30') settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings live_server_option = '--liveserver=localhost:8000' if _django_11: live_server_option = '' info("Running the tests now...") if name == 'geonode.geoserver.integration.tests': sh(('%s python -W ignore manage.py test %s' ' %s --noinput %s' % (settings, 'geonode.upload.tests.integration', _keepdb, live_server_option))) sh(('%s python -W ignore manage.py test %s' ' %s --noinput %s' % (settings, 'geonode.monitoring.tests.integration', _keepdb, live_server_option))) else: sh(('%s python -W ignore manage.py test %s' ' %s --noinput %s' % (settings, name, _keepdb, live_server_option))) except BuildFailure as e: info('Tests failed! %s' % str(e)) else: success = True finally: # don't use call task here - it won't run since it already has stop(options) _reset() if not success: sys.exit(1)
def start_messaging(): """ Start the GeoNode messaging server """ foreground = '' if options.get('foreground', False) else '&' sh('python manage.py runmessaging %s' % foreground)
def deb(options): """ Creates debian packages. Example uses: paver deb paver deb -k 12345 paver deb -k 12345 -p geonode/testing """ key = options.get('key', None) ppa = options.get('ppa', None) import geonode from geonode.version import get_git_changeset raw_version = geonode.__version__ version = geonode.get_version() timestamp = get_git_changeset() major, minor, revision, stage, edition = raw_version branch = 'dev' if stage == 'alpha' and edition == 0: tail = '%s%s' % (branch, timestamp) else: tail = '%s%s' % (stage, edition) simple_version = '%s.%s.%s+%s' % (major, minor, revision, tail) info('Creating package for GeoNode version %s' % version) with pushd('package'): # Get rid of any uncommitted changes to debian/changelog info('Getting rid of any uncommitted changes in debian/changelog') sh('git checkout debian/changelog') # Workaround for git-dch bug # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580 path('.git').makedirs() # Install requirements #sh('sudo apt-get -y install debhelper devscripts git-buildpackage') sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s' ' --id-length=6 --ignore-branch --release' % ( simple_version))) ## Revert workaround for git-dhc bug path('.git').rmtree() if key is None and ppa is None: # A local installable package sh('debuild -uc -us -A') elif key is None and ppa is not None: # A sources package, signed by daemon sh('debuild -S') elif key is not None and ppa is None: # A signed installable package sh('debuild -k%s -A' % key) elif key is not None and ppa is not None: # A signed, source package sh('debuild -k%s -S' % key) if ppa is not None: sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))