def test_safelint_violation_number_not_found(self, _mock_counts, _mock_report_dir, _mock_write_metric):
"""
run_safelint encounters an error parsing the safelint output log
"""
_mock_counts.return_value = {}
with self.assertRaises(SystemExit):
call_task('pavelib.quality.run_safelint')
def localpi():
"""Make local package index (used by tox)."""
download_dir = path(options.develop.download_dir)
if not download_dir.exists():
call_task("download_depends")
info("MAKE LOCAL PACKAGE-INDEX: %s/" % download_dir)
sh("dir2pi %s" % download_dir)
def test_safelint_under_rule_limit(self, _mock_counts, _mock_report_dir, _mock_write_metric):
"""
run_safelint finds fewer rule violations than are allowed
"""
_mock_counts.return_value = {'total': 4, 'rules': {'javascript-escape': 4}}
# No System Exit is expected
call_task('pavelib.quality.run_safelint', options={"thresholds": '{"rules": {"javascript-escape": 5}}'})
def verify_server_task(self, task_name, options, contracts_default=False):
"""
Verify the output of a server task.
"""
settings = options.get("settings", None)
asset_settings = options.get("asset-settings", None)
is_optimized = options.get("optimized", False)
is_fast = options.get("fast", False)
no_contracts = options.get("no-contracts", not contracts_default)
if task_name == "devstack":
system = options.get("system")
elif task_name == "studio":
system = "cms"
else:
system = "lms"
port = options.get("port", "8000" if system == "lms" else "8001")
self.reset_task_messages()
if task_name == "devstack":
args = ["studio" if system == "cms" else system]
if settings:
args.append("--settings={settings}".format(settings=settings))
if asset_settings:
args.append("--asset-settings={asset_settings}".format(asset_settings=asset_settings))
if is_optimized:
args.append("--optimized")
if is_fast:
args.append("--fast")
if no_contracts:
args.append("--no-contracts")
call_task("pavelib.servers.devstack", args=args)
else:
call_task("pavelib.servers.{task_name}".format(task_name=task_name), options=options)
expected_messages = options.get("expected_messages", [])
expected_settings = settings if settings else "devstack"
expected_asset_settings = asset_settings if asset_settings else expected_settings
if is_optimized:
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != "devstack"
if not is_fast:
expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format(
system=system, asset_settings=expected_asset_settings
))
expected_messages.append(u"xmodule_assets common/static/xmodule")
expected_messages.append(u"install npm_assets")
expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=self.platform_root))
expected_messages.extend(self.expected_sass_commands(system=system))
if expected_collect_static:
expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
system=system, asset_settings=expected_asset_settings
))
expected_run_server_command = EXPECTED_RUN_SERVER_COMMAND.format(
system=system,
settings=expected_settings,
port=port,
)
if not no_contracts:
expected_run_server_command += " --contracts"
expected_messages.append(expected_run_server_command)
self.assertEquals(self.task_messages, expected_messages)
def run_integration_tests():
"""
Executes the entire test suite.
"""
call_task('start')
sh('python manage.py test tests.integration --settings=settings.test --failfast')
call_task('stop')
def gem_check(args):
"""Install a gem if it's not installed"""
to_install = [gem for gem in args if gem not in installed_gems]
if to_install:
call_task('gem_install', args=to_install)
else:
debug('Gems are synchonized')
def run_tests(options):
"""
Executes the entire test suite.
"""
if options.get('coverage'):
prefix = 'coverage run --branch --source=geonode --omit="*/management/*,geonode/contrib/*,*/test*,*/wsgi*,*/middleware*"'
else:
prefix = 'python'
local = options.get('local', 'false') # travis uses default to false
if not integration_tests:
sh('%s manage.py test geonode.tests.smoke %s %s' % (prefix, _keepdb, _parallel))
call_task('test', options={'prefix': prefix})
else:
call_task('test_integration')
call_task('test_integration', options={'name': 'geonode.tests.csw'})
# only start if using Geoserver backend
if 'geonode.geoserver' in INSTALLED_APPS and OGC_SERVER['default']['BACKEND'] == 'geonode.geoserver':
call_task('test_integration',
options={'name': 'geonode.upload.tests.integration',
'settings': 'geonode.upload.tests.test_settings'})
call_task('test_bdd', options={'local': local})
sh('flake8 geonode')
def test_safelint_invalid_thresholds_option_key(self, _mock_counts, _mock_report_dir, _mock_write_metric):
"""
run_safelint fails when thresholds option is poorly formatted
"""
_mock_counts.return_value = {'total': 0}
with self.assertRaises(SystemExit):
call_task('pavelib.quality.run_safelint', options={"thresholds": '{"invalid": 3}'})
def clean_migrations(env):
"""
Merge uncommitted migrations
"""
migration_re = re.compile('^\d{4}_[a-z0-9_]+\.py$')
git_status = subprocess.Popen([
which.git, 'ls-files', '--others', '--exclude-standard',
],
stdout=subprocess.PIPE,
)
encoding = sys.getfilesystemencoding()
matches = collections.defaultdict(list)
for line in git_status.stdout:
file_path = path(line.strip().decode(encoding))
if '/migrations/' in file_path and migration_re.match(file_path.basename()):
matches[file_path.splitall()[1]].append(file_path)
git_status.wait()
for app_label, scripts in matches.items():
if len(scripts) <= 1:
scripts.clear()
if not any(matches.values()):
debug('No uncommited migration to merge')
for scripts in matches.values():
for script in scripts:
script.remove()
if not env.dry_run:
args = ['makemigrations']
args.extend(m for m in matches if matches[m])
call_task('django', args=args)
def test_diff(args):
"""
Run tests changes since master with coverage.
"""
tests = []
project_tests = os.path.join(SETUP['folders']['source'], 'tests')
for change, path in pave.git.diffFileNames():
if change == 'd':
continue
if project_tests not in path:
# Not a test.
continue
parts = path.split(os.sep)
module = parts[-1][:-3]
if module == '__init__':
# We don't care about init.
continue
parts = parts[3:-1] + [module]
tests.append('.'.join(parts))
if not tests:
print('No test changes detected.')
return
args = tests + args
# Trigger coverage creation.
os.environ['CODECOV_TOKEN'] = 'local'
call_task('test', args=args)
_generate_coverate_reports()
def test_safelint_too_many_violations(self, _mock_counts, _mock_report_dir, _mock_write_metric):
"""
run_safelint finds more violations than are allowed
"""
_mock_counts.return_value = {'total': 4}
with self.assertRaises(SystemExit):
call_task('pavelib.quality.run_safelint', options={"thresholds": '{"total": 3}'})
def test_celery(self, options):
"""
Test the "celery" task.
"""
settings = options.get("settings", "dev_with_worker")
call_task("pavelib.servers.celery", options=options)
self.assertEquals(self.task_messages, [EXPECTED_CELERY_COMMAND.format(settings=settings)])
def dmg(options):
try:
pyver = options.dmg.python_version
except:
pyver = DEFAULT_PYTHON
idirs = options.installers.installersdir
# Check if docs exist. If not, say so and quit.
ref = os.path.join(options.doc.destdir_pdf, "reference.pdf")
user = os.path.join(options.doc.destdir_pdf, "userguide.pdf")
if (not os.path.exists(ref)) or (not os.path.exists(user)):
import warnings
warnings.warn("Docs need to be built first! Can't find them.")
# Build the mpkg package
call_task("clean")
_build_mpkg(pyver)
macosx_installer_dir = "tools/numpy-macosx-installer"
dmg = os.path.join(macosx_installer_dir, dmg_name(FULLVERSION, pyver))
if os.path.exists(dmg):
os.remove(dmg)
# Clean the image source
content = os.path.join(macosx_installer_dir, "content")
if os.path.exists(content):
shutil.rmtree(content)
os.makedirs(content)
# Copy mpkg into image source
mpkg_source = os.path.join("dist", mpkg_name(pyver))
mpkg_target = os.path.join(content, "numpy-%s-py%s.mpkg" % (FULLVERSION, pyver))
shutil.copytree(mpkg_source, mpkg_target)
# Copy docs into image source
pdf_docs = os.path.join(content, "Documentation")
if os.path.exists(pdf_docs):
shutil.rmtree(pdf_docs)
os.makedirs(pdf_docs)
shutil.copy(user, os.path.join(pdf_docs, "userguide.pdf"))
shutil.copy(ref, os.path.join(pdf_docs, "reference.pdf"))
# Build the dmg
cmd = [
"./new-create-dmg",
"--pkgname",
os.path.basename(mpkg_target),
"--volname",
"numpy",
os.path.basename(dmg),
"./content",
]
st = subprocess.check_call(cmd, cwd=macosx_installer_dir)
source = dmg
target = os.path.join(idirs, os.path.basename(dmg))
if not os.path.exists(os.path.dirname(target)):
os.makedirs(os.path.dirname(target))
shutil.copy(source, target)
def test_watch_theme_assets(self):
"""
Test the Paver watch asset tasks with theming enabled.
"""
self.expected_sass_directories.extend([
path(TEST_THEME_DIR) / 'lms/static/sass',
path(TEST_THEME_DIR) / 'lms/static/sass/partials',
path(TEST_THEME_DIR) / 'cms/static/sass',
path(TEST_THEME_DIR) / 'cms/static/sass/partials',
])
with patch('pavelib.assets.SassWatcher.register') as mock_register:
with patch('pavelib.assets.Observer.start'):
with patch('pavelib.assets.execute_webpack_watch') as mock_webpack:
call_task(
'pavelib.assets.watch_assets',
options={
"background": True,
"theme_dirs": [TEST_THEME_DIR.dirname()],
"themes": [TEST_THEME_DIR.basename()]
},
)
self.assertEqual(mock_register.call_count, 2)
self.assertEqual(mock_webpack.call_count, 1)
sass_watcher_args = mock_register.call_args_list[0][0]
self.assertIsInstance(sass_watcher_args[0], Observer)
self.assertIsInstance(sass_watcher_args[1], list)
self.assertItemsEqual(sass_watcher_args[1], self.expected_sass_directories)
def run_tests(options):
"""
Executes the entire test suite.
"""
if options.get('coverage'):
prefix = 'coverage run --branch --source=geonode \
--omit="*/management/*,*/__init__*,*/views*,*/signals*,*/tasks*,*/test*,*/wsgi*,*/middleware*,*/search_indexes*,\
*/migrations*,*/context_processors*,geonode/qgis_server/*,geonode/upload/*,geonode/monitoring/*"'
else:
prefix = 'python'
local = options.get('local', 'false') # travis uses default to false
if not integration_tests and not integration_csw_tests and not integration_bdd_tests:
call_task('test', options={'prefix': prefix})
else:
if integration_tests:
call_task('test_integration')
# only start if using Geoserver backend
_backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND'])
if _backend == 'geonode.geoserver' and 'geonode.geoserver' in INSTALLED_APPS:
call_task('test_integration',
options={'name': 'geonode.upload.tests.integration'})
elif integration_csw_tests:
call_task('test_integration', options={'name': 'geonode.tests.csw'})
if integration_bdd_tests:
call_task('test_bdd', options={'local': local})
sh('flake8 geonode')
def test_safelint_under_limit(self, _mock_counts, _mock_report_dir, _mock_write_metric):
"""
run_safelint finds fewer violations than are allowed
"""
_mock_counts.return_value = {'total': 4}
# No System Exit is expected
call_task('pavelib.quality.run_safelint', options={"thresholds": '{"total": 5}'})
def bootstrap():
"""create virtualenv in ./install"""
install = paver.path.path(BOOTSTRAP_DIR)
if not install.exists():
install.mkdir()
call_task('paver.virtual.bootstrap')
sh('cd %s; %s bootstrap.py' % (BOOTSTRAP_DIR, sys.executable))
def publish():
if 'GPG_KEY_GEONODE' in os.environ:
key = os.environ['GPG_KEY_GEONODE']
else:
print "You need to set the GPG_KEY_GEONODE environment variable"
return
if 'PPA_GEONODE' in os.environ:
ppa = os.environ['PPA_GEONODE']
else:
ppa = None
call_task('deb', options={
'key': key,
'ppa': ppa,
# 'ppa': 'geonode/testing',
# 'ppa': 'geonode/unstable',
})
version, simple_version = versions()
if ppa:
sh('git add package/debian/changelog')
sh('git commit -m "Updated changelog for version %s"' % version)
sh('git tag -f %s' % version)
sh('git push origin %s' % version)
sh('git tag -f debian/%s' % simple_version)
sh('git push origin debian/%s' % simple_version)
# sh('git push origin master')
sh('python setup.py sdist upload -r pypi')
def start():
"""
Start GeoNode (Django, GeoServer & Client)
"""
call_task('start_messaging')
info("GeoNode is now available.")
def coverage(options):
"""
Build the html, xml, and diff coverage reports
"""
report_dir = Env.REPORT_DIR
rcfile = Env.PYTHON_COVERAGERC
if not (report_dir / '.coverage').isfile():
# This may be that the coverage files were generated using -p,
# try to combine them to the one file that we need.
sh("coverage combine --rcfile={}".format(rcfile))
if not os.path.getsize(report_dir / '.coverage') > 50:
# Check if the .coverage data file is larger than the base file,
# because coverage combine will always at least make the "empty" data
# file even when there isn't any data to be combined.
err_msg = colorize(
'red',
"No coverage info found. Run `paver test` before running "
"`paver coverage`.\n"
)
sys.stderr.write(err_msg)
return
# Generate the coverage.py XML report
sh("coverage xml --rcfile={}".format(rcfile))
# Generate the coverage.py HTML report
sh("coverage html --rcfile={}".format(rcfile))
call_task('diff_coverage', options=options.coverage)
def upload():
"""upload package zipfile to server"""
user = options.get("user", False)
if not user:
raise ValueError("OSGeo userid required")
password = getpass.getpass("Enter your password: "******"":
raise ValueError("password required")
call_task("package")
zipf = get_package_filename()
url = "http://%s:%s@%s:%d/%s" % (user, password, options.upload.host, options.upload.port, options.upload.endpoint)
info("Uploading to http://%s/%s" % (options.upload.host, options.upload.endpoint))
server = xmlrpclib.ServerProxy(url, verbose=False)
try:
with open(zipf) as zfile:
plugin_id, version_id = server.plugin.upload(xmlrpclib.Binary(zfile.read()))
info("Plugin ID: %s", plugin_id)
info("Version ID: %s", version_id)
except xmlrpclib.Fault as err:
error("ERROR: fault error")
error("Fault code: %d", err.faultCode)
error("Fault string: %s", err.faultString)
except xmlrpclib.ProtocolError as err:
error("Error: Protocol error")
error("%s : %s", err.errcode, err.errmsg)
if err.errcode == 403:
error("Invalid name and password")
def test_compile_sass(self, options):
"""
Test the "compile_sass" task.
"""
parameters = options.split(" ")
system = []
if "--system=studio" not in parameters:
system += ["lms"]
if "--system=lms" not in parameters:
system += ["studio"]
debug = "--debug" in parameters
force = "--force" in parameters
self.reset_task_messages()
call_task('pavelib.assets.compile_sass', options={"system": system, "debug": debug, "force": force})
expected_messages = []
if force:
expected_messages.append("rm -rf common/static/css/*.css")
expected_messages.append("libsass common/static/sass")
if "lms" in system:
if force:
expected_messages.append("rm -rf lms/static/css/*.css")
expected_messages.append("libsass lms/static/sass")
if force:
expected_messages.append("rm -rf lms/static/certificates/css/*.css")
expected_messages.append("libsass lms/static/certificates/sass")
if "studio" in system:
if force:
expected_messages.append("rm -rf cms/static/css/*.css")
expected_messages.append("libsass cms/static/sass")
self.assertEquals(self.task_messages, expected_messages)
def test_safelint_under_limit(self, _mock_count, _mock_report_dir, _mock_write_metric):
"""
run_safelint finds fewer violations than are allowed
"""
_mock_count.return_value = 4
# No System Exit is expected
call_task('pavelib.quality.run_safelint', options={"limit": "5"})
def _update_assets(self):
args = [self.system, '--settings=acceptance']
if self.fasttest:
args.append('--skip-collect')
call_task('pavelib.assets.update_assets', args=args)
def test_safelint_too_many_violations(self, _mock_count, _mock_report_dir, _mock_write_metric):
"""
run_safelint finds more violations than are allowed
"""
_mock_count.return_value = 4
with self.assertRaises(SystemExit):
call_task('pavelib.quality.run_safelint', options={"limit": "3"})
def bdist_wininst_simple():
"""Simple wininst-based installer."""
call_task("clean")
env = os.environ.copy()
for k, v in SITECFG['nosse'].items():
env[k] = v
_bdist_wininst(options.bdist_wininst_simple.python_version, env)
def after_bootstrap():
info("under construction")
call_task("dir_layout")
call_task("install_c_base")
call_task("install_postgis")
call_task("write_supervisor_config")
call_task("save_cfg")
def bootstrap():
"""create virtualenv in ./install"""
install = paver.path.path('install')
if not install.exists():
install.mkdir()
call_task('paver.virtual.bootstrap')
sh('cd install; %s bootstrap.py' % sys.executable)
def upload():
"""upload package zipfile to server"""
user = options.get('user', False)
if not user:
raise ValueError('OSGeo userid required')
password = getpass.getpass('Enter your password: '******'':
raise ValueError('password required')
call_task('package')
zipf = get_package_filename()
url = 'http://%s:%s@%s:%d/%s' % (user, password, options.upload.host,
options.upload.port,
options.upload.endpoint)
info('Uploading to http://%s/%s' % (options.upload.host,
options.upload.endpoint))
server = xmlrpclib.ServerProxy(url, verbose=False)
try:
with open(zipf) as zfile:
plugin_id, version_id = \
server.plugin.upload(xmlrpclib.Binary(zfile.read()))
info('Plugin ID: %s', plugin_id)
info('Version ID: %s', version_id)
except xmlrpclib.Fault, err:
error('ERROR: fault error')
error('Fault code: %d', err.faultCode)
error('Fault string: %s', err.faultString)
def bdist_mpkg(options):
call_task("clean")
try:
pyver = options.bdist_mpkg.python_version
except AttributeError:
pyver = options.python_version
_build_mpkg(pyver)
def all_checkers(args):
"""Run all python checkers."""
call_task("pychecker")
call_task("pylint")
def setup():
"""setup plugin dependencies"""
config_file = options.base.home / 'GeoHealthCheck/config_main.py'
config_site = options.base.instance / 'config_site.py'
# setup dirs
if not os.path.exists(options.base.static_lib):
options.base.static_lib.mkdir()
if not os.path.exists(options.base.instance):
options.base.instance.mkdir()
data_dir = options.base.instance / 'data'
data_dir.mkdir()
# data_dir.chmod(0777) gives failure on Python 2.7 Paver 1.2.1
os.chmod(path(data_dir), 0777)
# setup config
config_file.copy(config_site)
# setup deps
sh('pip install -r requirements.txt')
skin = 'http://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v3.3.7+1.zip' # noqa
skin_dirs = ['dist', 'vendor']
need_to_fetch = False
for skin_dir in skin_dirs:
skin_dir_path = os.sep.join(
['startbootstrap-sb-admin-2-3.3.7-1', skin_dir])
if not os.path.exists(skin_dir_path):
need_to_fetch = True
if need_to_fetch:
zipstr = StringIO(urlopen(skin).read())
zipfile_obj = zipfile.ZipFile(zipstr)
zipfile_obj.extractall(options.base.static_lib)
for zf_mem in skin_dirs:
src_loc = path(options.base.static_lib /
'startbootstrap-sb-admin-2-3.3.7-1' / zf_mem)
dest_loc = path(options.base.static_lib / zf_mem)
if not os.path.exists(dest_loc):
src_loc.move(dest_loc)
else:
info('directory already exists. Skipping')
shutil.rmtree(path(options.base.static_lib /
'startbootstrap-sb-admin-2-3.3.7-1'))
# install sparklines to static/site/js
with open(path(options.base.static_lib / 'jspark.js'), 'w') as f:
content = urlopen('http://ejohn.org/files/jspark.js').read()
content.replace('red', 'green')
f.write(content)
# install bootstrap-tagsinput to static/lib
select2 = 'https://github.com/select2/select2/archive/4.0.3.zip'
zipstr = StringIO(urlopen(select2).read())
zipfile_obj = zipfile.ZipFile(zipstr)
zipfile_obj.extractall(options.base.static_lib)
dirname = glob.glob(options.base.static_lib / 'select2-*')[0]
dstdir = ''.join(dirname.rsplit('-', 1)[:-1])
try:
os.rename(dirname, dstdir)
except OSError:
shutil.rmtree(dstdir)
os.rename(dirname, dstdir)
# install leafletjs to static/lib
leafletjs = 'http://cdn.leafletjs.com/downloads/leaflet-0.7.5.zip'
zipstr = StringIO(urlopen(leafletjs).read())
zipfile_obj = zipfile.ZipFile(zipstr)
zipfile_obj.extractall(options.base.static_lib / 'leaflet')
# install html5shiv to static/lib
with open(path(options.base.static_lib / 'html5shiv.min.js'), 'w') as f:
url = 'http://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js'
content = urlopen(url).read()
f.write(content)
# install respond to static/lib
with open(path(options.base.static_lib / 'respond.min.js'), 'w') as f:
url = 'http://oss.maxcdn.com/respond/1.4.2/respond.min.js'
content = urlopen(url).read()
f.write(content)
# build i18n .mo files
call_task('compile_translations')
# build local docs
call_task('refresh_docs')
# message user
info('GeoHealthCheck is now built. Edit settings in %s' % config_site)
info('before deploying the application. Alternatively, you can start a')
info('development instance with "python GeoHealthCheck/app.py"')
def verify_server_task(self, task_name, options):
"""
Verify the output of a server task.
"""
log_string = options.get("log_string", "> /dev/null")
settings = options.get("settings", None)
asset_settings = options.get("asset-settings", None)
is_optimized = options.get("optimized", False)
is_fast = options.get("fast", False)
if task_name == "devstack":
system = options.get("system")
elif task_name == "studio":
system = "cms"
else:
system = "lms"
port = options.get("port", "8000" if system == "lms" else "8001")
self.reset_task_messages()
if task_name == "devstack":
args = ["studio" if system == "cms" else system]
if settings:
args.append(f"--settings={settings}")
if asset_settings:
args.append(f"--asset-settings={asset_settings}")
if is_optimized:
args.append("--optimized")
if is_fast:
args.append("--fast")
call_task("pavelib.servers.devstack", args=args)
else:
call_task(f"pavelib.servers.{task_name}", options=options)
expected_messages = options.get("expected_messages", [])
expected_settings = settings if settings else Env.DEVSTACK_SETTINGS
expected_asset_settings = asset_settings if asset_settings else expected_settings
if is_optimized:
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS
if not is_fast:
expected_messages.append("xmodule_assets common/static/xmodule")
expected_messages.append("install npm_assets")
expected_messages.extend([
c.format(settings=expected_asset_settings,
log_file=Env.PRINT_SETTINGS_LOG_FILE)
for c in EXPECTED_PRINT_SETTINGS_COMMAND
])
expected_messages.append(
EXPECTED_WEBPACK_COMMAND.format(
node_env="production",
static_root_lms=None,
static_root_cms=None,
js_env_extra_config=json.dumps("{}"),
webpack_config_path=None))
expected_messages.extend(
self.expected_sass_commands(
system=system, asset_settings=expected_asset_settings))
if expected_collect_static:
expected_messages.append(
EXPECTED_COLLECT_STATIC_COMMAND.format(
system=system,
asset_settings=expected_asset_settings,
log_string=log_string))
expected_run_server_command = EXPECTED_RUN_SERVER_COMMAND.format(
system=system,
settings=expected_settings,
port=port,
)
expected_messages.append(expected_run_server_command)
assert self.task_messages == expected_messages
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
_backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND'])
if _backend == 'geonode.geoserver' or 'geonode.qgis_server' not in INSTALLED_APPS:
call_task('stop_geoserver')
_reset()
# Start GeoServer
call_task('start_geoserver')
else:
call_task('stop_qgis_server')
_reset()
# Start QGis Server
call_task('start_qgis_server')
sh('sleep 30')
name = options.get('name', 'geonode.tests.integration')
settings = options.get('settings', '')
if not settings and name == 'geonode.upload.tests.integration':
if _django_11:
sh("cp geonode/upload/tests/test_settings.py geonode/")
settings = 'geonode.test_settings'
else:
settings = 'geonode.upload.tests.test_settings'
success = False
try:
if name == 'geonode.tests.csw':
call_task('sync', options={'settings': settings})
call_task('start', options={'settings': settings})
call_task('setup_data', options={'settings': settings})
settings = 'DJANGO_SETTINGS_MODULE=%s' % settings if settings else ''
if name == 'geonode.upload.tests.integration':
sh("%s python -W ignore manage.py makemigrations --noinput" %
settings)
sh("%s python -W ignore manage.py migrate --noinput" % settings)
sh("%s python -W ignore manage.py loaddata sample_admin.json" %
settings)
sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json"
% settings)
sh("%s python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json"
% settings)
call_task('start_geoserver')
bind = options.get('bind', '0.0.0.0:8000')
foreground = '' if options.get('foreground', False) else '&'
sh('%s python -W ignore manage.py runmessaging %s' %
(settings, foreground))
sh('%s python -W ignore manage.py runserver %s %s' %
(settings, bind, foreground))
sh('sleep 30')
settings = 'REUSE_DB=1 %s' % settings
live_server_option = '--liveserver=localhost:8000'
if _django_11:
live_server_option = ''
info("GeoNode is now available, running the tests now.")
sh(('%s python -W ignore manage.py test %s'
' %s --noinput %s' %
(settings, name, _keepdb, live_server_option)))
except BuildFailure as e:
info('Tests failed! %s' % str(e))
else:
success = True
finally:
# don't use call task here - it won't run since it already has
stop()
call_task('stop_geoserver')
_reset()
if not success:
sys.exit(1)
def update_translations():
"""update language strings"""
call_task('extract_translations')
sh('pybabel update -i %st -d %s' % (
options.base.pot, options.base.translations))
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
prefix = options.get('prefix')
local = str2bool(options.get('local', 'false'))
if local:
call_task('stop_geoserver')
_reset()
name = options.get('name', None)
settings = options.get('settings', '')
success = False
try:
call_task('setup', options={'settings': settings, 'force_exec': True})
if not settings:
settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=geonode.settings'
if name and name in ('geonode.tests.csw', 'geonode.tests.integration',
'geonode.geoserver.tests.integration'):
call_task('sync', options={'settings': settings})
if local:
call_task('start_geoserver',
options={
'settings': settings,
'force_exec': True
})
call_task('start', options={'settings': settings})
if integration_server_tests:
call_task('setup_data', options={'settings': settings})
elif 'geonode.geoserver' in INSTALLED_APPS:
if local:
sh("cp geonode/upload/tests/test_settings.py geonode/")
settings = 'geonode.test_settings'
sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py "
"makemigrations --noinput")
sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py "
"migrate --noinput")
sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py "
"loaddata sample_admin.json")
sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py "
"loaddata geonode/base/fixtures/default_oauth_apps.json")
sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py "
"loaddata geonode/base/fixtures/initial_data.json")
call_task('start_geoserver')
bind = options.get('bind', '0.0.0.0:8000')
foreground = '' if options.get('foreground', False) else '&'
sh(f'DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py runmessaging {foreground}'
)
sh(f'DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py runserver {bind} {foreground}'
)
sh('sleep 30')
settings = f'REUSE_DB=1 DJANGO_SETTINGS_MODULE={settings}'
else:
call_task('sync', options={'settings': settings})
live_server_option = ''
info("Running the tests now...")
sh(f'{settings} {prefix} manage.py test {name} -v 3 {_keepdb} --noinput {live_server_option}'
)
except BuildFailure as e:
info(f'Tests failed! {str(e)}')
else:
success = True
finally:
if local:
stop(options)
_reset()
if not success:
sys.exit(1)
def test_bdd():
"""
Run GeoNode's BDD Test Suite
"""
local = str2bool(options.get('local', 'false'))
if local:
call_task('reset_hard')
call_task('setup')
else:
call_task('reset')
call_task('setup')
call_task('sync')
sh('sleep 30')
info("GeoNode is now available, running the bdd tests now.")
sh('py.test')
if local:
call_task('reset_hard')
def verify_run_all_servers_task(self, options):
"""
Verify the output of a server task.
"""
log_string = options.get("log_string", "> /dev/null")
settings = options.get("settings", None)
asset_settings = options.get("asset_settings", None)
is_optimized = options.get("optimized", False)
is_fast = options.get("fast", False)
self.reset_task_messages()
call_task("pavelib.servers.run_all_servers", options=options)
expected_settings = settings if settings else Env.DEVSTACK_SETTINGS
expected_asset_settings = asset_settings if asset_settings else expected_settings
if is_optimized:
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS
expected_messages = []
if not is_fast:
expected_messages.append(u"xmodule_assets common/static/xmodule")
expected_messages.append(u"install npm_assets")
expected_messages.append(
EXPECTED_COFFEE_COMMAND.format(
platform_root=self.platform_root))
expected_messages.extend([
c.format(settings=expected_asset_settings)
for c in EXPECTED_PRINT_SETTINGS_COMMAND
])
expected_messages.append(
EXPECTED_WEBPACK_COMMAND.format(
node_env="production"
if expected_asset_settings != Env.DEVSTACK_SETTINGS else
"development",
static_root_lms=None,
static_root_cms=None,
webpack_config_path=None))
expected_messages.extend(
self.expected_sass_commands(
asset_settings=expected_asset_settings))
if expected_collect_static:
expected_messages.append(
EXPECTED_COLLECT_STATIC_COMMAND.format(
system="lms",
asset_settings=expected_asset_settings,
log_string=log_string))
expected_messages.append(
EXPECTED_COLLECT_STATIC_COMMAND.format(
system="cms",
asset_settings=expected_asset_settings,
log_string=log_string))
expected_messages.append(
EXPECTED_RUN_SERVER_COMMAND.format(
system="lms",
settings=expected_settings,
port=8000,
))
expected_messages.append(
EXPECTED_RUN_SERVER_COMMAND.format(
system="cms",
settings=expected_settings,
port=8001,
))
expected_messages.append(
EXPECTED_CELERY_COMMAND.format(settings="devstack_with_worker"))
self.assertEquals(self.task_messages, expected_messages)
def test_xsslint_vanilla(self, _mock_counts, _mock_report_dir, _mock_write_metric):
"""
run_xsslint finds violations, but a limit was not set
"""
_mock_counts.return_value = {'total': 0}
call_task('pavelib.quality.run_xsslint')
def verify_server_task(self, task_name, options, contracts_default=False):
"""
Verify the output of a server task.
"""
log_string = options.get("log_string", "> /dev/null")
settings = options.get("settings", None)
asset_settings = options.get("asset-settings", None)
is_optimized = options.get("optimized", False)
is_fast = options.get("fast", False)
no_contracts = options.get("no-contracts", not contracts_default)
if task_name == "devstack":
system = options.get("system")
elif task_name == "studio":
system = "cms"
else:
system = "lms"
port = options.get("port", "8000" if system == "lms" else "8001")
self.reset_task_messages()
if task_name == "devstack":
args = ["studio" if system == "cms" else system]
if settings:
args.append("--settings={settings}".format(settings=settings))
if asset_settings:
args.append("--asset-settings={asset_settings}".format(
asset_settings=asset_settings))
if is_optimized:
args.append("--optimized")
if is_fast:
args.append("--fast")
if no_contracts:
args.append("--no-contracts")
call_task("pavelib.servers.devstack", args=args)
else:
call_task(
"pavelib.servers.{task_name}".format(task_name=task_name),
options=options)
expected_messages = options.get("expected_messages", [])
expected_settings = settings if settings else "devstack"
expected_asset_settings = asset_settings if asset_settings else expected_settings
if is_optimized:
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != "devstack"
if not is_fast:
expected_messages.append(u"xmodule_assets common/static/xmodule")
expected_messages.append(u"install npm_assets")
expected_messages.append(
EXPECTED_COFFEE_COMMAND.format(
platform_root=self.platform_root))
expected_messages.extend(
self.expected_sass_commands(
system=system, asset_settings=expected_asset_settings))
if expected_collect_static:
expected_messages.append(
EXPECTED_COLLECT_STATIC_COMMAND.format(
system=system,
asset_settings=expected_asset_settings,
log_string=log_string))
expected_run_server_command = EXPECTED_RUN_SERVER_COMMAND.format(
system=system,
settings=expected_settings,
port=port,
)
if not no_contracts:
expected_run_server_command += " --contracts"
expected_messages.append(expected_run_server_command)
self.assertEquals(self.task_messages, expected_messages)
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
call_task('stop_geoserver')
_reset()
# Start GeoServer
call_task('start_geoserver')
info("GeoNode is now available, running the tests now.")
name = options.get('name', 'geonode.tests.integration')
settings = options.get('settings', '')
if not settings and name == 'geonode.upload.tests.integration':
settings = 'geonode.upload.tests.test_settings'
success = False
try:
if name == 'geonode.tests.csw':
call_task('sync', options={'settings': settings})
call_task('start', options={'settings': settings})
sh('sleep 30')
call_task('setup_data', options={'settings': settings})
settings = 'DJANGO_SETTINGS_MODULE=%s' % settings if settings else ''
if name == 'geonode.upload.tests.integration':
sh("%s python manage.py makemigrations --noinput" % settings)
sh("%s python manage.py migrate --noinput" % settings)
sh("%s python manage.py loaddata sample_admin.json" % settings)
sh("%s python manage.py loaddata geonode/base/fixtures/default_oauth_apps.json" % settings)
sh("%s python manage.py loaddata geonode/base/fixtures/initial_data.json" % settings)
call_task('start_geoserver')
bind = options.get('bind', '0.0.0.0:8000')
foreground = '' if options.get('foreground', False) else '&'
sh('%s python manage.py runmessaging %s' % (settings, foreground))
sh('%s python manage.py runserver %s %s' % (settings, bind, foreground))
sh('sleep 30')
settings = 'REUSE_DB=1 %s' % settings
sh(('%s python manage.py test %s'
' --noinput --liveserver=0.0.0.0:8000' % (settings, name)))
except BuildFailure as e:
info('Tests failed! %s' % str(e))
else:
success = True
finally:
# don't use call task here - it won't run since it already has
stop()
call_task('stop_geoserver')
_reset()
if not success:
sys.exit(1)
def _update_assets(self):
"""
Internal helper method to manage asset compilation
"""
args = [self.system, '--settings={}'.format(self.settings)]
call_task('pavelib.assets.update_assets', args=args)
def run_tests(options):
"""
Executes the entire test suite.
"""
if options.get('coverage'):
prefix = 'coverage run --branch --source=geonode \
--omit="*/__init__*,*/test*,*/wsgi*,*/version*,*/migrations*,\
*/search_indexes*,*/management/*,*/context_processors*,*/upload/*,*/qgis_server/*"'
else:
prefix = 'python'
local = options.get('local', 'false') # travis uses default to false
if not integration_tests and not integration_csw_tests and not integration_bdd_tests:
call_task('test', options={'prefix': prefix})
elif integration_tests:
if integration_server_tests:
call_task('test_integration',
options={
'prefix': prefix,
'name': 'geonode.geoserver.tests.integration'
})
elif integration_upload_tests:
call_task('test_integration',
options={
'prefix': prefix,
'name': 'geonode.upload.tests.integration'
})
elif integration_monitoring_tests:
call_task('test_integration',
options={
'prefix': prefix,
'name': 'geonode.monitoring.tests.integration'
})
elif integration_csw_tests:
call_task('test_integration',
options={
'prefix': prefix,
'name': 'geonode.tests.csw'
})
elif integration_bdd_tests:
call_task('test_bdd', options={'local': local})
else:
call_task('test_integration',
options={
'prefix': prefix,
'name': 'geonode.tests.integration'
})
sh('flake8 geonode')
def verify_server_task(self, task_name, options, contracts_default=False):
"""
Verify the output of a server task.
"""
log_string = options.get("log_string", "> /dev/null")
settings = options.get("settings", None)
asset_settings = options.get("asset-settings", None)
is_optimized = options.get("optimized", False)
is_fast = options.get("fast", False)
no_contracts = options.get("no-contracts", not contracts_default)
if task_name == "devstack":
system = options.get("system")
elif task_name == "studio":
system = "cms"
else:
system = "lms"
port = options.get("port", "8000" if system == "lms" else "8001")
self.reset_task_messages()
if task_name == "devstack":
args = ["studio" if system == "cms" else system]
if settings:
args.append("--settings={settings}".format(settings=settings))
if asset_settings:
args.append("--asset-settings={asset_settings}".format(asset_settings=asset_settings))
if is_optimized:
args.append("--optimized")
if is_fast:
args.append("--fast")
if no_contracts:
args.append("--no-contracts")
call_task("pavelib.servers.devstack", args=args)
else:
call_task("pavelib.servers.{task_name}".format(task_name=task_name), options=options)
expected_messages = options.get("expected_messages", [])
expected_settings = settings if settings else Env.DEVSTACK_SETTINGS
expected_asset_settings = asset_settings if asset_settings else expected_settings
if is_optimized:
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS
if not is_fast:
expected_messages.append(u"xmodule_assets common/static/xmodule")
expected_messages.append(u"install npm_assets")
expected_messages.extend(
[c.format(settings=expected_asset_settings,
log_file=Env.PRINT_SETTINGS_LOG_FILE) for c in EXPECTED_PRINT_SETTINGS_COMMAND]
)
expected_messages.append(EXPECTED_WEBPACK_COMMAND.format(
node_env="production",
static_root_lms=None,
static_root_cms=None,
lms_root_url=None,
jwt_auth_cookie_header_payload_name=None,
user_info_cookie_name=None,
webpack_config_path=None
))
expected_messages.extend(self.expected_sass_commands(system=system, asset_settings=expected_asset_settings))
if expected_collect_static:
expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
system=system, asset_settings=expected_asset_settings, log_string=log_string
))
expected_run_server_command = EXPECTED_RUN_SERVER_COMMAND.format(
system=system,
settings=expected_settings,
port=port,
)
if not no_contracts:
expected_run_server_command += " --contracts"
expected_messages.append(expected_run_server_command)
self.assertEqual(self.task_messages, expected_messages)
def test_compile_theme_sass(self, options):
"""
Test the "compile_sass" task.
"""
parameters = options.split(" ")
system = []
if '--system=studio' not in parameters:
system += ['lms']
if "--system=lms" not in parameters:
system += ['studio']
debug = '--debug' in parameters
force = '--force' in parameters
self.reset_task_messages()
call_task(
'pavelib.assets.compile_sass',
options=dict(
system=system,
debug=debug,
force=force,
theme_dirs=[TEST_THEME_DIR.dirname()],
themes=[TEST_THEME_DIR.basename()]
),
)
expected_messages = []
if force:
expected_messages.append(u'rm -rf common/static/css/*.css')
expected_messages.append(u'libsass common/static/sass')
if 'lms' in system:
expected_messages.append(u'mkdir_p ' + repr(TEST_THEME_DIR / 'lms/static/css'))
if force:
expected_messages.append(
u'rm -rf {test_theme_dir}/lms/static/css/*.css'.format(test_theme_dir=str(TEST_THEME_DIR))
)
expected_messages.append("libsass lms/static/sass")
expected_messages.append(
u'rtlcss {test_theme_dir}/lms/static/css/bootstrap/lms-main.css'
u' {test_theme_dir}/lms/static/css/bootstrap/lms-main-rtl.css'.format(
test_theme_dir=str(TEST_THEME_DIR),
)
)
expected_messages.append(
u'rtlcss {test_theme_dir}/lms/static/css/discussion/lms-discussion-bootstrap.css'
u' {test_theme_dir}/lms/static/css/discussion/lms-discussion-bootstrap-rtl.css'.format(
test_theme_dir=str(TEST_THEME_DIR),
)
)
if force:
expected_messages.append(
u'rm -rf {test_theme_dir}/lms/static/css/*.css'.format(test_theme_dir=str(TEST_THEME_DIR))
)
expected_messages.append(
u'libsass {test_theme_dir}/lms/static/sass'.format(test_theme_dir=str(TEST_THEME_DIR))
)
if force:
expected_messages.append(u'rm -rf lms/static/css/*.css')
expected_messages.append(u'libsass lms/static/sass')
expected_messages.append(
u'rtlcss lms/static/css/bootstrap/lms-main.css lms/static/css/bootstrap/lms-main-rtl.css'
)
expected_messages.append(
u'rtlcss lms/static/css/discussion/lms-discussion-bootstrap.css'
u' lms/static/css/discussion/lms-discussion-bootstrap-rtl.css'
)
if force:
expected_messages.append(u'rm -rf lms/static/certificates/css/*.css')
expected_messages.append(u'libsass lms/static/certificates/sass')
if "studio" in system:
expected_messages.append(u'mkdir_p ' + repr(TEST_THEME_DIR / 'cms/static/css'))
if force:
expected_messages.append(
u'rm -rf {test_theme_dir}/cms/static/css/*.css'.format(test_theme_dir=str(TEST_THEME_DIR))
)
expected_messages.append(u'libsass cms/static/sass')
expected_messages.append(
u'rtlcss {test_theme_dir}/cms/static/css/bootstrap/studio-main.css'
u' {test_theme_dir}/cms/static/css/bootstrap/studio-main-rtl.css'.format(
test_theme_dir=str(TEST_THEME_DIR),
)
)
if force:
expected_messages.append(
u'rm -rf {test_theme_dir}/cms/static/css/*.css'.format(test_theme_dir=str(TEST_THEME_DIR))
)
expected_messages.append(
u'libsass {test_theme_dir}/cms/static/sass'.format(test_theme_dir=str(TEST_THEME_DIR))
)
if force:
expected_messages.append(u'rm -rf cms/static/css/*.css')
expected_messages.append(u'libsass cms/static/sass')
expected_messages.append(
u'rtlcss cms/static/css/bootstrap/studio-main.css cms/static/css/bootstrap/studio-main-rtl.css'
)
self.assertItemsEqual(self.task_messages, expected_messages)
def update_assets(args):
"""
Compile CoffeeScript and Sass, then collect static assets.
"""
parser = argparse.ArgumentParser(prog='paver update_assets')
parser.add_argument(
'system',
type=str,
nargs='*',
default=ALL_SYSTEMS,
help="lms or studio",
)
parser.add_argument(
'--settings',
type=str,
default=Env.DEVSTACK_SETTINGS,
help="Django settings module",
)
parser.add_argument(
'--debug',
action='store_true',
default=False,
help="Disable Sass compression",
)
parser.add_argument(
'--skip-collect',
dest='collect',
action='store_false',
default=True,
help="Skip collection of static assets",
)
parser.add_argument(
'--watch',
action='store_true',
default=False,
help="Watch files for changes",
)
parser.add_argument(
'--theme-dirs',
dest='theme_dirs',
type=str,
nargs='+',
default=None,
help="base directories where themes are placed",
)
parser.add_argument(
'--themes',
type=str,
nargs='+',
default=None,
help="list of themes to compile sass for",
)
parser.add_argument(
'--collect-log',
dest=COLLECTSTATIC_LOG_DIR_ARG,
default=None,
help=
"When running collectstatic, direct output to specified log directory",
)
parser.add_argument('--wait',
type=float,
default=0.0,
help="How long to pause between filesystem scans")
args = parser.parse_args(args)
collect_log_args = {}
process_xmodule_assets()
process_npm_assets()
compile_coffeescript()
# Build Webpack
call_task('pavelib.assets.webpack', options={'settings': args.settings})
# Compile sass for themes and system
execute_compile_sass(args)
if args.collect:
if args.debug:
collect_log_args.update({COLLECTSTATIC_LOG_DIR_ARG: None})
if args.collect_log_dir:
collect_log_args.update(
{COLLECTSTATIC_LOG_DIR_ARG: args.collect_log_dir})
collect_assets(args.system, args.settings, **collect_log_args)
if args.watch:
call_task(
'pavelib.assets.watch_assets',
options={
'background': not args.debug,
'theme_dirs': args.theme_dirs,
'themes': args.themes,
'wait': [float(args.wait)]
},
)
def update_assets(args):
"""
Compile CoffeeScript and Sass, then collect static assets.
"""
parser = argparse.ArgumentParser(prog='paver update_assets')
parser.add_argument(
'system',
type=str,
nargs='*',
default=ALL_SYSTEMS,
help="lms or studio",
)
parser.add_argument(
'--settings',
type=str,
default="devstack",
help="Django settings module",
)
parser.add_argument(
'--debug',
action='store_true',
default=False,
help="Disable Sass compression",
)
parser.add_argument(
'--skip-collect',
dest='collect',
action='store_false',
default=True,
help="Skip collection of static assets",
)
parser.add_argument(
'--watch',
action='store_true',
default=False,
help="Watch files for changes",
)
parser.add_argument(
'--themes_dir',
type=str,
default=None,
help="base directory where themes are placed",
)
parser.add_argument(
'--themes',
type=str,
nargs='*',
default=None,
help="list of themes to compile sass for",
)
args = parser.parse_args(args)
compile_templated_sass(args.system, args.settings)
process_xmodule_assets()
process_npm_assets()
compile_coffeescript()
call_task(
'pavelib.assets.compile_sass',
options={
'system': args.system,
'debug': args.debug,
'themes_dir': args.themes_dir,
'themes': args.themes
},
)
if args.collect:
collect_assets(args.system, args.settings)
if args.watch:
call_task(
'pavelib.assets.watch_assets',
options={
'background': not args.debug,
'themes_dir': args.themes_dir,
'themes': args.themes
},
)
def web(options):
web_archive = getattr(options.web, 'web_archive', remote_archive)
call_task('install_remote_tar', args=[web_archive, 'napixd/web'])
def test_integration(options):
"""
Run GeoNode's Integration test suite against the external apps
"""
prefix = options.get('prefix')
_backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND'])
if _backend == 'geonode.geoserver' or 'geonode.qgis_server' not in INSTALLED_APPS:
call_task('stop_geoserver')
_reset()
else:
call_task('stop_qgis_server')
_reset()
name = options.get('name', None)
settings = options.get('settings', '')
success = False
try:
call_task('setup', options={'settings': settings, 'force_exec': True})
if name and name in ('geonode.tests.csw', 'geonode.tests.integration',
'geonode.geoserver.tests.integration'):
if not settings:
settings = 'geonode.local_settings' if _backend == 'geonode.qgis_server' else 'geonode.settings'
settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings
call_task('sync', options={'settings': settings})
if _backend == 'geonode.geoserver':
call_task('start_geoserver',
options={
'settings': settings,
'force_exec': True
})
call_task('start', options={'settings': settings})
call_task('setup_data', options={'settings': settings})
elif not integration_csw_tests and _backend == 'geonode.geoserver' and 'geonode.geoserver' in INSTALLED_APPS:
sh("cp geonode/upload/tests/test_settings.py geonode/")
settings = 'geonode.test_settings'
sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py "
"makemigrations --noinput".format(settings))
sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py "
"migrate --noinput".format(settings))
sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py "
"loaddata sample_admin.json".format(settings))
sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py "
"loaddata geonode/base/fixtures/default_oauth_apps.json".format(
settings))
sh("DJANGO_SETTINGS_MODULE={} python -W ignore manage.py "
"loaddata geonode/base/fixtures/initial_data.json".format(
settings))
call_task('start_geoserver')
bind = options.get('bind', '0.0.0.0:8000')
foreground = '' if options.get('foreground', False) else '&'
sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runmessaging %s'
% (settings, foreground))
sh('DJANGO_SETTINGS_MODULE=%s python -W ignore manage.py runserver %s %s'
% (settings, bind, foreground))
sh('sleep 30')
settings = 'REUSE_DB=1 DJANGO_SETTINGS_MODULE=%s' % settings
live_server_option = ''
info("Running the tests now...")
sh(('%s %s manage.py test %s'
' %s --noinput %s' %
(settings, prefix, name, _keepdb, live_server_option)))
except BuildFailure as e:
info('Tests failed! %s' % str(e))
else:
success = True
finally:
stop(options)
_reset()
if not success:
sys.exit(1)
def test(options):
"""Run unit tests"""
db_setup = False
db_conn = None
cfg_files = []
status = 0
url = options.get('url', None)
suites = options.get('suites', None)
database = options.get('database', 'SQLite3')
remote = options.get('remote')
timems = options.get('time', None)
if url is None:
# run against default server
call_task('stop')
call_task('reset')
if database == 'SQLite3':
call_task('setup_testdata')
call_task('start')
url = 'http://*****:*****@localhost/%s' % (user, password,
temp_db)
if password:
sh('set PGPASSWORD=%s' % password)
sh('createdb %s -U %s' % (temp_db, user))
sh('createlang --dbname=%s plpythonu -U %s' % (temp_db, user))
# update all default.cfg files to point to test DB
cfg_files = glob.glob('tests%ssuites%s*%s*.cfg' % (3 * (os.sep, )))
for cfg in cfg_files:
# generate table
suite = cfg.split(os.sep)[2]
tablename = 'records_cite'
if suite == 'manager':
tablename = 'records_manager'
elif suite == 'apiso':
tablename = 'records_apiso'
config = configparser.SafeConfigParser()
with open(cfg) as read_data:
config.readfp(read_data)
config.set('repository', 'database', db_conn)
config.set('repository', 'table', tablename)
with open(cfg, 'wb') as config2:
config.write(config2)
if suite in ['cite', 'manager', 'apiso']: # setup tables
setup_db(db_conn, tablename, home, init_sfsql, init_sfsql)
init_sfsql = False
if suite in ['cite', 'apiso']: # load test data
dirname = '%s%sdata' % (os.path.dirname(cfg), os.sep)
load_records(context, db_conn, tablename, dirname)
else:
raise Exception('Invalid database specified')
with pushd('tests'):
try:
sh(cmd)
except BuildFailure as err:
status = 1
# stop pycsw instance
call_task('stop')
if db_setup: # tearDown
for cfg in cfg_files:
sh('git checkout %s' % cfg)
if database == 'PostgreSQL':
sh("psql -c \"select pg_terminate_backend(procpid) from pg_stat_activity where datname='%s';\" -U %s"
% (temp_db, user))
sh('dropdb %s -U %s' % (temp_db, user))
sh('unset PGPASSWORD')
sys.exit(status)
def setup():
"""setup plugin dependencies"""
config_file = options.base.home / 'GeoHealthCheck/config_main.py'
config_site = options.base.instance / 'config_site.py'
# setup dirs
if not os.path.exists(options.base.static_lib):
options.base.static_lib.mkdir()
if not os.path.exists(options.base.instance):
options.base.instance.mkdir()
data_dir = options.base.instance / 'data'
data_dir.mkdir()
data_dir.chmod(0777)
# setup config
config_file.copy(config_site)
# setup deps
sh('pip install -r requirements.txt')
# split URL to keep pep8 happy
skin = '/'.join([
'http://github.com', 'IronSummitMedia/startbootstrap-sb-admin-2',
'archive/v1.0.3.zip'
])
skin_dirs = ['dist', 'bower_components']
need_to_fetch = False
for skin_dir in skin_dirs:
skin_dir_path = os.sep.join(
['startbootstrap-sb-admin-2-1.0.3', skin_dir])
if not os.path.exists(skin_dir_path):
need_to_fetch = True
if need_to_fetch:
zipstr = StringIO(urlopen(skin).read())
zipfile_obj = zipfile.ZipFile(zipstr)
zipfile_obj.extractall(options.base.static_lib)
for zf_mem in skin_dirs:
src_loc = path(options.base.static_lib /
'startbootstrap-sb-admin-2-1.0.3' / zf_mem)
dest_loc = path(options.base.static_lib / zf_mem)
if not os.path.exists(dest_loc):
src_loc.move(dest_loc)
else:
info('directory already exists. Skipping')
shutil.rmtree(
path(options.base.static_lib / 'startbootstrap-sb-admin-2-1.0.3'))
# install sparklines to static/site/js
with open(path(options.base.static_lib / 'jspark.js'), 'w') as f:
content = urlopen('http://ejohn.org/files/jspark.js').read()
content.replace('red', 'green')
f.write(content)
# install leafletjs to static/lib
leafletjs = 'http://cdn.leafletjs.com/downloads/leaflet-0.7.5.zip'
zipstr = StringIO(urlopen(leafletjs).read())
zipfile_obj = zipfile.ZipFile(zipstr)
zipfile_obj.extractall(options.base.static_lib / 'leaflet')
# install html5shiv to static/lib
with open(path(options.base.static_lib / 'html5shiv.min.js'), 'w') as f:
url = 'http://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js'
content = urlopen(url).read()
f.write(content)
# install respond to static/lib
with open(path(options.base.static_lib / 'respond.min.js'), 'w') as f:
url = 'http://oss.maxcdn.com/respond/1.4.2/respond.min.js'
content = urlopen(url).read()
f.write(content)
# build i18n .mo files
call_task('compile_translations')
# build local docs
call_task('refresh_docs')
# message user
info('GeoHealthCheck is now built. Edit settings in %s' % config_site)
info('before deploying the application. Alternatively, you can start a')
info('development instance with "python GeoHealthCheck/app.py"')
def default():
call_task('test_documentation')
def test_safecommit_vanilla(self, _mock_count, _mock_report_dir, _mock_write_metric):
"""
run_safecommit_report finds violations.
"""
_mock_count.return_value = 0
call_task('pavelib.quality.run_safecommit_report')
def run_all_servers(options):
"""
Runs Celery workers, Studio, and LMS.
"""
# settings = getattr(options, 'settings', DEFAULT_SETTINGS)
settings = 'dev'
# asset_settings = getattr(options, 'asset_settings', settings)
asset_settings = 'dev'
worker_settings = getattr(options, 'worker_settings', 'dev_with_worker')
fast = getattr(options, 'fast', False)
optimized = getattr(options, 'optimized', False)
if optimized:
settings = OPTIMIZED_SETTINGS
asset_settings = OPTIMIZED_ASSETS_SETTINGS
settings_lms = 'cms.dev'
# settings_lms = getattr(options, 'settings_lms', settings)
settings_cms = 'dev'
# settings_cms = getattr(options, 'settings_cms', settings)
asset_settings_lms = 'cms.dev'
# asset_settings_lms = getattr(options, 'asset_settings_lms', asset_settings)
asset_settings_cms = 'dev'
# asset_settings_cms = getattr(options, 'asset_settings_cms', asset_settings)
if not fast:
# First update assets for both LMS and Studio but don't collect static yet
args = [
'lms', 'studio',
'--settings={}'.format(asset_settings),
'--skip-collect'
]
call_task('pavelib.assets.update_assets', args=args)
# Now collect static for each system separately with the appropriate settings.
# Note that the default settings use DEBUG mode for running the server which
# means that the optimized assets are ignored, so we skip collectstatic in that
# case to save time.
if settings != DEFAULT_SETTINGS:
collect_assets(['lms'], asset_settings_lms)
collect_assets(['studio'], asset_settings_cms)
# Install an asset watcher to regenerate files that change
call_task('pavelib.assets.watch_assets', options={'background': True})
# Start up LMS, CMS and Celery
lms_port = DEFAULT_PORT['lms']
cms_port = DEFAULT_PORT['studio']
lms_runserver_args = ["0.0.0.0:{}".format(lms_port)]
cms_runserver_args = ["0.0.0.0:{}".format(cms_port)]
run_multi_processes([
django_cmd(
'lms', settings_lms, 'runserver', '--traceback', '--pythonpath=.', *lms_runserver_args
),
django_cmd(
'studio', settings_cms, 'runserver', '--traceback', '--pythonpath=.', *cms_runserver_args
),
django_cmd(
'lms', worker_settings, 'celery', 'worker', '--beat', '--loglevel=INFO', '--pythonpath=.'
)
])
def make(options):
"""Overrides sdist to make sure that our setup.py is generated."""
call_task('sett.build.make')
def bdist_mpkg():
call_task("clean")
pyver = "".join([str(i) for i in sys.version_info[:2]])
sh("%s setupegg.py bdist_mpkg" % MPKG_PYTHON[pyver])
"""create virtualenv in ./bootstrap"""
try:
import virtualenv
except ImportError, e:
raise RuntimeError("virtualenv is needed for bootstrap")
bdir = options.bootstrap_dir
if not os.path.exists(bdir):
os.makedirs(bdir)
bscript = "boostrap.py"
options.virtualenv.script_name = os.path.join(options.bootstrap_dir,
bscript)
options.virtualenv.no_site_packages = False
options.bootstrap.no_site_packages = False
call_task('paver.virtual.bootstrap')
sh('cd %s; %s %s' % (bdir, sys.executable, bscript))
@task
def clean():
"""Remove build, dist, egg-info garbage."""
d = ['build', 'dist', 'numpy.egg-info']
for i in d:
if os.path.exists(i):
shutil.rmtree(i)
bdir = os.path.join('doc', options.sphinx.builddir)
if os.path.exists(bdir):
shutil.rmtree(bdir)
def run_dev():
os.environ['APP_SETTINGS_YAML'] = 'development.yaml'
os.environ['RELOAD'] = '1'
os.environ['DEBUG'] = '1'
call_task('run')
def test_compile_theme_sass(self, options):
"""
Test the "compile_sass" task.
"""
parameters = options.split(" ")
system = []
if "--system=studio" not in parameters:
system += ["lms"]
if "--system=lms" not in parameters:
system += ["studio"]
debug = "--debug" in parameters
force = "--force" in parameters
self.reset_task_messages()
call_task(
'pavelib.assets.compile_sass',
options={
"system": system,
"debug": debug,
"force": force,
"themes_dir": TEST_THEME.dirname(),
"themes": [TEST_THEME.basename()]
},
)
expected_messages = []
if force:
expected_messages.append("rm -rf common/static/css/*.css")
expected_messages.append("libsass common/static/sass")
if "lms" in system:
expected_messages.append("mkdir_p " +
repr(TEST_THEME / "lms/static/css"))
if force:
expected_messages.append("rm -rf " + str(TEST_THEME) +
"/lms/static/css/*.css")
expected_messages.append("libsass lms/static/sass")
if force:
expected_messages.append("rm -rf " + str(TEST_THEME) +
"/lms/static/css/*.css")
expected_messages.append("libsass " + str(TEST_THEME) +
"/lms/static/sass")
if force:
expected_messages.append("rm -rf lms/static/css/*.css")
expected_messages.append("libsass lms/static/sass")
if force:
expected_messages.append(
"rm -rf lms/static/certificates/css/*.css")
expected_messages.append("libsass lms/static/certificates/sass")
if "studio" in system:
expected_messages.append("mkdir_p " +
repr(TEST_THEME / "cms/static/css"))
if force:
expected_messages.append("rm -rf " + str(TEST_THEME) +
"/cms/static/css/*.css")
expected_messages.append("libsass cms/static/sass")
if force:
expected_messages.append("rm -rf " + str(TEST_THEME) +
"/cms/static/css/*.css")
expected_messages.append("libsass " + str(TEST_THEME) +
"/cms/static/sass")
if force:
expected_messages.append("rm -rf cms/static/css/*.css")
expected_messages.append("libsass cms/static/sass")
self.assertEquals(self.task_messages, expected_messages)
def test_and_code_style():
call_task('flake8')
call_task('unit_test')
