def _adjust_options():
"""
Set default sphinx and gh_pages options.
"""
if options.get('_github_tools_options_adjusted') is None:
options.setdefault('sphinx', Bunch())
options.setdefault('gh_pages', Bunch())
options.sphinx.docroot = docroot \
= path( options.sphinx.get('docroot', 'docs'))
options.sphinx._buildir = buildir = \
docroot / options.sphinx.get('builddir', 'build')
options.sphinx._sourcedir = \
docroot / options.sphinx.get('sourcedir', 'source')
options.sphinx._doctrees = buildir / "doctrees"
options.sphinx._htmldir = htmldir = \
buildir / 'html'
gh_pages_root = options.gh_pages.get('root', None)
if gh_pages_root is None:
options.gh_pages.root = htmldir
else:
options.gh_pages.root = path(gh_pages_root)
gh_pages_htmlroot = options.gh_pages.get('htmlroot', None)
if gh_pages_htmlroot is None:
options.gh_pages.htmlroot = options.gh_pages.root
else:
options.gh_pages.htmlroot = path(gh_pages_htmlroot)
options.gh_pages.setdefault('remote_name', 'origin')
options.gh_pages.setdefault('master_branch', 'master')
options._github_tools_options_adjusted = True
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
fast = options.get('fast', False)
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL)
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
data_dir = download_dir / os.path.basename(DATA_DIR_URL)
grab(GEOSERVER_URL, geoserver_bin, "geoserver binary")
grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner")
grab(DATA_DIR_URL, data_dir, "data dir")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / 'geoserver'
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
with zipfile.ZipFile(geoserver_bin, "r") as z:
z.extractall(webapp_dir)
_install_data_dir()
def _install_data_dir():
data_dir = path('geoserver/data')
if data_dir.exists():
data_dir.rmtree()
geoserver_dir = path('geoserver')
download_dir = path('downloaded')
data_dir_zip = download_dir / os.path.basename(DATA_DIR_URL)
if os.path.exists(data_dir_zip):
print 'extracting datadir'
with zipfile.ZipFile(data_dir_zip, "r") as z:
z.extractall(geoserver_dir)
config = (
geoserver_dir / 'data/security/auth/geonodeAuthProvider/config.xml'
)
with open(config) as f:
xml = f.read()
m = re.search('baseUrl>([^<]+)', xml)
xml = xml[:m.start(1)] + "http://localhost:" + GEM_GEONODE_PORT + "/" + xml[m.end(1):]
with open(config, 'w') as f:
f.write(xml)
else:
print 'data_dir_zip not found, unable to extract and configure'
def _install_data_dir():
target_data_dir = path('geoserver/data')
if target_data_dir.exists():
try:
target_data_dir.rmtree()
except OSError:
_robust_rmtree(target_data_dir, logger=True)
original_data_dir = path('geoserver/geoserver/data')
justcopy(original_data_dir, target_data_dir)
try:
config = path(
'geoserver/data/global.xml')
with open(config) as f:
xml = f.read()
m = re.search('proxyBaseUrl>([^<]+)', xml)
xml = xml[:m.start(1)] + \
"http://localhost:8080/geoserver" + xml[m.end(1):]
with open(config, 'w') as f:
f.write(xml)
except Exception as e:
print(e)
try:
config = path(
'geoserver/data/security/filter/geonode-oauth2/config.xml')
with open(config) as f:
xml = f.read()
m = re.search('accessTokenUri>([^<]+)', xml)
xml = xml[:m.start(1)] + \
"http://localhost:8000/o/token/" + xml[m.end(1):]
m = re.search('userAuthorizationUri>([^<]+)', xml)
xml = xml[:m.start(
1)] + "http://localhost:8000/o/authorize/" + xml[m.end(1):]
m = re.search('redirectUri>([^<]+)', xml)
xml = xml[:m.start(
1)] + "http://localhost:8080/geoserver/index.html" + xml[m.end(1):]
m = re.search('checkTokenEndpointUrl>([^<]+)', xml)
xml = xml[:m.start(
1)] + "http://localhost:8000/api/o/v4/tokeninfo/" + xml[m.end(1):]
m = re.search('logoutUri>([^<]+)', xml)
xml = xml[:m.start(
1)] + "http://localhost:8000/account/logout/" + xml[m.end(1):]
with open(config, 'w') as f:
f.write(xml)
except Exception as e:
print(e)
try:
config = path(
'geoserver/data/security/role/geonode REST role service/config.xml')
with open(config) as f:
xml = f.read()
m = re.search('baseUrl>([^<]+)', xml)
xml = xml[:m.start(1)] + "http://localhost:8000" + xml[m.end(1):]
with open(config, 'w') as f:
f.write(xml)
except Exception as e:
print(e)
def __init__(self, name, source, out, params, cache):
self.name = name
self.source = path(source)
self.out = path(out)
self.params = dict(params)
self.cache = cache
self.config_js = self.source.joinpath(self.params.pop('mainConfigFile', 'config.js'))
def build_objc_static_lib(dest_path, additional_sources=[], addition_include_dirs=[]):
# Compile Tapstream objc sources into static library
sdk_root = sh('echo $(xcodebuild -version -sdk iphoneos Path)', capture=True).strip()
simulator_sdk_root = sh('echo $(xcodebuild -version -sdk iphonesimulator Path)', capture=True).strip()
include_dirs = listify(['objc/Core', 'objc/Tapstream', '/usr/local/include/']+addition_include_dirs, prefix='-I')
core_sources = list(path('objc/Core').walkfiles('*.m'))
inputs = core_sources + list(path('objc/Tapstream').walkfiles('*.m')) + [path(p) for p in additional_sources]
sh('rm -f ./*.o')
sh('xcrun -sdk iphoneos clang -isysroot %s -miphoneos-version-min=4.3 -arch armv7 -fno-objc-arc %s -c %s' % (
sdk_root, include_dirs, listify(inputs)
))
sh('xcrun -sdk iphoneos ar rcu objc/TapstreamArm7.a ./*.o')
sh('rm ./*.o')
sh('xcrun -sdk iphoneos clang -isysroot %s -miphoneos-version-min=4.3 -arch armv7s -fno-objc-arc %s -c %s' % (
sdk_root, include_dirs, listify(inputs)
))
sh('xcrun -sdk iphoneos ar rcu objc/TapstreamArm7s.a ./*.o')
sh('rm ./*.o')
sh('xcrun -sdk iphonesimulator clang -isysroot %s -miphoneos-version-min=4.3 -arch i386 -fobjc-abi-version=2 -fno-objc-arc %s -c %s' % (
simulator_sdk_root, include_dirs, listify(inputs)
))
sh('xcrun -sdk iphonesimulator ar rcu objc/Tapstreami386.a ./*.o')
sh('rm ./*.o')
sh('xcrun -sdk iphoneos lipo -create objc/TapstreamArm7.a objc/TapstreamArm7s.a objc/Tapstreami386.a -output %s' % dest_path)
sh('rm objc/Tapstream*.a')
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
fast = options.get('fast', False)
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = path('geoserver_ext/target/geoserver.war')
if not geoserver_bin.exists():
geoserver_bin = download_dir / os.path.basename(ROGUE_GEOSERVER_URL)
grab(ROGUE_GEOSERVER_URL, geoserver_bin, "geoserver binary")
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner")
data_dir = download_dir / os.path.basename(DATA_DIR_URL)
grab(DATA_DIR_URL, data_dir, "data dir")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
# Set the geonode auth config to dev port 8000
sh("perl -pi.back -e 's/localhost/localhost:8000/g;' geoserver/data/security/auth/geonodeAuthProvider/config.xml")
def package_tar_gz(options):
"""Package a .tar.gz distribution"""
import tarfile
package_name = options.get("package_name", None)
if package_name is None:
raise Exception("Package name required")
filename = path("%s/%s.tar.gz" % (STAGE_DIR, package_name))
if filename.exists():
info("Package %s already exists" % filename)
return
with pushd(STAGE_DIR):
stage_path = "%s/%s" % (STAGE_DIR, package_name)
if not path(stage_path).exists():
raise Exception("Directory %s does not exist" % stage_path)
tar = tarfile.open(filename, "w:gz")
tar.add(package_name)
tar.close()
def setup_geoserver():
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL)
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
grab(GEOSERVER_URL, geoserver_bin, "geoserver binary")
grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner")
geoserver_dir = path('geoserver')
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / 'geoserver'
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
target_data_dir = path('geoserver/data')
if not target_data_dir.exists():
target_data_dir.makedirs()
_install_data_dir()
def start_geoserver(options):
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
data_dir = path('geoserver/data').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/jetty-runner.xml').abspath()
with pushd(data_dir):
sh(('java -Xmx512m -XX:MaxPermSize=1024m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --log %(log_file)s'
' %(config)s'
' > /dev/null &' % locals()
))
info('Starting GeoServer on %s' % GEOSERVER_BASE_URL)
# wait for GeoServer to start
started = waitfor(GEOSERVER_BASE_URL)
info('The logs are available at %s' % log_file)
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
def bundle():
"""Bundle required Python packages as ZIP archive."""
packages = options.bundle.packages
builddir = path("build")/"bundle"
builddir.makedirs()
errors = 0
for package in packages:
try:
info("bundle %s ..." % package)
module = __import__(package)
pkgfile = path(module.__file__)
basedir = pkgfile.dirname()
destdir = builddir/package
if destdir.exists():
destdir.rmtree()
basedir.copytree(destdir)
except Exception as e:
info("FAILED %s: %s" % (package, e))
errors += 1
archive = options.bundle.get("archive", "python-bundle.zip")
stored_files = make_zip_archive(archive, builddir)
# -- SUMMARY:
message1 = "bundle {pkgno} packages into {archive} "
message1 += "(with {errorno} errors, files: {fileno})"
message = message1.format(archive=archive, pkgno=len(packages),
errorno=errors, fileno=stored_files)
output = info
if errors:
message = "FAILED: {message}".format(message=message)
output = error
output(message)
def get_system_sass_dirs(system):
"""
Return list of sass dirs that need to be compiled for the given system.
:param system: name if the system for which to compile sass e.g. 'lms', 'cms'
"""
if system not in ('lms', 'cms'):
raise ValueError('"system" must either be "lms" or "cms"')
dirs = []
sass_dir = path(system) / "static" / "sass"
css_dir = path(system) / "static" / "css"
dependencies = SASS_LOOKUP_DEPENDENCIES.get(system, [])
dirs.append({
"sass_source_dir": sass_dir,
"css_destination_dir": css_dir,
"lookup_paths": dependencies + [
sass_dir / "partials",
sass_dir,
],
})
if system == 'lms':
dirs.append({
"sass_source_dir": path(system) / "static" / "certificates" / "sass",
"css_destination_dir": path(system) / "static" / "certificates" / "css",
"lookup_paths": [
sass_dir / "partials",
sass_dir
],
})
return dirs
def configure_paths():
"""Configure our paths based on settings. Called immediately."""
edxapp_env = Env()
if edxapp_env.feature_flags.get('USE_CUSTOM_THEME', False):
theme_name = edxapp_env.env_tokens.get('THEME_NAME', '')
parent_dir = path(edxapp_env.REPO_ROOT).abspath().parent
theme_root = parent_dir / "themes" / theme_name
COFFEE_DIRS.append(theme_root)
sass_dir = theme_root / "static" / "sass"
css_dir = theme_root / "static" / "css"
if sass_dir.isdir():
css_dir.mkdir_p()
SASS_DIRS.append(sass_dir)
if edxapp_env.env_tokens.get("COMPREHENSIVE_THEME_DIR", ""):
theme_dir = path(edxapp_env.env_tokens["COMPREHENSIVE_THEME_DIR"])
lms_sass = theme_dir / "lms" / "static" / "sass"
lms_css = theme_dir / "lms" / "static" / "css"
if lms_sass.isdir():
lms_css.mkdir_p()
SASS_DIRS.append(lms_sass)
cms_sass = theme_dir / "cms" / "static" / "sass"
cms_css = theme_dir / "cms" / "static" / "css"
if cms_sass.isdir():
cms_css.mkdir_p()
SASS_DIRS.append(cms_sass)
def install_remote_tar(args, options):
"""Usage: install_remote_tar [-t|--target TARGET] ARCHIVE DESTINATION
Extracts an archive in a directory. The archive is either a file on the local
file system or a remote URL fetched by http/https. A file whose name is TARGET
is looked for inside the archive and the directory containing the target is
moved as the DESTINATION directory.
Any exisiting directory will be removed before the new directory is copied.
"""
web_archive, destination = args
target = getattr(options, 'target', 'index.html')
temp_dir = path(tempfile.mkdtemp())
try:
with TarExtract(web_archive) as tf:
for ti in sorted(tf.getmembers(), key=lambda x: (x.name.count('/'), x.name)):
name = path(ti.name)
if name.basename() == target:
target_root = name.dirname()
break
else:
raise ValueError('Missing target: {}'.format(target))
tf.extractall(temp_dir)
except Exception:
temp_dir.rmtree()
raise
else:
path(destination).rmtree()
temp_dir.joinpath(target_root).move(destination)
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
from geonode.settings import OGC_SERVER
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = "http://localhost:8080/geoserver/"
if GEOSERVER_BASE_URL != url:
print 'your GEOSERVER_BASE_URL does not match %s' % url
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
data_dir = path('geoserver/data').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/misc/jetty-runner.xml').abspath()
# @todo - we should not have set workdir to the datadir but a bug in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
loggernullpath = "/dev/null"
try:
sh(('java -version'))
except:
if not options.get('java_path', None):
print "Paver cannot find java in the Windows Environment. Please provide the --java_path flag with your full path to java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + options['java_path'] + '"'
# cmd log file needs to exist in windows
# using folder from .gitignore
open("../../downloaded/null.txt", 'w+').close()
loggernullpath = "../../downloaded/null.txt"
sh((
'%(javapath)s -Xmx512m -XX:MaxPermSize=256m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
# workaround for JAI sealed jar issue and jetty classloader
' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()
))
info('Starting GeoServer on %s' % url)
# wait for GeoServer to start
started = waitfor(url)
info('The logs are available at %s' % log_file)
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
def configure_paths():
"""Configure our paths based on settings. Called immediately."""
edxapp_env = Env()
# if edxapp_env.feature_flags.get('USE_CUSTOM_THEME', False):
theme_name = 'default'
parent_dir = path(COMP_THEME_DIR).abspath().parent
theme_root = parent_dir / "themes" / theme_name
COFFEE_DIRS.append(theme_root)
sass_dir = theme_root / "static" / "sass"
css_dir = theme_root / "static" / "css"
if sass_dir.isdir():
css_dir.mkdir_p()
SASS_DIRS.append(sass_dir)
# if edxapp_env.env_tokens.get("COMP_THEME_DIR", ""):
theme_dir = path(COMP_THEME_DIR)
lms_sass = theme_dir / "static" / "sass"
lms_css = theme_dir / "static" / "css"
if lms_sass.isdir():
lms_css.mkdir_p()
SASS_DIRS.append(lms_sass)
studio_sass = theme_dir / "studio" / "static" / "sass"
studio_css = theme_dir / "studio" / "static" / "css"
if studio_sass.isdir():
studio_css.mkdir_p()
SASS_DIRS.append(studio_sass)
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
fast = options.get("fast", False)
download_dir = path("downloaded")
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path("geoserver")
geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL)
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
grab(GEOSERVER_URL, geoserver_bin, "geoserver binary")
grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / "geoserver"
if not webapp_dir:
webapp_dir.makedirs()
print "extracting geoserver"
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
_install_data_dir()
def package_tar_gz(options):
"""Package a .tar.gz distribution"""
import tarfile
package_name = options.get('package_name', None)
if package_name is None:
raise Exception('Package name required')
filename = path('%s/%s.tar.gz' % (STAGE_DIR, package_name))
if filename.exists():
info('Package %s already exists' % filename)
return
with pushd(STAGE_DIR):
stage_path = '%s/%s' % (STAGE_DIR, package_name)
if not path(stage_path).exists():
raise Exception('Directory %s does not exist' % stage_path)
tar = tarfile.open(filename, 'w:gz')
tar.add(package_name)
tar.close()
def docs():
import markdown
from django.template import Context, Template
from django.conf import settings
settings.configure()
path('builds/docs').rmtree()
path('builds/docs').makedirs()
path.copy(path('./docs/bootstrap.min.css'), path('./builds/docs/'))
path.copy(path('./docs/pygments.css'), path('./builds/docs/'))
with open('docs/base.html') as f:
base_template = Template(f.read())
with open('docs/docs.md') as f:
md_template = Template(f.read())
with pushd('builds/docs'):
for platform in ('android', 'ios', 'mac', 'win8', 'winphone'):
md = md_template.render(Context({'platform': platform, 'pretty_platform': PRETTY_PLATFORMS[platform]}))
with open('docs_%s.md' % platform, 'w') as f:
f.write(md)
md = markdown.markdown(md, ['fenced_code', 'codehilite'])
page = base_template.render(Context({'md': md}))
with open('docs_%s.html' % platform, 'w') as f:
f.write(page)
path.copy(path('phonegap/PhoneGap.md'), path('./builds/docs/docs_phonegap.md'))
with open('phonegap/PhoneGap.md') as f:
md = markdown.markdown(f.read(), ['fenced_code', 'codehilite'])
page = base_template.render(Context({'md': md}))
with pushd('builds/docs'):
with open('docs_phonegap.html', 'w') as f:
f.write(page)
def setup_geoserver(options):
"""Prepare a testing instance of GeoServer."""
download_dir = path('downloaded')
if not download_dir.exists():
download_dir.makedirs()
geoserver_dir = path('geoserver')
geoserver_bin = download_dir / os.path.basename(dev_config['GEOSERVER_URL'])
jetty_runner = download_dir / os.path.basename(dev_config['JETTY_RUNNER_URL'])
grab(options.get('geoserver', dev_config['GEOSERVER_URL']), geoserver_bin, "geoserver binary")
grab(options.get('jetty', dev_config['JETTY_RUNNER_URL']), jetty_runner, "jetty runner")
if not geoserver_dir.exists():
geoserver_dir.makedirs()
webapp_dir = geoserver_dir / 'geoserver'
if not webapp_dir:
webapp_dir.makedirs()
print 'extracting geoserver'
z = zipfile.ZipFile(geoserver_bin, "r")
z.extractall(webapp_dir)
_install_data_dir()
def setup_testdata():
"""Create test databases and load test data"""
test_database_parameters = {
# suite: has_testdata
'apiso': True,
'cite': True,
'harvesting': False,
'manager': False
}
# remove CITE database so we can build fresh
for suite in test_database_parameters.keys():
dbfile = 'tests/suites/%s/data/records.db' % suite
if os.path.isfile(dbfile):
os.remove(dbfile)
for database, has_testdata in test_database_parameters.iteritems():
info('Setting up test database %s' % database)
cfg = path('tests/suites/%s/default.cfg' % database)
sh('pycsw-admin.py -c setup_db -f %s' % cfg)
if has_testdata:
datapath = path('tests/suites/%s/data' % database)
info('Loading test data from %s' % datapath)
sh('pycsw-admin.py -c load_records -f %s -p %s' % (cfg, datapath))
def test_watch_theme_assets(self):
"""
Test the Paver watch asset tasks with theming enabled.
"""
self.expected_sass_directories.extend([
path(TEST_THEME_DIR) / 'lms/static/sass',
path(TEST_THEME_DIR) / 'lms/static/sass/partials',
path(TEST_THEME_DIR) / 'cms/static/sass',
path(TEST_THEME_DIR) / 'cms/static/sass/partials',
])
with patch('pavelib.assets.SassWatcher.register') as mock_register:
with patch('pavelib.assets.Observer.start'):
with patch('pavelib.assets.execute_webpack_watch') as mock_webpack:
call_task(
'pavelib.assets.watch_assets',
options={
"background": True,
"theme_dirs": [TEST_THEME_DIR.dirname()],
"themes": [TEST_THEME_DIR.basename()]
},
)
self.assertEqual(mock_register.call_count, 2)
self.assertEqual(mock_webpack.call_count, 1)
sass_watcher_args = mock_register.call_args_list[0][0]
self.assertIsInstance(sass_watcher_args[0], Observer)
self.assertIsInstance(sass_watcher_args[1], list)
self.assertItemsEqual(sass_watcher_args[1], self.expected_sass_directories)
def sphinx_index():
"""Update all indices."""
path(options.sphinx.index_path).makedirs()
command = 'indexer --all --config %s' % options.sphinx.config
if is_sphinx_daemon_running():
command += ' --rotate'
sh(command)
def make_phonegap():
path('builds/phonegap').rmtree()
path('builds/phonegap').makedirs()
sh('cp ./phonegap/tapstream.js ./builds/phonegap/')
sh('cp -r ./phonegap/objc_plugin ./builds/phonegap/')
sh('cp -r ./phonegap/java_plugin ./builds/phonegap/')
sh('cp builds/android/Tapstream.jar ./builds/phonegap')
sh('cp -r builds/ios/Tapstream ./builds/phonegap')
def setup_ceph_client(options):
ceph_client_log_dir = path('geonode/cephgeo/logs')
if not ceph_client_log_dir.exists():
ceph_client_log_dir.makedirs()
ceph_client_log_file = path('geonode/cephgeo/logs/ceph_storage.log')
if not ceph_client_log_file.isfile():
ceph_client_log_file.touch()
def _package_java_whitelabel():
path('builds/android-whitelabel').rmtree()
path('builds/android-whitelabel').makedirs()
path.copy(path('./java-whitelabel/Tapstream/build/jar/Tapstream.jar'), path('./builds/android-whitelabel/ConversionTracker.jar'))
path('builds/TapstreamSDK-android-whitelabel.zip').remove()
with pushd('builds/android-whitelabel'):
sh('7z a -tzip ../TapstreamSDK-android-whitelabel.zip ConversionTracker.jar')
path('java-whitelabel').rmtree()
def _package_java():
path('builds/android').rmtree()
path('builds/android').makedirs()
path.copy(path('./java/Tapstream/build/jar/Tapstream.jar'), path('./builds/android/'))
path.copy(path('./java/Tapstream/build/jar/Tapstream.jar'), path('./examples/Android/Example/libs/'))
path('builds/TapstreamSDK-android.zip').remove()
with pushd('builds/android'):
sh('7z a -tzip ../TapstreamSDK-android.zip Tapstream.jar')
def _package_java():
path('builds/android').rmtree()
path('builds/android').makedirs()
path.copy(path('./java/Tapstream/build/jar/Tapstream.jar'), path('./builds/android/'))
path.copy(path('./java/Tapstream/build/jar/Tapstream.jar'), path('./examples/Android/Example/libs/'))
path('builds/tapstream-%s-android.zip' % VERSION).remove()
with pushd('builds/android'):
_zip("../tapstream-%s-android.zip" % VERSION, 'Tapstream.jar')
def setup_celery(options):
celery_pid_dir = path('celery/pid')
if not celery_pid_dir.exists():
celery_pid_dir.makedirs()
celery_log_dir = path('celery/log')
if not celery_log_dir.exists():
celery_log_dir.makedirs()
def _package_java_whitelabel():
path('builds/android-whitelabel').rmtree()
path('builds/android-whitelabel').makedirs()
path.copy(path('./java-whitelabel/Tapstream/build/jar/Tapstream.jar'), path('./builds/android-whitelabel/ConversionTracker.jar'))
path('builds/tapstream-%s-android-whitelabel.zip' % VERSION).remove()
with pushd('builds/android-whitelabel'):
_zip("../tapstream-%s-android-whitelabel.zip" % VERSION, 'ConversionTracker.jar')
path('java-whitelabel').rmtree()
# setup baseline paths
ALL_SYSTEMS = ['lms', 'studio']
LMS = 'lms'
CMS = 'cms'
SYSTEMS = {
'lms': LMS,
'cms': CMS,
'studio': CMS
}
# Common lookup paths that are added to the lookup paths for all sass compilations
COMMON_LOOKUP_PATHS = [
path("common/static"),
path("common/static/sass"),
path('node_modules/@edx'),
path('node_modules'),
]
# A list of NPM installed libraries that should be copied into the common
# static directory.
# If string ends with '/' then all file in the directory will be copied.
NPM_INSTALLED_LIBRARIES = [
'backbone.paginator/lib/backbone.paginator.js',
'backbone/backbone.js',
'bootstrap/dist/js/bootstrap.bundle.js',
'hls.js/dist/hls.js',
'jquery-migrate/dist/jquery-migrate.js',
'jquery.scrollto/jquery.scrollTo.js',
def compile_sass(options):
"""
Compile Sass to CSS. If command is called without any arguments, it will
only compile lms, cms sass for the open source theme. And none of the comprehensive theme's sass would be compiled.
If you want to compile sass for all comprehensive themes you will have to run compile_sass
specifying all the themes that need to be compiled..
The following is a list of some possible ways to use this command.
Command:
paver compile_sass
Description:
compile sass files for both lms and cms. If command is called like above (i.e. without any arguments) it will
only compile lms, cms sass for the open source theme. None of the theme's sass will be compiled.
Command:
paver compile_sass --theme-dirs /edx/app/edxapp/edx-platform/themes --themes=red-theme
Description:
compile sass files for both lms and cms for 'red-theme' present in '/edx/app/edxapp/edx-platform/themes'
Command:
paver compile_sass --theme-dirs=/edx/app/edxapp/edx-platform/themes --themes red-theme stanford-style
Description:
compile sass files for both lms and cms for 'red-theme' and 'stanford-style' present in
'/edx/app/edxapp/edx-platform/themes'.
Command:
paver compile_sass --system=cms
--theme-dirs /edx/app/edxapp/edx-platform/themes /edx/app/edxapp/edx-platform/common/test/
--themes red-theme stanford-style test-theme
Description:
compile sass files for cms only for 'red-theme', 'stanford-style' and 'test-theme' present in
'/edx/app/edxapp/edx-platform/themes' and '/edx/app/edxapp/edx-platform/common/test/'.
"""
debug = options.get('debug')
force = options.get('force')
systems = get_parsed_option(options, 'system', ALL_SYSTEMS)
themes = get_parsed_option(options, 'themes', [])
theme_dirs = get_parsed_option(options, 'theme_dirs', [])
if not theme_dirs and themes:
# We can not compile a theme sass without knowing the directory that contains the theme.
raise ValueError('theme-dirs must be provided for compiling theme sass.')
if themes and theme_dirs:
themes = get_theme_paths(themes=themes, theme_dirs=theme_dirs)
# Compile sass for OpenEdx theme after comprehensive themes
if None not in themes:
themes.append(None)
timing_info = []
dry_run = tasks.environment.dry_run
compilation_results = {'success': [], 'failure': []}
print("\t\tStarted compiling Sass:")
# compile common sass files
is_successful = _compile_sass('common', None, debug, force, timing_info)
if is_successful:
print("Finished compiling 'common' sass.")
compilation_results['success' if is_successful else 'failure'].append('"common" sass files.')
for system in systems:
for theme in themes:
print("Started compiling '{system}' Sass for '{theme}'.".format(system=system, theme=theme or 'system'))
# Compile sass files
is_successful = _compile_sass(
system=system,
theme=path(theme) if theme else None,
debug=debug,
force=force,
timing_info=timing_info
)
if is_successful:
print("Finished compiling '{system}' Sass for '{theme}'.".format(
system=system, theme=theme or 'system'
))
compilation_results['success' if is_successful else 'failure'].append('{system} sass for {theme}.'.format(
system=system, theme=theme or 'system',
))
print("\t\tFinished compiling Sass:")
if not dry_run:
for sass_dir, css_dir, duration in timing_info:
print(">> {} -> {} in {}s".format(sass_dir, css_dir, duration))
if compilation_results['success']:
print("\033[92m\nSuccessful compilations:\n--- " + "\n--- ".join(compilation_results['success']) + "\n\033[00m")
if compilation_results['failure']:
print("\033[91m\nFailed compilations:\n--- " + "\n--- ".join(compilation_results['failure']) + "\n\033[00m")
def deb(options):
"""
Creates debian packages.
Example uses:
paver deb
paver deb -k 12345
paver deb -k 12345 -p geonode/testing
"""
key = options.get('key', None)
ppa = options.get('ppa', None)
version, simple_version = versions()
info('Creating package for GeoNode version %s' % version)
# Get rid of any uncommitted changes to debian/changelog
info('Getting rid of any uncommitted changes in debian/changelog')
sh('git checkout package/debian/changelog')
# Workaround for git-dch bug
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580
sh('rm -rf %s/.git' % (os.path.realpath('package')))
sh('ln -s %s %s' % (os.path.realpath('.git'), os.path.realpath('package')))
with pushd('package'):
# Install requirements
# sh('sudo apt-get -y install debhelper devscripts git-buildpackage')
# sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s'
# ' --id-length=6 --ignore-branch --release' % (simple_version)))
# In case you publish from Ubuntu Xenial (git-dch is removed from upstream)
# use the following line instead:
# sh(('gbp dch --spawn-editor=snapshot --git-author --new-version=%s'
# ' --id-length=6 --ignore-branch --release' % (simple_version)))
distribution = "bionic"
# sh(('gbp dch --distribution=%s --force-distribution --spawn-editor=snapshot --git-author --new-version=%s'
# ' --id-length=6 --ignore-branch --release' % (distribution, simple_version)))
deb_changelog = path('debian') / 'changelog'
for idx, line in enumerate(
fileinput.input([deb_changelog], inplace=True)):
if idx == 0:
print "geonode (%s) %s; urgency=high" % (simple_version,
distribution),
else:
print line.replace("urgency=medium", "urgency=high"),
# Revert workaround for git-dhc bug
sh('rm -rf .git')
if key is None and ppa is None:
print("A local installable package")
sh('debuild -uc -us -A')
elif key is None and ppa is not None:
print("A sources package, signed by daemon")
sh('debuild -S')
elif key is not None and ppa is None:
print("A signed installable package")
sh('debuild -k%s -A' % key)
elif key is not None and ppa is not None:
print("A signed, source package")
sh('debuild -k%s -S' % key)
if ppa is not None:
sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
def _install_data_dir():
target_data_dir = path('geoserver/data')
if target_data_dir.exists():
try:
target_data_dir.rmtree()
except OSError:
_robust_rmtree(target_data_dir, logger=True)
original_data_dir = path('geoserver/geoserver/data')
justcopy(original_data_dir, target_data_dir)
try:
config = path('geoserver/data/global.xml')
with open(config) as f:
xml = f.read()
m = re.search('proxyBaseUrl>([^<]+)', xml)
xml = xml[:m.start(1)] + \
"http://localhost:8080/geoserver" + xml[m.end(1):]
with open(config, 'w') as f:
f.write(xml)
except Exception as e:
print(e)
try:
config = path(
'geoserver/data/security/filter/geonode-oauth2/config.xml')
with open(config) as f:
xml = f.read()
m = re.search('accessTokenUri>([^<]+)', xml)
xml = xml[:m.start(1)] + \
"http://localhost:8000/o/token/" + xml[m.end(1):]
m = re.search('userAuthorizationUri>([^<]+)', xml)
xml = xml[:m.start(
1)] + "http://localhost:8000/o/authorize/" + xml[m.end(1):]
m = re.search('redirectUri>([^<]+)', xml)
xml = xml[:m.start(
1
)] + "http://localhost:8080/geoserver/index.html" + xml[m.end(1):]
m = re.search('checkTokenEndpointUrl>([^<]+)', xml)
xml = xml[:m.start(
1
)] + "http://localhost:8000/api/o/v4/tokeninfo/" + xml[m.end(1):]
m = re.search('logoutUri>([^<]+)', xml)
xml = xml[:m.start(
1)] + "http://localhost:8000/account/logout/" + xml[m.end(1):]
with open(config, 'w') as f:
f.write(xml)
except Exception as e:
print(e)
try:
config = path(
'geoserver/data/security/role/geonode REST role service/config.xml'
)
with open(config) as f:
xml = f.read()
m = re.search('baseUrl>([^<]+)', xml)
xml = xml[:m.start(1)] + "http://localhost:8000" + xml[m.end(1):]
with open(config, 'w') as f:
f.write(xml)
except Exception as e:
print(e)
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
# we use docker-compose for integration tests
if integration_tests:
return
# only start if using Geoserver backend
_backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND'])
if _backend == 'geonode.qgis_server' or 'geonode.geoserver' not in INSTALLED_APPS:
return
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = GEOSERVER_BASE_URL
if urlparse(GEOSERVER_BASE_URL).hostname != 'localhost':
print "Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'"
if not GEOSERVER_BASE_URL.endswith('/'):
print "Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'"
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / \
os.path.basename(dev_config['JETTY_RUNNER_URL'])
data_dir = path('geoserver/data').abspath()
geofence_dir = path('geoserver/data/geofence').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/misc/jetty-runner.xml').abspath()
jetty_port = urlparse(GEOSERVER_BASE_URL).port
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket_free = True
try:
s.bind(("127.0.0.1", jetty_port))
except socket.error as e:
socket_free = False
if e.errno == 98:
info('Port %s is already in use' % jetty_port)
else:
info(
'Something else raised the socket.error exception while checking port %s'
% jetty_port)
print(e)
finally:
s.close()
if socket_free:
# @todo - we should not have set workdir to the datadir but a bug in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
loggernullpath = os.devnull
# checking if our loggernullpath exists and if not, reset it to
# something manageable
if loggernullpath == "nul":
try:
open("../../downloaded/null.txt", 'w+').close()
except IOError as e:
print "Chances are that you have Geoserver currently running. You \
can either stop all servers with paver stop or start only \
the django application with paver start_django."
sys.exit(1)
loggernullpath = "../../downloaded/null.txt"
try:
sh(('java -version'))
except BaseException:
print "Java was not found in your path. Trying some other options: "
javapath_opt = None
if os.environ.get('JAVA_HOME', None):
print "Using the JAVA_HOME environment variable"
javapath_opt = os.path.join(
os.path.abspath(os.environ['JAVA_HOME']), "bin",
"java.exe")
elif options.get('java_path'):
javapath_opt = options.get('java_path')
else:
print "Paver cannot find java in the Windows Environment. \
Please provide the --java_path flag with your full path to \
java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + javapath_opt + '"'
sh((
'%(javapath)s -Xms512m -Xmx2048m -server -XX:+UseConcMarkSweepGC -XX:MaxPermSize=512m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
' -Dgeofence.dir=%(geofence_dir)s'
# ' -Dgeofence-ovr=geofence-datasource-ovr.properties'
# workaround for JAI sealed jar issue and jetty classloader
# ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --port %(jetty_port)i'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()))
info('Starting GeoServer on %s' % url)
# wait for GeoServer to start
started = waitfor(url)
info('The logs are available at %s' % log_file)
if not started:
# If applications did not start in time we will give the user a chance
# to inspect them and stop them manually.
info(('GeoServer never started properly or timed out.'
'It may still be running in the background.'))
sys.exit(1)
import os
import pkg_resources
import platform
import sys
from paver.easy import task, needs, path
from paver.setuputils import setup
root_dir = path(__file__).parent.abspath()
if root_dir not in sys.path:
sys.path.insert(0, str(root_dir))
import version
install_requires = [
'application_repository>=0.5', 'blinker', 'configobj',
'droplet-planning>=0.2', 'flatland-fork', 'geo-util', 'ipython',
'ipython-helpers>=0.4', 'jinja2', 'matplotlib>=1.5.0',
'microdrop-device-converter>=0.1.post5',
'microdrop-plugin-template>=1.1.post30', 'microdrop_utility>=0.4.post2',
'networkx', 'openpyxl', 'pandas>=0.17.1', 'path-helpers>=0.2.post4',
'paver>=1.2.4', 'pip-helpers>=0.6', 'pygtk_textbuffer_with_undo',
'pyparsing', 'pyutilib.component.core>=4.4.1',
'pyutilib.component.loader>=3.3.1', 'pyyaml', 'pyzmq', 'run-exe>=0.5',
'si-prefix>=0.4.post10', 'scipy', 'svgwrite', 'svg-model>=0.6',
'task_scheduler', 'tornado', 'wheeler.pygtkhelpers>=0.13.post17',
'zmq-plugin>=0.2.post2'
]
if platform.system() == 'Windows':
install_requires += ['pycairo-gtk2-win', 'pywin32']
else:
def start_geoserver(options):
"""
Start GeoServer with GeoNode extensions
"""
from geonode.settings import OGC_SERVER
GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION']
url = "http://localhost:8080/geoserver/"
if GEOSERVER_BASE_URL != url:
print 'your GEOSERVER_BASE_URL does not match %s' % url
sys.exit(1)
download_dir = path('downloaded').abspath()
jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL)
data_dir = path('geoserver/data').abspath()
web_app = path('geoserver/geoserver').abspath()
log_file = path('geoserver/jetty.log').abspath()
config = path('scripts/misc/jetty-runner.xml').abspath()
# @todo - we should not have set workdir to the datadir but a bug in geoserver
# prevents geonode security from initializing correctly otherwise
with pushd(data_dir):
javapath = "java"
loggernullpath = os.devnull
# checking if our loggernullpath exists and if not, reset it to something manageable
if loggernullpath == "nul":
try:
open("../../downloaded/null.txt", 'w+').close()
except IOError, e:
print "Chances are that you have Geoserver currently running. You \
can either stop all servers with paver stop or start only \
the django application with paver start_django."
sys.exit(1)
loggernullpath = "../../downloaded/null.txt"
try:
sh(('java -version'))
except:
print "Java was not found in your path. Trying some other options: "
javapath_opt = None
if os.environ.get('JAVA_HOME', None):
print "Using the JAVA_HOME environment variable"
javapath_opt = os.path.join(
os.path.abspath(os.environ['JAVA_HOME']), "bin",
"java.exe")
elif options.get('java_path'):
javapath_opt = options.get('java_path')
else:
print "Paver cannot find java in the Windows Environment. \
Please provide the --java_path flag with your full path to \
java.exe e.g. --java_path=C:/path/to/java/bin/java.exe"
sys.exit(1)
# if there are spaces
javapath = 'START /B "" "' + javapath_opt + '"'
sh((
'%(javapath)s -Xmx512m -XX:MaxPermSize=256m'
' -DGEOSERVER_DATA_DIR=%(data_dir)s'
# workaround for JAI sealed jar issue and jetty classloader
' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true'
' -jar %(jetty_runner)s'
' --log %(log_file)s'
' %(config)s'
' > %(loggernullpath)s &' % locals()))
