def _adjust_options(): """ Set default sphinx and gh_pages options. """ if options.get('_github_tools_options_adjusted') is None: options.setdefault('sphinx', Bunch()) options.setdefault('gh_pages', Bunch()) options.sphinx.docroot = docroot \ = path( options.sphinx.get('docroot', 'docs')) options.sphinx._buildir = buildir = \ docroot / options.sphinx.get('builddir', 'build') options.sphinx._sourcedir = \ docroot / options.sphinx.get('sourcedir', 'source') options.sphinx._doctrees = buildir / "doctrees" options.sphinx._htmldir = htmldir = \ buildir / 'html' gh_pages_root = options.gh_pages.get('root', None) if gh_pages_root is None: options.gh_pages.root = htmldir else: options.gh_pages.root = path(gh_pages_root) gh_pages_htmlroot = options.gh_pages.get('htmlroot', None) if gh_pages_htmlroot is None: options.gh_pages.htmlroot = options.gh_pages.root else: options.gh_pages.htmlroot = path(gh_pages_htmlroot) options.gh_pages.setdefault('remote_name', 'origin') options.gh_pages.setdefault('master_branch', 'master') options._github_tools_options_adjusted = True
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" fast = options.get('fast', False) download_dir = path('downloaded') if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path('geoserver') geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL) jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) data_dir = download_dir / os.path.basename(DATA_DIR_URL) grab(GEOSERVER_URL, geoserver_bin, "geoserver binary") grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner") grab(DATA_DIR_URL, data_dir, "data dir") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir / 'geoserver' if not webapp_dir: webapp_dir.makedirs() print 'extracting geoserver' with zipfile.ZipFile(geoserver_bin, "r") as z: z.extractall(webapp_dir) _install_data_dir()
def _install_data_dir(): data_dir = path('geoserver/data') if data_dir.exists(): data_dir.rmtree() geoserver_dir = path('geoserver') download_dir = path('downloaded') data_dir_zip = download_dir / os.path.basename(DATA_DIR_URL) if os.path.exists(data_dir_zip): print 'extracting datadir' with zipfile.ZipFile(data_dir_zip, "r") as z: z.extractall(geoserver_dir) config = ( geoserver_dir / 'data/security/auth/geonodeAuthProvider/config.xml' ) with open(config) as f: xml = f.read() m = re.search('baseUrl>([^<]+)', xml) xml = xml[:m.start(1)] + "http://localhost:" + GEM_GEONODE_PORT + "/" + xml[m.end(1):] with open(config, 'w') as f: f.write(xml) else: print 'data_dir_zip not found, unable to extract and configure'
def _install_data_dir(): target_data_dir = path('geoserver/data') if target_data_dir.exists(): try: target_data_dir.rmtree() except OSError: _robust_rmtree(target_data_dir, logger=True) original_data_dir = path('geoserver/geoserver/data') justcopy(original_data_dir, target_data_dir) try: config = path( 'geoserver/data/global.xml') with open(config) as f: xml = f.read() m = re.search('proxyBaseUrl>([^<]+)', xml) xml = xml[:m.start(1)] + \ "http://localhost:8080/geoserver" + xml[m.end(1):] with open(config, 'w') as f: f.write(xml) except Exception as e: print(e) try: config = path( 'geoserver/data/security/filter/geonode-oauth2/config.xml') with open(config) as f: xml = f.read() m = re.search('accessTokenUri>([^<]+)', xml) xml = xml[:m.start(1)] + \ "http://localhost:8000/o/token/" + xml[m.end(1):] m = re.search('userAuthorizationUri>([^<]+)', xml) xml = xml[:m.start( 1)] + "http://localhost:8000/o/authorize/" + xml[m.end(1):] m = re.search('redirectUri>([^<]+)', xml) xml = xml[:m.start( 1)] + "http://localhost:8080/geoserver/index.html" + xml[m.end(1):] m = re.search('checkTokenEndpointUrl>([^<]+)', xml) xml = xml[:m.start( 1)] + "http://localhost:8000/api/o/v4/tokeninfo/" + xml[m.end(1):] m = re.search('logoutUri>([^<]+)', xml) xml = xml[:m.start( 1)] + "http://localhost:8000/account/logout/" + xml[m.end(1):] with open(config, 'w') as f: f.write(xml) except Exception as e: print(e) try: config = path( 'geoserver/data/security/role/geonode REST role service/config.xml') with open(config) as f: xml = f.read() m = re.search('baseUrl>([^<]+)', xml) xml = xml[:m.start(1)] + "http://localhost:8000" + xml[m.end(1):] with open(config, 'w') as f: f.write(xml) except Exception as e: print(e)
def __init__(self, name, source, out, params, cache): self.name = name self.source = path(source) self.out = path(out) self.params = dict(params) self.cache = cache self.config_js = self.source.joinpath(self.params.pop('mainConfigFile', 'config.js'))
def build_objc_static_lib(dest_path, additional_sources=[], addition_include_dirs=[]): # Compile Tapstream objc sources into static library sdk_root = sh('echo $(xcodebuild -version -sdk iphoneos Path)', capture=True).strip() simulator_sdk_root = sh('echo $(xcodebuild -version -sdk iphonesimulator Path)', capture=True).strip() include_dirs = listify(['objc/Core', 'objc/Tapstream', '/usr/local/include/']+addition_include_dirs, prefix='-I') core_sources = list(path('objc/Core').walkfiles('*.m')) inputs = core_sources + list(path('objc/Tapstream').walkfiles('*.m')) + [path(p) for p in additional_sources] sh('rm -f ./*.o') sh('xcrun -sdk iphoneos clang -isysroot %s -miphoneos-version-min=4.3 -arch armv7 -fno-objc-arc %s -c %s' % ( sdk_root, include_dirs, listify(inputs) )) sh('xcrun -sdk iphoneos ar rcu objc/TapstreamArm7.a ./*.o') sh('rm ./*.o') sh('xcrun -sdk iphoneos clang -isysroot %s -miphoneos-version-min=4.3 -arch armv7s -fno-objc-arc %s -c %s' % ( sdk_root, include_dirs, listify(inputs) )) sh('xcrun -sdk iphoneos ar rcu objc/TapstreamArm7s.a ./*.o') sh('rm ./*.o') sh('xcrun -sdk iphonesimulator clang -isysroot %s -miphoneos-version-min=4.3 -arch i386 -fobjc-abi-version=2 -fno-objc-arc %s -c %s' % ( simulator_sdk_root, include_dirs, listify(inputs) )) sh('xcrun -sdk iphonesimulator ar rcu objc/Tapstreami386.a ./*.o') sh('rm ./*.o') sh('xcrun -sdk iphoneos lipo -create objc/TapstreamArm7.a objc/TapstreamArm7s.a objc/Tapstreami386.a -output %s' % dest_path) sh('rm objc/Tapstream*.a')
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" fast = options.get('fast', False) download_dir = path('downloaded') if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path('geoserver') geoserver_bin = path('geoserver_ext/target/geoserver.war') if not geoserver_bin.exists(): geoserver_bin = download_dir / os.path.basename(ROGUE_GEOSERVER_URL) grab(ROGUE_GEOSERVER_URL, geoserver_bin, "geoserver binary") jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner") data_dir = download_dir / os.path.basename(DATA_DIR_URL) grab(DATA_DIR_URL, data_dir, "data dir") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir if not webapp_dir: webapp_dir.makedirs() print 'extracting geoserver' z = zipfile.ZipFile(geoserver_bin, "r") z.extractall(webapp_dir) # Set the geonode auth config to dev port 8000 sh("perl -pi.back -e 's/localhost/localhost:8000/g;' geoserver/data/security/auth/geonodeAuthProvider/config.xml")
def package_tar_gz(options): """Package a .tar.gz distribution""" import tarfile package_name = options.get("package_name", None) if package_name is None: raise Exception("Package name required") filename = path("%s/%s.tar.gz" % (STAGE_DIR, package_name)) if filename.exists(): info("Package %s already exists" % filename) return with pushd(STAGE_DIR): stage_path = "%s/%s" % (STAGE_DIR, package_name) if not path(stage_path).exists(): raise Exception("Directory %s does not exist" % stage_path) tar = tarfile.open(filename, "w:gz") tar.add(package_name) tar.close()
def setup_geoserver(): download_dir = path('downloaded') if not download_dir.exists(): download_dir.makedirs() geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL) jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) grab(GEOSERVER_URL, geoserver_bin, "geoserver binary") grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner") geoserver_dir = path('geoserver') if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir / 'geoserver' if not webapp_dir: webapp_dir.makedirs() print 'extracting geoserver' z = zipfile.ZipFile(geoserver_bin, "r") z.extractall(webapp_dir) target_data_dir = path('geoserver/data') if not target_data_dir.exists(): target_data_dir.makedirs() _install_data_dir()
def start_geoserver(options): download_dir = path('downloaded').abspath() jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) data_dir = path('geoserver/data').abspath() web_app = path('geoserver/geoserver').abspath() log_file = path('geoserver/jetty.log').abspath() config = path('scripts/jetty-runner.xml').abspath() with pushd(data_dir): sh(('java -Xmx512m -XX:MaxPermSize=1024m' ' -DGEOSERVER_DATA_DIR=%(data_dir)s' ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true' ' -jar %(jetty_runner)s' ' --log %(log_file)s' ' %(config)s' ' > /dev/null &' % locals() )) info('Starting GeoServer on %s' % GEOSERVER_BASE_URL) # wait for GeoServer to start started = waitfor(GEOSERVER_BASE_URL) info('The logs are available at %s' % log_file) if not started: # If applications did not start in time we will give the user a chance # to inspect them and stop them manually. info(('GeoServer never started properly or timed out.' 'It may still be running in the background.')) sys.exit(1)
def bundle(): """Bundle required Python packages as ZIP archive.""" packages = options.bundle.packages builddir = path("build")/"bundle" builddir.makedirs() errors = 0 for package in packages: try: info("bundle %s ..." % package) module = __import__(package) pkgfile = path(module.__file__) basedir = pkgfile.dirname() destdir = builddir/package if destdir.exists(): destdir.rmtree() basedir.copytree(destdir) except Exception as e: info("FAILED %s: %s" % (package, e)) errors += 1 archive = options.bundle.get("archive", "python-bundle.zip") stored_files = make_zip_archive(archive, builddir) # -- SUMMARY: message1 = "bundle {pkgno} packages into {archive} " message1 += "(with {errorno} errors, files: {fileno})" message = message1.format(archive=archive, pkgno=len(packages), errorno=errors, fileno=stored_files) output = info if errors: message = "FAILED: {message}".format(message=message) output = error output(message)
def get_system_sass_dirs(system): """ Return list of sass dirs that need to be compiled for the given system. :param system: name if the system for which to compile sass e.g. 'lms', 'cms' """ if system not in ('lms', 'cms'): raise ValueError('"system" must either be "lms" or "cms"') dirs = [] sass_dir = path(system) / "static" / "sass" css_dir = path(system) / "static" / "css" dependencies = SASS_LOOKUP_DEPENDENCIES.get(system, []) dirs.append({ "sass_source_dir": sass_dir, "css_destination_dir": css_dir, "lookup_paths": dependencies + [ sass_dir / "partials", sass_dir, ], }) if system == 'lms': dirs.append({ "sass_source_dir": path(system) / "static" / "certificates" / "sass", "css_destination_dir": path(system) / "static" / "certificates" / "css", "lookup_paths": [ sass_dir / "partials", sass_dir ], }) return dirs
def configure_paths(): """Configure our paths based on settings. Called immediately.""" edxapp_env = Env() if edxapp_env.feature_flags.get('USE_CUSTOM_THEME', False): theme_name = edxapp_env.env_tokens.get('THEME_NAME', '') parent_dir = path(edxapp_env.REPO_ROOT).abspath().parent theme_root = parent_dir / "themes" / theme_name COFFEE_DIRS.append(theme_root) sass_dir = theme_root / "static" / "sass" css_dir = theme_root / "static" / "css" if sass_dir.isdir(): css_dir.mkdir_p() SASS_DIRS.append(sass_dir) if edxapp_env.env_tokens.get("COMPREHENSIVE_THEME_DIR", ""): theme_dir = path(edxapp_env.env_tokens["COMPREHENSIVE_THEME_DIR"]) lms_sass = theme_dir / "lms" / "static" / "sass" lms_css = theme_dir / "lms" / "static" / "css" if lms_sass.isdir(): lms_css.mkdir_p() SASS_DIRS.append(lms_sass) cms_sass = theme_dir / "cms" / "static" / "sass" cms_css = theme_dir / "cms" / "static" / "css" if cms_sass.isdir(): cms_css.mkdir_p() SASS_DIRS.append(cms_sass)
def install_remote_tar(args, options): """Usage: install_remote_tar [-t|--target TARGET] ARCHIVE DESTINATION Extracts an archive in a directory. The archive is either a file on the local file system or a remote URL fetched by http/https. A file whose name is TARGET is looked for inside the archive and the directory containing the target is moved as the DESTINATION directory. Any exisiting directory will be removed before the new directory is copied. """ web_archive, destination = args target = getattr(options, 'target', 'index.html') temp_dir = path(tempfile.mkdtemp()) try: with TarExtract(web_archive) as tf: for ti in sorted(tf.getmembers(), key=lambda x: (x.name.count('/'), x.name)): name = path(ti.name) if name.basename() == target: target_root = name.dirname() break else: raise ValueError('Missing target: {}'.format(target)) tf.extractall(temp_dir) except Exception: temp_dir.rmtree() raise else: path(destination).rmtree() temp_dir.joinpath(target_root).move(destination)
def start_geoserver(options): """ Start GeoServer with GeoNode extensions """ from geonode.settings import OGC_SERVER GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION'] url = "http://localhost:8080/geoserver/" if GEOSERVER_BASE_URL != url: print 'your GEOSERVER_BASE_URL does not match %s' % url sys.exit(1) download_dir = path('downloaded').abspath() jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) data_dir = path('geoserver/data').abspath() web_app = path('geoserver/geoserver').abspath() log_file = path('geoserver/jetty.log').abspath() config = path('scripts/misc/jetty-runner.xml').abspath() # @todo - we should not have set workdir to the datadir but a bug in geoserver # prevents geonode security from initializing correctly otherwise with pushd(data_dir): javapath = "java" loggernullpath = "/dev/null" try: sh(('java -version')) except: if not options.get('java_path', None): print "Paver cannot find java in the Windows Environment. Please provide the --java_path flag with your full path to java.exe e.g. --java_path=C:/path/to/java/bin/java.exe" sys.exit(1) # if there are spaces javapath = 'START /B "" "' + options['java_path'] + '"' # cmd log file needs to exist in windows # using folder from .gitignore open("../../downloaded/null.txt", 'w+').close() loggernullpath = "../../downloaded/null.txt" sh(( '%(javapath)s -Xmx512m -XX:MaxPermSize=256m' ' -DGEOSERVER_DATA_DIR=%(data_dir)s' # workaround for JAI sealed jar issue and jetty classloader ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true' ' -jar %(jetty_runner)s' ' --log %(log_file)s' ' %(config)s' ' > %(loggernullpath)s &' % locals() )) info('Starting GeoServer on %s' % url) # wait for GeoServer to start started = waitfor(url) info('The logs are available at %s' % log_file) if not started: # If applications did not start in time we will give the user a chance # to inspect them and stop them manually. info(('GeoServer never started properly or timed out.' 'It may still be running in the background.')) sys.exit(1)
def configure_paths(): """Configure our paths based on settings. Called immediately.""" edxapp_env = Env() # if edxapp_env.feature_flags.get('USE_CUSTOM_THEME', False): theme_name = 'default' parent_dir = path(COMP_THEME_DIR).abspath().parent theme_root = parent_dir / "themes" / theme_name COFFEE_DIRS.append(theme_root) sass_dir = theme_root / "static" / "sass" css_dir = theme_root / "static" / "css" if sass_dir.isdir(): css_dir.mkdir_p() SASS_DIRS.append(sass_dir) # if edxapp_env.env_tokens.get("COMP_THEME_DIR", ""): theme_dir = path(COMP_THEME_DIR) lms_sass = theme_dir / "static" / "sass" lms_css = theme_dir / "static" / "css" if lms_sass.isdir(): lms_css.mkdir_p() SASS_DIRS.append(lms_sass) studio_sass = theme_dir / "studio" / "static" / "sass" studio_css = theme_dir / "studio" / "static" / "css" if studio_sass.isdir(): studio_css.mkdir_p() SASS_DIRS.append(studio_sass)
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" fast = options.get("fast", False) download_dir = path("downloaded") if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path("geoserver") geoserver_bin = download_dir / os.path.basename(GEOSERVER_URL) jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) grab(GEOSERVER_URL, geoserver_bin, "geoserver binary") grab(JETTY_RUNNER_URL, jetty_runner, "jetty runner") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir / "geoserver" if not webapp_dir: webapp_dir.makedirs() print "extracting geoserver" z = zipfile.ZipFile(geoserver_bin, "r") z.extractall(webapp_dir) _install_data_dir()
def package_tar_gz(options): """Package a .tar.gz distribution""" import tarfile package_name = options.get('package_name', None) if package_name is None: raise Exception('Package name required') filename = path('%s/%s.tar.gz' % (STAGE_DIR, package_name)) if filename.exists(): info('Package %s already exists' % filename) return with pushd(STAGE_DIR): stage_path = '%s/%s' % (STAGE_DIR, package_name) if not path(stage_path).exists(): raise Exception('Directory %s does not exist' % stage_path) tar = tarfile.open(filename, 'w:gz') tar.add(package_name) tar.close()
def docs(): import markdown from django.template import Context, Template from django.conf import settings settings.configure() path('builds/docs').rmtree() path('builds/docs').makedirs() path.copy(path('./docs/bootstrap.min.css'), path('./builds/docs/')) path.copy(path('./docs/pygments.css'), path('./builds/docs/')) with open('docs/base.html') as f: base_template = Template(f.read()) with open('docs/docs.md') as f: md_template = Template(f.read()) with pushd('builds/docs'): for platform in ('android', 'ios', 'mac', 'win8', 'winphone'): md = md_template.render(Context({'platform': platform, 'pretty_platform': PRETTY_PLATFORMS[platform]})) with open('docs_%s.md' % platform, 'w') as f: f.write(md) md = markdown.markdown(md, ['fenced_code', 'codehilite']) page = base_template.render(Context({'md': md})) with open('docs_%s.html' % platform, 'w') as f: f.write(page) path.copy(path('phonegap/PhoneGap.md'), path('./builds/docs/docs_phonegap.md')) with open('phonegap/PhoneGap.md') as f: md = markdown.markdown(f.read(), ['fenced_code', 'codehilite']) page = base_template.render(Context({'md': md})) with pushd('builds/docs'): with open('docs_phonegap.html', 'w') as f: f.write(page)
def setup_geoserver(options): """Prepare a testing instance of GeoServer.""" download_dir = path('downloaded') if not download_dir.exists(): download_dir.makedirs() geoserver_dir = path('geoserver') geoserver_bin = download_dir / os.path.basename(dev_config['GEOSERVER_URL']) jetty_runner = download_dir / os.path.basename(dev_config['JETTY_RUNNER_URL']) grab(options.get('geoserver', dev_config['GEOSERVER_URL']), geoserver_bin, "geoserver binary") grab(options.get('jetty', dev_config['JETTY_RUNNER_URL']), jetty_runner, "jetty runner") if not geoserver_dir.exists(): geoserver_dir.makedirs() webapp_dir = geoserver_dir / 'geoserver' if not webapp_dir: webapp_dir.makedirs() print 'extracting geoserver' z = zipfile.ZipFile(geoserver_bin, "r") z.extractall(webapp_dir) _install_data_dir()
def setup_testdata(): """Create test databases and load test data""" test_database_parameters = { # suite: has_testdata 'apiso': True, 'cite': True, 'harvesting': False, 'manager': False } # remove CITE database so we can build fresh for suite in test_database_parameters.keys(): dbfile = 'tests/suites/%s/data/records.db' % suite if os.path.isfile(dbfile): os.remove(dbfile) for database, has_testdata in test_database_parameters.iteritems(): info('Setting up test database %s' % database) cfg = path('tests/suites/%s/default.cfg' % database) sh('pycsw-admin.py -c setup_db -f %s' % cfg) if has_testdata: datapath = path('tests/suites/%s/data' % database) info('Loading test data from %s' % datapath) sh('pycsw-admin.py -c load_records -f %s -p %s' % (cfg, datapath))
def test_watch_theme_assets(self): """ Test the Paver watch asset tasks with theming enabled. """ self.expected_sass_directories.extend([ path(TEST_THEME_DIR) / 'lms/static/sass', path(TEST_THEME_DIR) / 'lms/static/sass/partials', path(TEST_THEME_DIR) / 'cms/static/sass', path(TEST_THEME_DIR) / 'cms/static/sass/partials', ]) with patch('pavelib.assets.SassWatcher.register') as mock_register: with patch('pavelib.assets.Observer.start'): with patch('pavelib.assets.execute_webpack_watch') as mock_webpack: call_task( 'pavelib.assets.watch_assets', options={ "background": True, "theme_dirs": [TEST_THEME_DIR.dirname()], "themes": [TEST_THEME_DIR.basename()] }, ) self.assertEqual(mock_register.call_count, 2) self.assertEqual(mock_webpack.call_count, 1) sass_watcher_args = mock_register.call_args_list[0][0] self.assertIsInstance(sass_watcher_args[0], Observer) self.assertIsInstance(sass_watcher_args[1], list) self.assertItemsEqual(sass_watcher_args[1], self.expected_sass_directories)
def sphinx_index(): """Update all indices.""" path(options.sphinx.index_path).makedirs() command = 'indexer --all --config %s' % options.sphinx.config if is_sphinx_daemon_running(): command += ' --rotate' sh(command)
def make_phonegap(): path('builds/phonegap').rmtree() path('builds/phonegap').makedirs() sh('cp ./phonegap/tapstream.js ./builds/phonegap/') sh('cp -r ./phonegap/objc_plugin ./builds/phonegap/') sh('cp -r ./phonegap/java_plugin ./builds/phonegap/') sh('cp builds/android/Tapstream.jar ./builds/phonegap') sh('cp -r builds/ios/Tapstream ./builds/phonegap')
def setup_ceph_client(options): ceph_client_log_dir = path('geonode/cephgeo/logs') if not ceph_client_log_dir.exists(): ceph_client_log_dir.makedirs() ceph_client_log_file = path('geonode/cephgeo/logs/ceph_storage.log') if not ceph_client_log_file.isfile(): ceph_client_log_file.touch()
def _package_java_whitelabel(): path('builds/android-whitelabel').rmtree() path('builds/android-whitelabel').makedirs() path.copy(path('./java-whitelabel/Tapstream/build/jar/Tapstream.jar'), path('./builds/android-whitelabel/ConversionTracker.jar')) path('builds/TapstreamSDK-android-whitelabel.zip').remove() with pushd('builds/android-whitelabel'): sh('7z a -tzip ../TapstreamSDK-android-whitelabel.zip ConversionTracker.jar') path('java-whitelabel').rmtree()
def _package_java(): path('builds/android').rmtree() path('builds/android').makedirs() path.copy(path('./java/Tapstream/build/jar/Tapstream.jar'), path('./builds/android/')) path.copy(path('./java/Tapstream/build/jar/Tapstream.jar'), path('./examples/Android/Example/libs/')) path('builds/TapstreamSDK-android.zip').remove() with pushd('builds/android'): sh('7z a -tzip ../TapstreamSDK-android.zip Tapstream.jar')
def _package_java(): path('builds/android').rmtree() path('builds/android').makedirs() path.copy(path('./java/Tapstream/build/jar/Tapstream.jar'), path('./builds/android/')) path.copy(path('./java/Tapstream/build/jar/Tapstream.jar'), path('./examples/Android/Example/libs/')) path('builds/tapstream-%s-android.zip' % VERSION).remove() with pushd('builds/android'): _zip("../tapstream-%s-android.zip" % VERSION, 'Tapstream.jar')
def setup_celery(options): celery_pid_dir = path('celery/pid') if not celery_pid_dir.exists(): celery_pid_dir.makedirs() celery_log_dir = path('celery/log') if not celery_log_dir.exists(): celery_log_dir.makedirs()
def _package_java_whitelabel(): path('builds/android-whitelabel').rmtree() path('builds/android-whitelabel').makedirs() path.copy(path('./java-whitelabel/Tapstream/build/jar/Tapstream.jar'), path('./builds/android-whitelabel/ConversionTracker.jar')) path('builds/tapstream-%s-android-whitelabel.zip' % VERSION).remove() with pushd('builds/android-whitelabel'): _zip("../tapstream-%s-android-whitelabel.zip" % VERSION, 'ConversionTracker.jar') path('java-whitelabel').rmtree()
# setup baseline paths ALL_SYSTEMS = ['lms', 'studio'] LMS = 'lms' CMS = 'cms' SYSTEMS = { 'lms': LMS, 'cms': CMS, 'studio': CMS } # Common lookup paths that are added to the lookup paths for all sass compilations COMMON_LOOKUP_PATHS = [ path("common/static"), path("common/static/sass"), path('node_modules/@edx'), path('node_modules'), ] # A list of NPM installed libraries that should be copied into the common # static directory. # If string ends with '/' then all file in the directory will be copied. NPM_INSTALLED_LIBRARIES = [ 'backbone.paginator/lib/backbone.paginator.js', 'backbone/backbone.js', 'bootstrap/dist/js/bootstrap.bundle.js', 'hls.js/dist/hls.js', 'jquery-migrate/dist/jquery-migrate.js', 'jquery.scrollto/jquery.scrollTo.js',
def compile_sass(options): """ Compile Sass to CSS. If command is called without any arguments, it will only compile lms, cms sass for the open source theme. And none of the comprehensive theme's sass would be compiled. If you want to compile sass for all comprehensive themes you will have to run compile_sass specifying all the themes that need to be compiled.. The following is a list of some possible ways to use this command. Command: paver compile_sass Description: compile sass files for both lms and cms. If command is called like above (i.e. without any arguments) it will only compile lms, cms sass for the open source theme. None of the theme's sass will be compiled. Command: paver compile_sass --theme-dirs /edx/app/edxapp/edx-platform/themes --themes=red-theme Description: compile sass files for both lms and cms for 'red-theme' present in '/edx/app/edxapp/edx-platform/themes' Command: paver compile_sass --theme-dirs=/edx/app/edxapp/edx-platform/themes --themes red-theme stanford-style Description: compile sass files for both lms and cms for 'red-theme' and 'stanford-style' present in '/edx/app/edxapp/edx-platform/themes'. Command: paver compile_sass --system=cms --theme-dirs /edx/app/edxapp/edx-platform/themes /edx/app/edxapp/edx-platform/common/test/ --themes red-theme stanford-style test-theme Description: compile sass files for cms only for 'red-theme', 'stanford-style' and 'test-theme' present in '/edx/app/edxapp/edx-platform/themes' and '/edx/app/edxapp/edx-platform/common/test/'. """ debug = options.get('debug') force = options.get('force') systems = get_parsed_option(options, 'system', ALL_SYSTEMS) themes = get_parsed_option(options, 'themes', []) theme_dirs = get_parsed_option(options, 'theme_dirs', []) if not theme_dirs and themes: # We can not compile a theme sass without knowing the directory that contains the theme. raise ValueError('theme-dirs must be provided for compiling theme sass.') if themes and theme_dirs: themes = get_theme_paths(themes=themes, theme_dirs=theme_dirs) # Compile sass for OpenEdx theme after comprehensive themes if None not in themes: themes.append(None) timing_info = [] dry_run = tasks.environment.dry_run compilation_results = {'success': [], 'failure': []} print("\t\tStarted compiling Sass:") # compile common sass files is_successful = _compile_sass('common', None, debug, force, timing_info) if is_successful: print("Finished compiling 'common' sass.") compilation_results['success' if is_successful else 'failure'].append('"common" sass files.') for system in systems: for theme in themes: print("Started compiling '{system}' Sass for '{theme}'.".format(system=system, theme=theme or 'system')) # Compile sass files is_successful = _compile_sass( system=system, theme=path(theme) if theme else None, debug=debug, force=force, timing_info=timing_info ) if is_successful: print("Finished compiling '{system}' Sass for '{theme}'.".format( system=system, theme=theme or 'system' )) compilation_results['success' if is_successful else 'failure'].append('{system} sass for {theme}.'.format( system=system, theme=theme or 'system', )) print("\t\tFinished compiling Sass:") if not dry_run: for sass_dir, css_dir, duration in timing_info: print(">> {} -> {} in {}s".format(sass_dir, css_dir, duration)) if compilation_results['success']: print("\033[92m\nSuccessful compilations:\n--- " + "\n--- ".join(compilation_results['success']) + "\n\033[00m") if compilation_results['failure']: print("\033[91m\nFailed compilations:\n--- " + "\n--- ".join(compilation_results['failure']) + "\n\033[00m")
def deb(options): """ Creates debian packages. Example uses: paver deb paver deb -k 12345 paver deb -k 12345 -p geonode/testing """ key = options.get('key', None) ppa = options.get('ppa', None) version, simple_version = versions() info('Creating package for GeoNode version %s' % version) # Get rid of any uncommitted changes to debian/changelog info('Getting rid of any uncommitted changes in debian/changelog') sh('git checkout package/debian/changelog') # Workaround for git-dch bug # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580 sh('rm -rf %s/.git' % (os.path.realpath('package'))) sh('ln -s %s %s' % (os.path.realpath('.git'), os.path.realpath('package'))) with pushd('package'): # Install requirements # sh('sudo apt-get -y install debhelper devscripts git-buildpackage') # sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s' # ' --id-length=6 --ignore-branch --release' % (simple_version))) # In case you publish from Ubuntu Xenial (git-dch is removed from upstream) # use the following line instead: # sh(('gbp dch --spawn-editor=snapshot --git-author --new-version=%s' # ' --id-length=6 --ignore-branch --release' % (simple_version))) distribution = "bionic" # sh(('gbp dch --distribution=%s --force-distribution --spawn-editor=snapshot --git-author --new-version=%s' # ' --id-length=6 --ignore-branch --release' % (distribution, simple_version))) deb_changelog = path('debian') / 'changelog' for idx, line in enumerate( fileinput.input([deb_changelog], inplace=True)): if idx == 0: print "geonode (%s) %s; urgency=high" % (simple_version, distribution), else: print line.replace("urgency=medium", "urgency=high"), # Revert workaround for git-dhc bug sh('rm -rf .git') if key is None and ppa is None: print("A local installable package") sh('debuild -uc -us -A') elif key is None and ppa is not None: print("A sources package, signed by daemon") sh('debuild -S') elif key is not None and ppa is None: print("A signed installable package") sh('debuild -k%s -A' % key) elif key is not None and ppa is not None: print("A signed, source package") sh('debuild -k%s -S' % key) if ppa is not None: sh('dput ppa:%s geonode_%s_source.changes' % (ppa, simple_version))
def _install_data_dir(): target_data_dir = path('geoserver/data') if target_data_dir.exists(): try: target_data_dir.rmtree() except OSError: _robust_rmtree(target_data_dir, logger=True) original_data_dir = path('geoserver/geoserver/data') justcopy(original_data_dir, target_data_dir) try: config = path('geoserver/data/global.xml') with open(config) as f: xml = f.read() m = re.search('proxyBaseUrl>([^<]+)', xml) xml = xml[:m.start(1)] + \ "http://localhost:8080/geoserver" + xml[m.end(1):] with open(config, 'w') as f: f.write(xml) except Exception as e: print(e) try: config = path( 'geoserver/data/security/filter/geonode-oauth2/config.xml') with open(config) as f: xml = f.read() m = re.search('accessTokenUri>([^<]+)', xml) xml = xml[:m.start(1)] + \ "http://localhost:8000/o/token/" + xml[m.end(1):] m = re.search('userAuthorizationUri>([^<]+)', xml) xml = xml[:m.start( 1)] + "http://localhost:8000/o/authorize/" + xml[m.end(1):] m = re.search('redirectUri>([^<]+)', xml) xml = xml[:m.start( 1 )] + "http://localhost:8080/geoserver/index.html" + xml[m.end(1):] m = re.search('checkTokenEndpointUrl>([^<]+)', xml) xml = xml[:m.start( 1 )] + "http://localhost:8000/api/o/v4/tokeninfo/" + xml[m.end(1):] m = re.search('logoutUri>([^<]+)', xml) xml = xml[:m.start( 1)] + "http://localhost:8000/account/logout/" + xml[m.end(1):] with open(config, 'w') as f: f.write(xml) except Exception as e: print(e) try: config = path( 'geoserver/data/security/role/geonode REST role service/config.xml' ) with open(config) as f: xml = f.read() m = re.search('baseUrl>([^<]+)', xml) xml = xml[:m.start(1)] + "http://localhost:8000" + xml[m.end(1):] with open(config, 'w') as f: f.write(xml) except Exception as e: print(e)
def start_geoserver(options): """ Start GeoServer with GeoNode extensions """ # we use docker-compose for integration tests if integration_tests: return # only start if using Geoserver backend _backend = os.environ.get('BACKEND', OGC_SERVER['default']['BACKEND']) if _backend == 'geonode.qgis_server' or 'geonode.geoserver' not in INSTALLED_APPS: return GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION'] url = GEOSERVER_BASE_URL if urlparse(GEOSERVER_BASE_URL).hostname != 'localhost': print "Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'" if not GEOSERVER_BASE_URL.endswith('/'): print "Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'" sys.exit(1) download_dir = path('downloaded').abspath() jetty_runner = download_dir / \ os.path.basename(dev_config['JETTY_RUNNER_URL']) data_dir = path('geoserver/data').abspath() geofence_dir = path('geoserver/data/geofence').abspath() web_app = path('geoserver/geoserver').abspath() log_file = path('geoserver/jetty.log').abspath() config = path('scripts/misc/jetty-runner.xml').abspath() jetty_port = urlparse(GEOSERVER_BASE_URL).port import socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) socket_free = True try: s.bind(("127.0.0.1", jetty_port)) except socket.error as e: socket_free = False if e.errno == 98: info('Port %s is already in use' % jetty_port) else: info( 'Something else raised the socket.error exception while checking port %s' % jetty_port) print(e) finally: s.close() if socket_free: # @todo - we should not have set workdir to the datadir but a bug in geoserver # prevents geonode security from initializing correctly otherwise with pushd(data_dir): javapath = "java" loggernullpath = os.devnull # checking if our loggernullpath exists and if not, reset it to # something manageable if loggernullpath == "nul": try: open("../../downloaded/null.txt", 'w+').close() except IOError as e: print "Chances are that you have Geoserver currently running. You \ can either stop all servers with paver stop or start only \ the django application with paver start_django." sys.exit(1) loggernullpath = "../../downloaded/null.txt" try: sh(('java -version')) except BaseException: print "Java was not found in your path. Trying some other options: " javapath_opt = None if os.environ.get('JAVA_HOME', None): print "Using the JAVA_HOME environment variable" javapath_opt = os.path.join( os.path.abspath(os.environ['JAVA_HOME']), "bin", "java.exe") elif options.get('java_path'): javapath_opt = options.get('java_path') else: print "Paver cannot find java in the Windows Environment. \ Please provide the --java_path flag with your full path to \ java.exe e.g. --java_path=C:/path/to/java/bin/java.exe" sys.exit(1) # if there are spaces javapath = 'START /B "" "' + javapath_opt + '"' sh(( '%(javapath)s -Xms512m -Xmx2048m -server -XX:+UseConcMarkSweepGC -XX:MaxPermSize=512m' ' -DGEOSERVER_DATA_DIR=%(data_dir)s' ' -Dgeofence.dir=%(geofence_dir)s' # ' -Dgeofence-ovr=geofence-datasource-ovr.properties' # workaround for JAI sealed jar issue and jetty classloader # ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true' ' -jar %(jetty_runner)s' ' --port %(jetty_port)i' ' --log %(log_file)s' ' %(config)s' ' > %(loggernullpath)s &' % locals())) info('Starting GeoServer on %s' % url) # wait for GeoServer to start started = waitfor(url) info('The logs are available at %s' % log_file) if not started: # If applications did not start in time we will give the user a chance # to inspect them and stop them manually. info(('GeoServer never started properly or timed out.' 'It may still be running in the background.')) sys.exit(1)
import os import pkg_resources import platform import sys from paver.easy import task, needs, path from paver.setuputils import setup root_dir = path(__file__).parent.abspath() if root_dir not in sys.path: sys.path.insert(0, str(root_dir)) import version install_requires = [ 'application_repository>=0.5', 'blinker', 'configobj', 'droplet-planning>=0.2', 'flatland-fork', 'geo-util', 'ipython', 'ipython-helpers>=0.4', 'jinja2', 'matplotlib>=1.5.0', 'microdrop-device-converter>=0.1.post5', 'microdrop-plugin-template>=1.1.post30', 'microdrop_utility>=0.4.post2', 'networkx', 'openpyxl', 'pandas>=0.17.1', 'path-helpers>=0.2.post4', 'paver>=1.2.4', 'pip-helpers>=0.6', 'pygtk_textbuffer_with_undo', 'pyparsing', 'pyutilib.component.core>=4.4.1', 'pyutilib.component.loader>=3.3.1', 'pyyaml', 'pyzmq', 'run-exe>=0.5', 'si-prefix>=0.4.post10', 'scipy', 'svgwrite', 'svg-model>=0.6', 'task_scheduler', 'tornado', 'wheeler.pygtkhelpers>=0.13.post17', 'zmq-plugin>=0.2.post2' ] if platform.system() == 'Windows': install_requires += ['pycairo-gtk2-win', 'pywin32'] else:
def start_geoserver(options): """ Start GeoServer with GeoNode extensions """ from geonode.settings import OGC_SERVER GEOSERVER_BASE_URL = OGC_SERVER['default']['LOCATION'] url = "http://localhost:8080/geoserver/" if GEOSERVER_BASE_URL != url: print 'your GEOSERVER_BASE_URL does not match %s' % url sys.exit(1) download_dir = path('downloaded').abspath() jetty_runner = download_dir / os.path.basename(JETTY_RUNNER_URL) data_dir = path('geoserver/data').abspath() web_app = path('geoserver/geoserver').abspath() log_file = path('geoserver/jetty.log').abspath() config = path('scripts/misc/jetty-runner.xml').abspath() # @todo - we should not have set workdir to the datadir but a bug in geoserver # prevents geonode security from initializing correctly otherwise with pushd(data_dir): javapath = "java" loggernullpath = os.devnull # checking if our loggernullpath exists and if not, reset it to something manageable if loggernullpath == "nul": try: open("../../downloaded/null.txt", 'w+').close() except IOError, e: print "Chances are that you have Geoserver currently running. You \ can either stop all servers with paver stop or start only \ the django application with paver start_django." sys.exit(1) loggernullpath = "../../downloaded/null.txt" try: sh(('java -version')) except: print "Java was not found in your path. Trying some other options: " javapath_opt = None if os.environ.get('JAVA_HOME', None): print "Using the JAVA_HOME environment variable" javapath_opt = os.path.join( os.path.abspath(os.environ['JAVA_HOME']), "bin", "java.exe") elif options.get('java_path'): javapath_opt = options.get('java_path') else: print "Paver cannot find java in the Windows Environment. \ Please provide the --java_path flag with your full path to \ java.exe e.g. --java_path=C:/path/to/java/bin/java.exe" sys.exit(1) # if there are spaces javapath = 'START /B "" "' + javapath_opt + '"' sh(( '%(javapath)s -Xmx512m -XX:MaxPermSize=256m' ' -DGEOSERVER_DATA_DIR=%(data_dir)s' # workaround for JAI sealed jar issue and jetty classloader ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true' ' -jar %(jetty_runner)s' ' --log %(log_file)s' ' %(config)s' ' > %(loggernullpath)s &' % locals()))