def builddoc(builders='', rootdir=None, sphinxopts='', apidoc='false'): """ Build documentation. Requires Sphinx. :param builders: List of document builders separated with comma :param rootdir: Directory where to generate documents, per builder (optional) :param sphinxopts: Additional parameters to pass to Sphinx :param apidoc: If true, runs also apidoc. Defaults to false. """ builders = builders.split(',') if builders else ['html'] rootdir = rootdir if rootdir else build_join('docs') sphinxopts = sphinxopts or '-E' sourcedir = rel_join('docs') pysourcedir = rel_join('plugins/multiproject/multiproject') exclude_paths = ['tests', 'core/test', 'core/tests'] for builder in builders: docbuild = os.path.join(rootdir, builder) if not os.path.exists(docbuild): os.makedirs(docbuild) if get_bool_str(apidoc): local('rm -rf %s' % join(sourcedir, 'reference')) local('sphinx-apidoc -o %s/reference %s %s' % (sourcedir, pysourcedir, ' '.join(exclude_paths))) local('sphinx-build %s -b %s %s %s' % (sphinxopts, builder, sourcedir, docbuild)) logger.info('Built %s documentation to: %s' % (builder, docbuild))
def test_perf(params=''): """ Runs the performance tests against the configured service and produce the report in dist/ :params str params: Parameters to pass to Funkload bench Examples:: fab dist.test_perf fab dist.test_perf:"-c 1:15 -D 1" """ try: import funkload except ImportError: abort('Funkload module missing, please install it first') # Create report folder if needed report_dir = dist_join('report/html') if not os.path.exists(report_dir): os.makedirs(report_dir) # Run the Funkload tests in perf test folder with lcd(rel_join('tests/perf')): with settings(show('running','stdout')): local('fl-run-bench %s tests.py MultiprojectTestCase.test_smoke' % params) local('fl-build-report -o %s --html smoke-bench.xml' % report_dir) logger.info('Testing completed. Test report can be found in: %s' % report_dir)
def test_perf(params=''): """ Runs the performance tests against the configured service and produce the report in dist/ :params str params: Parameters to pass to Funkload bench Examples:: fab dist.test_perf fab dist.test_perf:"-c 1:15 -D 1" """ try: import funkload except ImportError: abort('Funkload module missing, please install it first') # Create report folder if needed report_dir = dist_join('report/html') if not os.path.exists(report_dir): os.makedirs(report_dir) # Run the Funkload tests in perf test folder with lcd(rel_join('tests/perf')): with settings(show('running', 'stdout')): local('fl-run-bench %s tests.py MultiprojectTestCase.test_smoke' % params) local('fl-build-report -o %s --html smoke-bench.xml' % report_dir) logger.info('Testing completed. Test report can be found in: %s' % report_dir)
def test(case='', config='tests.ini'): """ Runs the functional tests against the setup specified in configuration :param str case: Name or path to case file :param str config: Path to config file, relative to current directory Examples:: fab dist.test:smoke fab dist.test:path/to/case.py fab dist.test:smoke,~/firefox.ini fab dist.test:smoke,config=path/to/config.ini """ try: from nose.core import TestProgram from nose.plugins import Plugin except ImportError: TestProgram = None Plugin = object return abort('For running tests, Nose testing framework is required. Please install it first: "pip install nose"') if not case: return abort('Please provide either name or path to test case') # Determine the case file: name or path accepted webtests_dir = rel_join('tests/webtests') casepath = os.path.abspath(case) if case.endswith('.py') else join(webtests_dir, 'cases/%s.py' % case) configpath = os.path.join(os.curdir, os.path.expanduser(config)) logger.info('Running functional tests from: %s' % casepath) logger.info('Reading tests configuration from: %s' % configpath) class TestConfigPlugin(Plugin): """ Simple Nose plugin to set test configuration path to testcase:: class MyTestcase(unittest.TestCase) def setUp(self): self.config_path """ name = 'testconfig' can_configure = True enabled = True def options(self, parser, env): pass def configure(self, options, conf): pass def startTest(self, test): test_case = test.test.__class__ test_case.config_path = configpath TestProgram(argv=['fab', casepath], addplugins=[TestConfigPlugin()])
def buildetc(outdir=None, section=None): """ Builds the configuration file based given configuration values in ``fabfile.ini``. Parameters: - outdir: Option path to directory where the built configuration files are placed. Subdirectory is created for each section. - section: Optional section name, if you want to build only one section. In this case, no subdirectory is created. """ # Copy configuration files under build dir where they can be processed; config.config is a ConfigParser # object. config is a global object imported via fablib.api sections = [section] if section else config.get_setups_sections() # Iterate all sections unless the specific name is given for csection in sections: dest = csection.replace('setup:', '') # Set the target directory based on given parameters if section: build_etc = outdir elif outdir: build_etc = build_join(outdir, dest) else: build_etc = build_join('etc', dest) logger.info('Target directory for configuration files: %s' % build_etc) # Create target dir if needed if not os.path.exists(build_etc): os.makedirs(build_etc) # Copy the config files from etc/templates, place them in build/etc/<section> and replace the placeholders etc_src = rel_join('etc/templates') for configfile in get_files( etc_src, pattern='*.ini, *.conf, *.sql, *.sh, multiproject', recursive=True): # Copy the configuration file into build_etc directory, using the same directory structure configcopy = build_join(build_etc, configfile.replace(etc_src, './')) configcopy_dir = os.path.dirname(configcopy) if not os.path.exists(configcopy_dir): os.makedirs(configcopy_dir) shutil.copy(configfile, configcopy) # Replace placeholder in configfile logger.info('Building configuration file: %s' % configcopy) template = TemplateResource(configcopy) template.build(**dict(config.config.items(csection)))
def buildetc(outdir=None, section=None): """ Builds the configuration file based given configuration values in ``fabfile.ini``. Parameters: - outdir: Option path to directory where the built configuration files are placed. Subdirectory is created for each section. - section: Optional section name, if you want to build only one section. In this case, no subdirectory is created. """ # Copy configuration files under build dir where they can be processed; config.config is a ConfigParser # object. config is a global object imported via fablib.api sections = [section] if section else config.get_setups_sections() # Iterate all sections unless the specific name is given for csection in sections: dest = csection.replace('setup:', '') # Set the target directory based on given parameters if section: build_etc = outdir elif outdir: build_etc = build_join(outdir, dest) else: build_etc = build_join('etc', dest) logger.info('Target directory for configuration files: %s' % build_etc) # Create target dir if needed if not os.path.exists(build_etc): os.makedirs(build_etc) # Copy the config files from etc/templates, place them in build/etc/<section> and replace the placeholders etc_src = rel_join('etc/templates') for configfile in get_files(etc_src, pattern='*.ini, *.conf, *.sql, *.sh, multiproject', recursive=True): # Copy the configuration file into build_etc directory, using the same directory structure configcopy = build_join(build_etc, configfile.replace(etc_src, './')) configcopy_dir = os.path.dirname(configcopy) if not os.path.exists(configcopy_dir): os.makedirs(configcopy_dir) shutil.copy(configfile, configcopy) # Replace placeholder in configfile logger.info('Building configuration file: %s' % configcopy) template = TemplateResource(configcopy) template.build(**dict(config.config.items(csection)))
def build(release='false', compress='false', docs='', pkgs='tar', version='', ext='true', extbranch='master'): """ Create distributable packages. Builds eggs and tar.gz compressed packages, based on parameters. Also capable of downloading and patching external dependencies. :param release: Make release build or not. Release sets/increments the version number. Default 'false' :param compress: Compress js/css files nor not. Default 'false' :param docs: Names of the documentation targets to build. Default '' means no doc building :param pkgs: Package formats to build sources into, separated with space. Valid values: tar deb rpm :param version: Version number to set for whole package. Default '' -> take the version from VERSION.txt (or default to 1.0.0) :param ext: Build and include external modules into big package. Default is 'false'. If ext is 'all', builds also other than own forks (GitResources). :param extbranch: Defines from which branch the fork packages are to be built from. Examples:: fab dist.build fab dist.build:release=true,docs=html fab dist.build:compress=true,version=1.2.3,pkgs="deb tar rpm" .. NOTE:: Python modules get their version number from setup.py """ # NOTE: Fabric parameters are always in string format # Get the list of package formats (space delimeter) pkg_formats = pkgs.split(' ') # Determine the version: parameter vs. VERSION.txt vs. default if not version: version_path = os.path.join(PROJECT_DIR, 'VERSION.txt') if os.path.exists(version_path): version = set_version_in_file(version_path, version) else: version = '1.0.0' # Create package name from pkg name and version package_name = '%s-%s' % (PKG_NAME, version) pkg_join = lambda *path: join(BUILD_DIR, package_name, *path) logger.info('Preparing build env...') # Copy relevant files to build dir (so that they can be edited directly) shutil.rmtree(BUILD_DIR, ignore_errors=True) del SRC_DIRS[SRC_DIRS.index('libs')] del SRC_DIRS[SRC_DIRS.index('etc')] for src_dir in SRC_DIRS: shutil.copytree(src_dir, pkg_join(src_dir)) if not os.path.exists(DIST_DIR): os.makedirs(DIST_DIR) # Copy additional files shutil.copy('README.rst', pkg_join('README')) os.makedirs(pkg_join('scripts')) shutil.copy(rel_join('scripts/deploy.sh'), pkg_join('scripts/deploy.sh')) shutil.copy(rel_join('scripts/update.py'), pkg_join('scripts/update.py')) shutil.copytree(rel_join('scripts/hooks'), pkg_join('scripts/hooks')) shutil.copytree(rel_join('scripts/cron'), pkg_join('scripts/cron')) # Build documentation if docs: # List the target formats/builders builddoc(docs, pkg_join('docs')) else: # Ensure there is at least empty directory (for archive) os.makedirs(pkg_join('docs')) # Build configuration buildetc(outdir=pkg_join('etc'), section='DEFAULT') # Increment version of each plugin if making a release if get_bool_str(release): logger.info('Setting/incrementing version numbers...') for setuppy_path in get_files(pkg_join('plugins'), 'setup.py', recursive=True): # Check if plugin folder contains VERSION.txt (non-versioned file) version_path = os.path.join(os.path.dirname(setuppy_path), 'VERSION.txt') if not os.path.exists(version_path): logger.warning('VERSION.txt missing, using version found in setup.py') version_path = setuppy_path # Set version information in file. # NOTE: If version is empty, it is determined from version file (either VERSION.txt or setup.py) set_version_in_file(version_path, version) # Optional compress (edits copied files under build) if get_bool_str(compress): logger.info('Compressing files...') with settings(warn_only=True): # Compress theme resources for respath in get_files(pkg_join('themes'), '*.css', recursive=True): local('yui-compressor --charset utf-8 -o %s %s' % (respath, respath)) for respath in get_files(pkg_join('themes'), '*.js', recursive=True): local('yui-compressor --charset utf-8 -o %s %s' % (respath, respath)) # Compress plugin resources for respath in get_files(pkg_join('plugins'), '*.css', recursive=True): local('yui-compressor --charset utf-8 -o %s %s' % (respath, respath)) for respath in get_files(pkg_join('plugins'), '*.js', recursive=True): local('yui-compressor --charset utf-8 -o %s %s' % (respath, respath)) # Aggregate js+css resources into bundle for template_path in get_files(pkg_join('themes'), 'resources.html', recursive=True): logger.info('Template path: %s' % template_path) bundle(template_path, pkg_join('themes/default/htdocs')) logger.info('Compression completed.') # Build eggs and source packages (in build dir) logger.info('Laying eggs and source packages...') with lcd(pkg_join()): for plugin_dir in PLUGIN_DIRS: with lcd(plugin_dir): local('python setup.py bdist_egg') local('python setup.py sdist') # Build external plugins as well, optionally even non-fork plugins # Retrieve and build external plugins and copy the artifacts into plugins folder. # NOTE: Next egg copying will put them into correct place, no need to rerun the file copy allext = 'true' if ext.lower() == 'all' else 'false' if get_bool_str(ext) or allext: buildext(allext=allext,branch=extbranch) for egg in get_files(build_join('ext'), '*.egg', recursive=True): shutil.copy(egg, pkg_join('plugins', os.path.basename(egg))) # Copy eggs and sdisted files from plugins directory to dist and plugin directories for egg in get_files(pkg_join('plugins/multiproject'), '*.egg', recursive=True): shutil.copy(egg, dist_join(os.path.basename(egg))) shutil.copy(egg, pkg_join('plugins')) for targz in get_files(pkg_join('plugins'), '*.tar.gz', recursive=True): shutil.copy(targz, dist_join(os.path.basename(targz))) # Create dist if not available if not os.path.exists(DIST_DIR): os.makedirs(DIST_DIR) # Create one big package to contain 'em all if 'tar.gz' in pkg_formats or 'tar' in pkg_formats: logger.info('Creating complete .tar.gz package...') # TODO: Archive could be implemented in pure python # TODO: These patterns seem to assume build dir == project dir exclude_patterns = [ '.*', 'tests', 'documents', '*.egg-info', 'ext/libs', 'ext/plugins', 'sample', 'build', 'plugins/multiproject' ] exclude_param = ' '.join(['--exclude=%s' % pt for pt in exclude_patterns]) with lcd(BUILD_DIR): #local('tar -czf %s.tar.gz --exclude-vcs %s %s' % # (dist_join(package_name), exclude_param, package_name)) local('tar -czf %s.tar.gz %s' % (dist_join(package_name), package_name)) # Debian package if 'deb' in pkg_formats: logger.info('Creating .deb package...') try: from stdeb import command except ImportError: command = None abort('Module stddep (http://pypi.python.org/pypi/stdeb) was not found, cannot build .deb package') # Run setup.py bdist_deb inside each plugin. It generates deb_dist/<pkgname>/ directory for setuppy_path in get_files(os.path.abspath(pkg_join('plugins')), 'setup.py', recursive=True): plugin_dir = os.path.dirname(setuppy_path) with settings(hide('stdout', 'stderr')): with lcd(plugin_dir): local('python setup.py --command-packages=stdeb.command bdist_deb') # Package command needs to be run inside the generated folder. Find it and run the command for debdist_path in get_files(os.path.join(plugin_dir, 'deb_dist'), 'setup.py', recursive=True): with lcd(os.path.dirname(debdist_path)): local('dpkg-buildpackage -rfakeroot -uc -us') # Copy .deb packages to dist for deb_path in get_files(pkg_join('plugins'), '*.deb', recursive=True): shutil.copy(deb_path, dist_join(os.path.basename(deb_path))) # Redhat package if 'rpm' in pkg_formats: logger.info('Creating .rpm package...') with settings(hide('stdout', 'running')): # Run setup.py bdist_rpm inside each plugin. It generates deb_dist/<pkgname>/ directory for setuppy_path in get_files(os.path.abspath(pkg_join('plugins')), 'setup.py', recursive=True): plugin_dir = os.path.dirname(setuppy_path) with lcd(plugin_dir): local('python setup.py bdist_rpm') # Copy .rpm packages to dist for rpm_path in get_files(pkg_join('plugins'), '*.rpm', recursive=True): shutil.copy(rpm_path, dist_join(os.path.basename(rpm_path))) logger.info('Building completed.')
def build(release='false', compress='false', docs='', pkgs='tar', version='', ext='true', extbranch='master'): """ Create distributable packages. Builds eggs and tar.gz compressed packages, based on parameters. Also capable of downloading and patching external dependencies. :param release: Make release build or not. Release sets/increments the version number. Default 'false' :param compress: Compress js/css files nor not. Default 'false' :param docs: Names of the documentation targets to build. Default '' means no doc building :param pkgs: Package formats to build sources into, separated with space. Valid values: tar deb rpm :param version: Version number to set for whole package. Default '' -> take the version from VERSION.txt (or default to 1.0.0) :param ext: Build and include external modules into big package. Default is 'false'. If ext is 'all', builds also other than own forks (GitResources). :param extbranch: Defines from which branch the fork packages are to be built from. Examples:: fab dist.build fab dist.build:release=true,docs=html fab dist.build:compress=true,version=1.2.3,pkgs="deb tar rpm" .. NOTE:: Python modules get their version number from setup.py """ # NOTE: Fabric parameters are always in string format # Get the list of package formats (space delimeter) pkg_formats = pkgs.split(' ') # Determine the version: parameter vs. VERSION.txt vs. default if not version: version_path = os.path.join(PROJECT_DIR, 'VERSION.txt') if os.path.exists(version_path): version = set_version_in_file(version_path, version) else: version = '1.0.0' # Create package name from pkg name and version package_name = '%s-%s' % (PKG_NAME, version) pkg_join = lambda *path: join(BUILD_DIR, package_name, *path) logger.info('Preparing build env...') # Copy relevant files to build dir (so that they can be edited directly) shutil.rmtree(BUILD_DIR, ignore_errors=True) del SRC_DIRS[SRC_DIRS.index('libs')] del SRC_DIRS[SRC_DIRS.index('etc')] for src_dir in SRC_DIRS: shutil.copytree(src_dir, pkg_join(src_dir)) if not os.path.exists(DIST_DIR): os.makedirs(DIST_DIR) # Copy additional files shutil.copy('README.rst', pkg_join('README')) os.makedirs(pkg_join('scripts')) shutil.copy(rel_join('scripts/deploy.sh'), pkg_join('scripts/deploy.sh')) shutil.copy(rel_join('scripts/update.py'), pkg_join('scripts/update.py')) shutil.copytree(rel_join('scripts/hooks'), pkg_join('scripts/hooks')) shutil.copytree(rel_join('scripts/cron'), pkg_join('scripts/cron')) # Build documentation if docs: # List the target formats/builders builddoc(docs, pkg_join('docs')) else: # Ensure there is at least empty directory (for archive) os.makedirs(pkg_join('docs')) # Build configuration buildetc(outdir=pkg_join('etc'), section='DEFAULT') # Increment version of each plugin if making a release if get_bool_str(release): logger.info('Setting/incrementing version numbers...') for setuppy_path in get_files(pkg_join('plugins'), 'setup.py', recursive=True): # Check if plugin folder contains VERSION.txt (non-versioned file) version_path = os.path.join(os.path.dirname(setuppy_path), 'VERSION.txt') if not os.path.exists(version_path): logger.warning( 'VERSION.txt missing, using version found in setup.py') version_path = setuppy_path # Set version information in file. # NOTE: If version is empty, it is determined from version file (either VERSION.txt or setup.py) set_version_in_file(version_path, version) # Optional compress (edits copied files under build) if get_bool_str(compress): logger.info('Compressing files...') with settings(warn_only=True): # Compress theme resources for respath in get_files(pkg_join('themes'), '*.css', recursive=True): local('yui-compressor --charset utf-8 -o %s %s' % (respath, respath)) for respath in get_files(pkg_join('themes'), '*.js', recursive=True): local('yui-compressor --charset utf-8 -o %s %s' % (respath, respath)) # Compress plugin resources for respath in get_files(pkg_join('plugins'), '*.css', recursive=True): local('yui-compressor --charset utf-8 -o %s %s' % (respath, respath)) for respath in get_files(pkg_join('plugins'), '*.js', recursive=True): local('yui-compressor --charset utf-8 -o %s %s' % (respath, respath)) # Aggregate js+css resources into bundle for template_path in get_files(pkg_join('themes'), 'resources.html', recursive=True): logger.info('Template path: %s' % template_path) bundle(template_path, pkg_join('themes/default/htdocs')) logger.info('Compression completed.') # Build eggs and source packages (in build dir) logger.info('Laying eggs and source packages...') with lcd(pkg_join()): for plugin_dir in PLUGIN_DIRS: with lcd(plugin_dir): local('python setup.py bdist_egg') local('python setup.py sdist') # Build external plugins as well, optionally even non-fork plugins # Retrieve and build external plugins and copy the artifacts into plugins folder. # NOTE: Next egg copying will put them into correct place, no need to rerun the file copy allext = 'true' if ext.lower() == 'all' else 'false' if get_bool_str(ext) or allext: buildext(allext=allext, branch=extbranch) for egg in get_files(build_join('ext'), '*.egg', recursive=True): shutil.copy(egg, pkg_join('plugins', os.path.basename(egg))) # Copy eggs and sdisted files from plugins directory to dist and plugin directories for egg in get_files(pkg_join('plugins/multiproject'), '*.egg', recursive=True): shutil.copy(egg, dist_join(os.path.basename(egg))) shutil.copy(egg, pkg_join('plugins')) for targz in get_files(pkg_join('plugins'), '*.tar.gz', recursive=True): shutil.copy(targz, dist_join(os.path.basename(targz))) # Create dist if not available if not os.path.exists(DIST_DIR): os.makedirs(DIST_DIR) # Create one big package to contain 'em all if 'tar.gz' in pkg_formats or 'tar' in pkg_formats: logger.info('Creating complete .tar.gz package...') # TODO: Archive could be implemented in pure python # TODO: These patterns seem to assume build dir == project dir exclude_patterns = [ '.*', 'tests', 'documents', '*.egg-info', 'ext/libs', 'ext/plugins', 'sample', 'build', 'plugins/multiproject' ] exclude_param = ' '.join( ['--exclude=%s' % pt for pt in exclude_patterns]) with lcd(BUILD_DIR): #local('tar -czf %s.tar.gz --exclude-vcs %s %s' % # (dist_join(package_name), exclude_param, package_name)) local('tar -czf %s.tar.gz %s' % (dist_join(package_name), package_name)) # Debian package if 'deb' in pkg_formats: logger.info('Creating .deb package...') try: from stdeb import command except ImportError: command = None abort( 'Module stddep (http://pypi.python.org/pypi/stdeb) was not found, cannot build .deb package' ) # Run setup.py bdist_deb inside each plugin. It generates deb_dist/<pkgname>/ directory for setuppy_path in get_files(os.path.abspath(pkg_join('plugins')), 'setup.py', recursive=True): plugin_dir = os.path.dirname(setuppy_path) with settings(hide('stdout', 'stderr')): with lcd(plugin_dir): local( 'python setup.py --command-packages=stdeb.command bdist_deb' ) # Package command needs to be run inside the generated folder. Find it and run the command for debdist_path in get_files(os.path.join( plugin_dir, 'deb_dist'), 'setup.py', recursive=True): with lcd(os.path.dirname(debdist_path)): local('dpkg-buildpackage -rfakeroot -uc -us') # Copy .deb packages to dist for deb_path in get_files(pkg_join('plugins'), '*.deb', recursive=True): shutil.copy(deb_path, dist_join(os.path.basename(deb_path))) # Redhat package if 'rpm' in pkg_formats: logger.info('Creating .rpm package...') with settings(hide('stdout', 'running')): # Run setup.py bdist_rpm inside each plugin. It generates deb_dist/<pkgname>/ directory for setuppy_path in get_files(os.path.abspath(pkg_join('plugins')), 'setup.py', recursive=True): plugin_dir = os.path.dirname(setuppy_path) with lcd(plugin_dir): local('python setup.py bdist_rpm') # Copy .rpm packages to dist for rpm_path in get_files(pkg_join('plugins'), '*.rpm', recursive=True): shutil.copy(rpm_path, dist_join(os.path.basename(rpm_path))) logger.info('Building completed.')
""" try: from nose.core import TestProgram from nose.plugins import Plugin except ImportError: TestProgram = None Plugin = object return abort( 'For running tests, Nose testing framework is required. Please install it first: "pip install nose"' ) if not case: return abort('Please provide either name or path to test case') # Determine the case file: name or path accepted webtests_dir = rel_join('tests/webtests') casepath = os.path.abspath(case) if case.endswith('.py') else join( webtests_dir, 'cases/%s.py' % case) configpath = os.path.join(os.curdir, os.path.expanduser(config)) logger.info('Running functional tests from: %s' % casepath) logger.info('Reading tests configuration from: %s' % configpath) class TestConfigPlugin(Plugin): """ Simple Nose plugin to set test configuration path to testcase:: class MyTestcase(unittest.TestCase) def setUp(self): self.config_path