def test(app=''):
"""
Tests one app or all apps
"""
print('Testing...')
run('./manage.py test "{0}"'.format(app))
print('... done testing')
def play(playbook, user, inventory=SITE_INVENTORY, sudo=True, ask_sudo_pass=True,
verbose=False, extra='', key=None, limit=None, tags=None):
"""Run a playbook. Defaults to using the "hosts" inventory"""
print('[invoke] Playing {0!r} on {1!r} with user {2!r}...'.format(
playbook, inventory, user)
)
# If private key not provided, take a good guess
if not key:
if user == 'vagrant':
key = '~/.vagrant.d/insecure_private_key'
else:
key = '~/.ssh/id_rsa'
cmd = 'ansible-playbook {playbook} -i {inventory} -u {user}'.format(**locals())
if sudo:
cmd += ' -s'
if ask_sudo_pass:
cmd += ' --ask-sudo-pass'
if verbose:
cmd += ' -vvvv'
if limit:
cmd += ' --limit={0}'.format(limit)
if key:
cmd += ' --private-key={0}'.format(key)
if extra:
cmd += ' -e {0!r}'.format(extra)
if tags:
cmd += ' --tags={0!r}'.format(tags)
run(cmd, echo=True, pty=True)
def base_case(self):
# Basic "doesn't explode" test: respond.py will exit nonzero unless
# this works, causing a Failure.
watcher = Responder(r"What's the password\?", "Rosebud\n")
# Gotta give -u or Python will line-buffer its stdout, so we'll
# never actually see the prompt.
run("python -u respond_base.py", watchers=[watcher], hide=True)
def test(coverage=False, browse=False):
command = "nosetests"
if coverage:
command += " --with-coverage --cover-html"
run(command, pty=True)
if coverage and browse:
run("open cover/index.html")
def rkhunter_propupd(group='vagrantbox', inventory='vagranthosts', user='******'):
"""Update rkhunter's baseline file configuration database."""
cmd = ('ansible {group} -i {inventory} -a '
'"rkhunter --propupd" --sudo --ask-sudo-pass').format(
group=group, inventory=inventory
)
run(cmd, echo=True)
def benchmark(database):
""" Executes benchmarks with the default settings for a given DB
Usage: `invoke benchmark <database> """
database = check_module_naming(database)
run("python main.py {db}".format(db=database))
def deploy(database):
""" Runs the ansible playbook for a given db """
database = check_module_naming(database)
run('cd {db}/ansible && ansible-playbook -u vagrant -i hosts -s'
' site.yml -vv'.format(db=database))
def compile_python_files(jython_jar, build_dir='build'):
run('java -jar {} -m compileall {}'.format(jython_jar, build_dir))
# Jython will not work without its py-files, but robot will
for directory, _, files in os.walk(os.path.join(build_dir, 'Lib', 'robot')):
for name in files:
if name.endswith('.py'):
os.remove(os.path.join(directory, name))
def help():
"""Show help, basically an alias for --help.
This task can be removed once the fix to this issue is released:
https://github.com/pyinvoke/invoke/issues/180
"""
run('invoke --help')
def clean_assets():
"""Remove built JS files."""
public_path = os.path.join(HERE, 'website', 'static', 'public')
js_path = os.path.join(public_path, 'js')
mfr_path = os.path.join(public_path, 'mfr')
run('rm -rf {0}'.format(js_path), echo=True)
run('rm -rf {0}'.format(mfr_path), echo=True)
def compile_java_files(jython_jar, build_dir='build'):
root = os.path.join('src', 'java', 'org', 'robotframework')
files = [os.path.join(root, name) for name in os.listdir(root)
if name.endswith('.java')]
print 'Compiling {} Java files.'.format(len(files))
run('javac -d {target} -target 1.5 -source 1.5 -cp {cp} {files}'.format(
target=build_dir, cp=jython_jar, files=' '.join(files)))
def mongorestore(path, drop=False):
"""Restores the running OSF database with the contents of the database at
the location given its argument.
By default, the contents of the specified database are added to
the existing database. The `--drop` option will cause the existing database
to be dropped.
A caveat: if you `invoke mongodump {path}`, you must restore with
`invoke mongorestore {path}/{settings.DB_NAME}, as that's where the
database dump will be stored.
"""
db = settings.DB_NAME
port = settings.DB_PORT
cmd = "mongorestore --db {db} --port {port}".format(
db=db,
port=port,
pty=True)
if settings.DB_USER:
cmd += ' --username {0}'.format(settings.DB_USER)
if settings.DB_PASS:
cmd += ' --password {0}'.format(settings.DB_PASS)
if drop:
cmd += " --drop"
cmd += " " + path
run(cmd, echo=True)
def rabbitmq():
"""Start a local rabbitmq server.
NOTE: this is for development only. The production environment should start
the server as a daemon.
"""
run("rabbitmq-server", pty=True)
def release(yes=False):
"""
Builds package and uploads to Anaconda.org
We only allow releases from the production branch
"""
_assert_release_ok()
version = pkg_conf.get_version()
_print("You are about to build and upload version {}, build {} of package {} to user {}".format(
version, pkg_conf.get_build_number(), pkg_conf.PKG_NAME, pkg_conf.ANACONDA_USER))
if yes or _confirm(prompt="Do you want to continue?"):
cmd = "conda build conda-recipe --output"
pkg_path = run(cmd, pty=on_linux, hide='stdout').stdout.strip()
_print(pkg_path)
cmd = "deactivate && conda build conda-recipe --no-anaconda-upload --quiet"
if on_linux:
cmd = "source " + cmd
run(cmd, pty=on_linux)
if os.path.exists(pkg_path):
try:
run("anaconda upload {} --user {}".format(pkg_path, pkg_conf.ANACONDA_USER))
_tag_revision("v{}".format(version))
return True
except:
traceback.print_exc()
_print("Upload failed.")
_print("Release failed.")
return False
def test(slow=False, azure_storage_emulator=False):
"""
Runs the test suite
"""
_print("Testing...")
pkg_name = pkg_conf.PKG_ROOT
options = [
"--pyargs {}".format(pkg_name), # which package we want to test
"--durations=3", # show n slowest tests
]
test_flags = [
"--azure-storage-emulator" if azure_storage_emulator else "", # enable tests depending on azure storage emu
"--slow" if slow else "", # enable slow tests
]
coverage_flags = [
"--cov={}".format(pkg_name), # enable coverage for this package
"--cov-config .coveragerc", # select the coverage configuratoin file
"--cov-report term-missing", # display missing lines in the report
]
xdist_flags = [
"-n{}".format(max(min(multiprocessing.cpu_count(), 4), 1)), # select amount of CPUs to use
]
can_parallelize = not azure_storage_emulator # azure storage emulator cant handle concurrent connections
# compose arguments
args = options + test_flags + coverage_flags
if can_parallelize:
args += xdist_flags
run("py.test {}".format(" ".join(args)))
_print("Testing passed!")
def _compile_less(source, target, sourcemap, minify=True, verbose=False):
"""Compile a less file by source and target relative to static_dir"""
min_flag = '-x' if minify else ''
ver_flag = '--verbose' if verbose else ''
# pin less to version number from above
try:
out = check_output(['lessc', '--version'])
except OSError as err:
raise ValueError("Unable to find lessc. Please install lessc >= %s and < %s "
% (min_less_version, max_less_version))
out = out.decode('utf8', 'replace')
less_version = out.split()[1]
if min_less_version and V(less_version) < V(min_less_version):
raise ValueError("lessc too old: %s < %s. Use `$ npm install [email protected]` to install a specific version of less" % (
less_version, min_less_version))
if max_less_version and V(less_version) >= V(max_less_version):
raise ValueError("lessc too new: %s >= %s. Use `$ npm install [email protected]` to install a specific version of less" % (
less_version, max_less_version))
static_path = pjoin(here, static_dir)
cwd = os.getcwd()
try:
os.chdir(static_dir)
run('lessc {min_flag} {ver_flag} --source-map={sourcemap} --source-map-basepath={static_path} --source-map-rootpath="../" {source} {target}'.format(**locals()),
echo=True,
)
finally:
os.chdir(cwd)
def _browse(url):
if on_win32:
run('explorer "{}"'.format(url), hide='stdout')
elif on_linux:
run('xdg-open "{}"'.format(url), hide='stdout')
else:
raise NotImplementedError("Browsing is not supported for this OS")
def run_translate_tests(env):
invoke.run(
"PYTHONPATH={rpython_path}:$PYTHONPATH python {rpython_path}/rpython/bin/rpython --batch targettopaz.py".format(
**env
)
)
run_specs("`pwd`/bin/topaz")
def upload_build(self):
if os.environ["TRAVIS_BRANCH"] == "master" and "BUILD_SECRET" in os.environ:
width = struct.calcsize("P") * 8
if "linux" in sys.platform:
platform = "linux{}".format(width)
elif "darwin" in sys.platform:
platform = "osx{}".format(width)
elif "win" in sys.platform:
platform = "windows{}".format(width)
else:
raise ValueError("Don't recognize platform: {!r}".format(sys.platform))
build_name = "topaz-{platform}-{sha1}.tar.bz2".format(platform=platform, sha1=os.environ["TRAVIS_COMMIT"])
invoke.run("python topaz/tools/make_release.py {}".format(build_name))
with open(build_name, "rb") as f:
response = requests.post(
"http://www.topazruby.com/builds/create/",
{
"build_secret": os.environ["BUILD_SECRET"],
"sha1": os.environ["TRAVIS_COMMIT"],
"platform": platform,
"success": "true",
},
files={"build": (build_name, f)},
)
response.raise_for_status()
def rebuild_search():
"""Delete and recreate the index for elasticsearch"""
run("curl -s -XDELETE {uri}/{index}*".format(uri=settings.ELASTIC_URI,
index=settings.ELASTIC_INDEX))
run("curl -s -XPUT {uri}/{index}".format(uri=settings.ELASTIC_URI,
index=settings.ELASTIC_INDEX))
migrate_search()
def wheelhouse(addons=False, release=False, dev=False, metrics=False):
"""Install python dependencies.
Examples:
inv wheelhouse --dev
inv wheelhouse --addons
inv wheelhouse --release
inv wheelhouse --metrics
"""
if release or addons:
for directory in os.listdir(settings.ADDON_PATH):
path = os.path.join(settings.ADDON_PATH, directory)
if os.path.isdir(path):
req_file = os.path.join(path, 'requirements.txt')
if os.path.exists(req_file):
cmd = 'pip wheel --find-links={} -r {} --wheel-dir={}'.format(WHEELHOUSE_PATH, req_file, WHEELHOUSE_PATH)
run(cmd, pty=True)
if release:
req_file = os.path.join(HERE, 'requirements', 'release.txt')
elif dev:
req_file = os.path.join(HERE, 'requirements', 'dev.txt')
elif metrics:
req_file = os.path.join(HERE, 'requirements', 'metrics.txt')
else:
req_file = os.path.join(HERE, 'requirements.txt')
cmd = 'pip wheel --find-links={} -r {} --wheel-dir={}'.format(WHEELHOUSE_PATH, req_file, WHEELHOUSE_PATH)
run(cmd, pty=True)
def install_roles(force=False, ignore_errors=False):
command = 'ansible-galaxy install -r roles.txt -p roles'
if force:
command += ' --force'
if ignore_errors:
command += ' --ignore-errors'
run(command, pty=True)
def test(ctx):
"""Lint, unit test, and check setup.py"""
cmd = "{} {}".format(
"nosetests",
"--with-coverage --cover-erase --cover-package=arghelper --cover-html")
run(cmd)
run("python setup.py check")
def mfr_requirements(download_cache=None):
"""Install modular file renderer requirements"""
print('Installing mfr requirements')
cmd = 'pip install --upgrade -r mfr/requirements.txt'
if download_cache:
cmd += ' --download-cache {0}'.format(download_cache)
run(bin_prefix(cmd), echo=True)
def encryption(owner=None):
"""Generate GnuPG key.
For local development:
> invoke encryption
On Linode:
> sudo env/bin/invoke encryption --owner www-data
"""
if not settings.USE_GNUPG:
print("GnuPG is not enabled. No GnuPG key will be generated.")
return
import gnupg
gpg = gnupg.GPG(gnupghome=settings.GNUPG_HOME, gpgbinary=settings.GNUPG_BINARY)
keys = gpg.list_keys()
if keys:
print("Existing GnuPG key found")
return
print("Generating GnuPG key")
input_data = gpg.gen_key_input(name_real="OSF Generated Key")
gpg.gen_key(input_data)
if owner:
run("sudo chown -R {0} {1}".format(owner, settings.GNUPG_HOME))
def release_test(database=None, verbose=False, **kwargs):
out = functools.partial(_out, "release.test")
hide = "out" if not verbose else None
if database:
os.environ["WAREHOUSE_DATABASE_URL"] = database
out("Creating a temporary directory to export Warehouse to")
curdir = os.getcwd()
tmpdir = tempfile.mkdtemp()
tmpdir = tmpdir if tmpdir.endswith("/") else tmpdir + "/"
try:
invoke.run("git checkout-index -f -a --prefix={}".format(tmpdir),
hide=hide,
)
os.chdir(tmpdir)
# Run all our various tests one last time before
envs = set(invoke.run("tox -l", hide=hide).stdout.split())
envs -= {"packaging"} # Packaging tests require a git clone
out("Running tests: {}".format(", ".join(envs)))
for env in envs:
out("Running the {} tests".format(env))
invoke.run("tox -e {}".format(env), hide=hide)
finally:
os.chdir(curdir)
shutil.rmtree(tmpdir, ignore_errors=True)
def publish():
"""
Publish to the cheeseshop
"""
run('python setup.py sdist upload', pty=True)
run('python setup.py bdist_wheel upload', pty=True)
log.info('published new release')
def deps_compile(ctx):
"""Compile new requirements-*.txt"""
env = create_env("tools", requirements=True)
for file in glob.glob(os.path.join(BASE, "requirements-*.in")):
invoke.run("%s/bin/pip-compile %s > %s" % (env, os.path.abspath(file),
os.path.abspath(file)[:-3] + ".txt"))
def mongo(daemon=False, port=20771):
"""Run the mongod process.
"""
cmd = "mongod --port {0}".format(port)
if daemon:
cmd += " --fork"
run(cmd)
def celery_beat(level="debug", schedule=None):
"""Run the Celery process."""
# beat sets up a cron like scheduler, refer to website/settings
cmd = 'celery beat -A framework.tasks -l {0}'.format(level)
if schedule:
cmd = cmd + ' --schedule={}'.format(schedule)
run(bin_prefix(cmd), pty=True)
def bandit():
run('find . -name \'*.py\' | xargs bandit')
def safety():
run('safety check')
def flake8():
run('flake8 .')
def pyflakes():
run('pyflakes .')
def pylint():
run('pylint *.py')
def ablog_deploy(website, message=None, github_pages=None,
push_quietly=False, push_force=False, github_token=None, repodir=None,
**kwargs):
confdir = find_confdir()
conf = read_conf(confdir)
github_pages = (github_pages or getattr(conf, 'github_pages', None))
website = (website or
os.path.join(confdir, getattr(conf, 'ablog_builddir', '_website')))
tomove = glob.glob(os.path.join(website, '*'))
if not tomove:
print('Nothing to deploy, build first.')
return
try:
from invoke import run
except ImportError:
raise ImportError("invoke is required by deploy command, "
"run `pip install invoke`")
if github_pages:
if repodir is None:
repodir = os.path.join(confdir, "{0}.github.io".format(github_pages))
if os.path.isdir(repodir):
os.chdir(repodir)
run("git pull", echo=True)
else:
run("git clone https://github.com/{0}/{0}.github.io.git {1}"
.format(github_pages, repodir), echo=True)
git_add = []
for tm in tomove:
for root, dirnames, filenames in os.walk(website):
for filename in filenames:
fn = os.path.join(root, filename)
fnnew = fn.replace(website, repodir)
try:
os.renames(fn, fnnew)
except OSError:
if os.path.isdir(fnnew):
shutil.rmtree(fnnew)
else:
os.remove(fnnew)
os.renames(fn, fnnew)
git_add.append(fnnew)
print('Moved {} files to {}.github.io'
.format(len(git_add), github_pages))
os.chdir(repodir)
run("git add -f " + " ".join(['"{}"'.format(os.path.relpath(p))
for p in git_add]), echo=True)
if not os.path.isfile('.nojekyll'):
open('.nojekyll', 'w')
run("git add -f .nojekyll")
commit = 'git commit -m "{}"'.format(message or 'Updates.')
if push_force:
commit += ' --amend'
run(commit, echo=True)
if github_token:
with open(os.path.join(repodir, '.git/credentials'), 'w') as out:
out.write('https://{}:@github.com'
.format(os.environ[github_token]))
run('git config credential.helper "store --file=.git/credentials"')
push = 'git push'
if push_quietly:
push += ' -q'
if push_force:
push += ' -f'
push += ' origin master'
run(push, echo=True)
else:
print('No place to deploy.')
def test():
run('python modulino.py')
run('python test.py')
def postrelease(ctx, version):
'''
Finalise the release
Running this task will:
- commit the version changes to source control
- tag the commit
- push changes to master
'''
run('git add gmaps/_version.py')
run('git add js/package.json')
run('git commit -m "Bump version to {}"'.format(version))
run('git tag -a v{} -F changelog.tmp'.format(version))
run('git push origin master --tags')
new_version = semver.bump_patch(version) + '-dev'
set_pyversion(new_version)
set_jsversion(new_version)
run('git add gmaps/_version.py')
run('git add js/package.json')
run('git commit -m "Back to dev"')
run('git push origin master')
def release_conda(ctx, version):
"""
Open a PR for the new release in conda-forge.
"""
sha256 = get_file_sha256('dist/gmaps-{}.tar.gz'.format(version))
print('Release SHA256 hash: {}'.format(sha256))
tempdir = tempfile.mkdtemp()
try:
print('Cloning gmaps-feedstock to {}'.format(tempdir))
os.chdir(tempdir)
run('git clone [email protected]:pbugnion/gmaps-feedstock.git')
os.chdir('gmaps-feedstock')
run('git remote add upstream [email protected]:conda-forge/gmaps-feedstock.git')
run('git pull --rebase upstream master')
branch_name = 'release-version-{}'.format(version)
run('git checkout -b {}'.format(branch_name))
update_conda_recipe(version, sha256)
run('git add recipe/meta.yaml')
run('git commit -m "Release version {}"'.format(version))
run('git push origin {}'.format(branch_name))
print('URL: https://github.com/conda-forge/gmaps-feedstock')
finally:
print('Deleting temporary directory {}'.format(tempdir))
shutil.rmtree(tempdir)
def pep8():
run('pep8 .')
def main():
args = sys.argv[1:]
if len(args):
run('invoke ' + ' '.join(args), pty=True)
else:
run('invoke --list')
def release_python_sdist():
run('rm -f dist/*')
run('python setup.py sdist')
run('twine upload dist/*')
def readme(ctx, browse=False):
run("rst2html.py README.rst > README.html")
if browse:
webbrowser.open_new_tab('README.html')
def destroy_ssh_keypair(ec2_client, key_filename):
key_name = os.path.basename(key_filename).split(".pem")[0]
response = ec2_client.delete_key_pair(KeyName=key_name)
run(f"rm -f {key_filename}")
return response, key_name
def clean(ctx):
run("rm -rf build")
run("rm -rf dist")
run("rm -rf marshmallow.egg-info")
clean_docs(ctx)
print("Cleaned up.")
def run_kubectl(ctx, cmd):
kubectl_exe = ctx.get('kubectl_exe', 'kubectl')
run('{kubectl_exe} {cmd}'.format(
kubectl_exe=kubectl_exe,
cmd=cmd,
))
def getdata(c):
"""
Get remote data from the production sever
"""
with Connection(prod_server, user=prod_user) as r:
with r.cd(remote_prod_dir):
with Connection(prod_server, user=prod_user) as r:
with r.prefix("source {0}/bin/activate".format(
remote_prod_virtualenv)):
with r.cd(remote_prod_dir):
command = dotenv.get_cli_string(
'.env',
'get',
'STATIC_ROOT',
)
ret = r.run(command)
remote_static_root = ret.stdout.split('=')[1].strip()
command = dotenv.get_cli_string(
'.env',
'get',
'MEDIA_ROOT',
)
ret = r.run(command)
remote_media_root = ret.stdout.split('=')[1].strip()
command = dotenv.get_cli_string(
'.env',
'get',
'DATABASE',
)
ret = r.run(command)
remote_database = ret.stdout.split('=')[1].strip()
command = dotenv.get_cli_string(
'.env',
'get',
'USERNAME',
)
ret = r.run(command)
remote_database_username = ret.stdout.split(
'=')[1].strip()
logger.info("Backing up the database")
r.run("pg_dump -U {} {} > {}/data/dump.sql".format(
remote_database, remote_database_username, remote_prod_dir))
logger.info("Getting remote data dump file")
run("rsync -vzh --info=progress2 {}@{}:{}/data/dump.sql data/dump.sql"
.format(
prod_user,
prod_server,
remote_prod_dir,
))
#r.get(remote_prod_dir + '/data/dump.sql', local=os.getcwd() + "/data/dump.sql")
logger.info("Recreating local database")
run("dropdb {}".format(database_local))
run("createdb {}".format(database_local))
run("psql -U {} {} < data/dump.sql".format(database_local_user,
database_local),
warn=True)
logger.info("Syncing static and media files")
run("rsync -avzh --info=progress2 --delete {}@{}:{}/ {}/".format(
prod_user, prod_server, remote_static_root, static_root))
run("rsync -avzh --info=progress2 --delete --exclude='applications/*' {}@{}:{}/ {}/"
.format(prod_user, prod_server, remote_media_root, media_root))
def clean_docs(ctx):
run("rm -rf %s" % build_dir)
def publish():
"""publish - package and upload a release to the cheeseshop."""
run('python setup.py sdist upload', pty=True)
run('python setup.py bdist_wheel upload', pty=True)
log.info('published new release')
def flake(ctx):
"""Run flake8 on codebase."""
run('flake8 .', echo=True)
def test():
"""test - run the test runner."""
run('py.test --flakes --cov-report html --cov lintful tests/', pty=True)
run('open htmlcov/index.html')
def clean():
"""clean - remove build artifacts."""
run('rm -rf build/')
run('rm -rf dist/')
run('rm -rf lintful.egg-info')
run('find . -name __pycache__ -delete')
run('find . -name *.pyc -delete')
run('find . -name *.pyo -delete')
run('find . -name *~ -delete')
log.info('cleaned up')
def test(docs=False):
run("nosetests --processes=-1")
def lint():
"""lint - check style with flake8."""
run('flake8 lintful tests')
def develop():
run("python setup.py develop")
def lrun(command, *args, **kwargs):
run('cd {0} && {1}'.format(ROOT, command), *args, **kwargs)
def lint():
for src in os.listdir("streamparse"):
if src.endswith(".py"):
run("pyflakes streamparse/{}".format(src))
def upload():
run("python setup.py sdist upload")
def build():
run('cookiecutter {0} --no-input'.format(cwd))
def build(docs=False):
run("python setup.py build")
if docs:
run("sphinx-build doc/source doc/_build")
