from invoke import Collection from . import openmpi import logging from tasks.kernels import ns as kernels_ns from tasks.lammps import ns as lammps_ns from tasks.makespan import ns as makespan_ns from tasks.migration import ns as migration_ns logging.getLogger().setLevel(logging.DEBUG) ns = Collection(openmpi, ) ns.add_collection(lammps_ns, name="lammps") ns.add_collection(kernels_ns, name="kernels") ns.add_collection(makespan_ns, name="makespan") ns.add_collection(migration_ns, name="migration")
@bashCommand: is list with the command and the options
return: list with the output and the posible error
"""
process = subprocess.Popen(command, stdout=subprocess.PIPE)
output, err = process.communicate()
if err and fail_msg:
print fail_msg
print err
return False
if not err and success_msg:
print success_msg
if not quiet:
print output
return True
def remove_dir(path, quiet=False):
if not quiet:
if not _ask_ok('Answer "yes" to remove path: "%s". [y/N] ' %
(path), 'n'):
return
shutil.rmtree(path)
UtilsCollection = Collection()
UtilsCollection.add_task(account_reconcile)
UtilsCollection.add_task(update_parent_left_right)
UtilsCollection.add_task(prepare_translations)
UtilsCollection.add_task(export_translations)
if isinstance(handler, logging.StreamHandler):
break
else:
handler = logging.StreamHandler()
logger.addHandler(handler)
handler.setFormatter(formatter)
from invoke import Collection
from invoke.executor import Executor
from . import app
# NOTE: `namespace` or `ns` name is required!
namespace = Collection(
app,
)
def invoke_execute(context, command_name, **kwargs):
"""
Helper function to make invoke-tasks execution easier.
"""
results = Executor(namespace, config=context.config).execute((command_name, kwargs))
target_task = context.root_namespace[command_name]
return results[target_task]
namespace.configure({
'run': {
'shell': '/bin/sh' if platform.system() != 'Windows' else os.environ.get('COMSPEC'),
},
'root_namespace': namespace,
from invoke import Collection
from . import cv
from . import deploy
from . import docker
from . import jekyll
ns = Collection(
cv,
deploy,
docker,
jekyll,
)
from fabric import task
from invoke import Collection
from ._common import project_path
@task()
def compose_up(ctx, daemon=False):
with ctx.cd(project_path("docker")):
command = "docker-compose --file docker-compose-local.yml --project-name={{ cookiecutter.project_slug }} up"
if daemon:
command = f"{command} -d"
ctx.run(command, pty=True, replace_env=False)
@task()
def compose_down(ctx, volumes=False):
with ctx.cd(project_path("docker")):
command = "docker-compose --file docker-compose-local.yml --project-name={{ cookiecutter.project_slug }} down"
if volumes:
command = f"{command} -v"
ctx.run(command, pty=True, replace_env=False)
compose_collection = Collection("compose")
compose_collection.add_task(compose_up, name="up")
compose_collection.add_task(compose_down, name="down")
"You need to modify this call before being able to use it")
from aml_compute import PyTorchExperimentCLI
exp = PyTorchExperimentCLI("<YOUR-EXPERIMENT-NAME>")
run = exp.submit_local(
os.path.join(_BASE_PATH, "src"),
"<YOUR-TRAINING-SCRIPT>",
{
"--training_data_path": "/data/train",
"--validation_data_path": "/data/validation",
"--epochs": "1",
"--data_type": "images",
"--data-format": "channels_first",
},
dependencies_file=os.path.join(_BASE_PATH, "environment_gpu.yml"),
docker_args=["-v", f"{env_values['data']}:/data"],
wait_for_completion=True,
)
print(run)
remote_collection = Collection("remote")
remote_collection.add_task(submit_images_remote, "images")
remote_collection.add_task(submit_remote, "synthetic")
local_collection = Collection("local")
local_collection.add_task(submit_images_local, "images")
local_collection.add_task(submit_local, "synthetic")
submit_collection = Collection("submit", local_collection, remote_collection)
namespace = Collection("pytorch_experiment", submit_collection)
#!/usr/bin/env python
import sys
import os
from invoke import task, Collection, UnexpectedExit, Failure
sys.path.append(
os.path.abspath(
os.path.join(os.path.dirname(__file__),
os.path.pardir + '/policy_sentry/')))
from policy_sentry.command import initialize
# Create the necessary collections (namespaces)
ns = Collection()
docs = Collection('docs')
ns.add_collection(docs)
test = Collection('test')
ns.add_collection(test)
integration = Collection('integration')
ns.add_collection(integration)
unit = Collection('unit')
ns.add_collection(unit)
build = Collection('build')
ns.add_collection(build)
@task
def make_html(c):
@task
def clean(ctx):
''' Cleans up the crap '''
print('Cleaning')
ctx.run("rm -rf htmlcov")
ctx.run("rm -rf build")
ctx.run("rm -rf dist")
@task(clean)
def deploy(ctx, repo=None):
''' Deploy to pypi '''
print('Deploying to Pypi!')
rstr = ''
if repo:
rstr = '-r {0}'.format(repo)
ctx.run("python setup.py sdist bdist_wheel --universal")
ctx.run("twine upload {0} dist/*".format(rstr))
os.chdir(os.path.dirname(__file__))
ns = Collection(clean, deploy)
docs = Collection('docs')
docs.add_task(build_docs, 'build')
docs.add_task(clean_docs, 'clean')
docs.add_task(show_docs, 'show')
ns.add_collection(docs)
""" This module exists to setup the pyinvoke namespace. """ from invoke import Collection, task import deployments import rollouts namespace = Collection(deployments, rollouts)
os.path.exists(demofolder)):
demofiles(ctx)
os.chdir('..')
copied_path = os.path.join(talk_name, os.path.basename(f))
shutil.copy(f, copied_path)
assert os.path.isfile(copied_path), \
'{} failed to copy into {}'.format(f, talk_name)
if 'folders' in talks[talk_name]:
for src, dst in talks[talk_name]['folders'].items():
dst = os.path.join(talk_name, dst)
if not os.path.exists(dst):
shutil.copytree(src, dst)
if 'rename' in talks[talk_name]:
for old_file, new_file in talks[talk_name]['rename'].items():
moved_file = os.path.join(talk_name, os.path.basename(old_file))
if os.path.isfile(moved_file):
os.rename(moved_file, os.path.join(talk_name, new_file))
elif os.path.isfile(old_file):
shutil.copy(old_file, os.path.join(talk_name, new_file))
# Configure cross-platform settings.
ns = Collection(environment, build, demofiles, r, clean, talk)
ns.configure({
'run': {
'shell': which('bash') if os.name != 'nt' else which('cmd'),
'pty': False if os.name == 'nt' else True
}
})
# -*- encoding: utf-8 -*-
from invoke import Collection, run, task
@task
def build():
print('Building!')
@task
def hi(name):
print('Hi %s!' % name)
@task
def service_manager():
service('services.manager')
@task
def service(name):
config_command = "--config ./project/nameko_config.yaml"
service_command = 'nameko run %s %s' % (config_command, name)
run("{}".format(service_command))
ns = Collection('service')
ns.add_task(service_manager, name='run_manager')
# === Update ===
@task(pre=[_setup], aliases=["univ"])
def update_niv(context):
"""
Update niv dependencies
"""
context.run("niv update")
@task(pre=[_setup], aliases=["unpm"])
def update_npm(context):
"""
Update npm packages
"""
context.run("npm run update")
update = Collection("update")
update.add_task(update_niv, "niv")
update.add_task(update_npm, "npm")
#
ns = Collection()
ns.add_task(init)
ns.add_collection(update)
from faasmcli.tasks import ns as faasm_ns
from invoke import Collection
from . import azure
from . import container
from . import covid
ns = Collection(
azure,
container,
covid,
)
ns.add_collection(faasm_ns, name="faasm")
#! /bin/env python3
from invoke import task, Collection
from policy_sentry.command import initialize as sentry
# Create the necessary collections (namespaces)
ns = Collection()
test = Collection('test')
ns.add_collection(test)
# FIXME Add an option so we don't trample the users db that they may have installed for prod usage.
# FIXME Document this in the wiki
# TODO Document this in a contributing.md doc
@task
def create_db(c):
sentry.initialize('')
# TODO Some kind of non-zero check to make sure that this passes.
@task
def security_scan(c):
c.run('bandit -r policy_sentry/')
c.run('safety check', warn=True)
# TODO Some kind of non-zero check to make sure that this passes.
@task
def run_linter(c):
c.run('pylint policy_sentry/', warn=True)
c.run('autopep8 -r --in-place policy_sentry/', warn=True)
# TODO If the database is not found we should build it, otherwise just run the tests.
from invoke import task, Collection
@task
def toplevel(ctx):
pass
@task
def subtask(ctx):
pass
ns = Collection(toplevel,
Collection('a', subtask, Collection('nother', subtask)))
# Helmfile deployment requires information on accessing the ECR
def ecr_helmfile_values_factory(*, context):
with tasks.terraform.ecr.ecr_read_only(context=context) as ecr_read_only:
return {
'registryUrl': ecr_read_only.output.registry_url,
'registryUser': ecr_read_only.output.registry_user,
'registryPassword': ecr_read_only.output.registry_password,
}
task_helmfile_migraine_apply = aws_infrastructure.tasks.library.instance_helmfile.task_helmfile_apply(
config_key=CONFIG_KEY,
ssh_config_path=SSH_CONFIG_PATH,
staging_local_dir=STAGING_LOCAL_HELMFILE_DIR,
staging_remote_dir=STAGING_REMOTE_HELMFILE_DIR,
helmfile_path=HELMFILE_PATH,
helmfile_config_path=HELMFILE_CONFIG_PATH,
helmfile_values_factories={
'ecr_generated': ecr_helmfile_values_factory,
'secrets_couchdb_dev_generated': couchdb_dev_helmfile_values_factory,
'secrets_couchdb_prod_generated': couchdb_prod_helmfile_values_factory,
'secrets_flask_prod_generated': flask_prod_helmfile_values_factory,
},
)
task_helmfile_migraine_apply.__doc__ = 'Apply helmfile/helmfile.yaml in the instance.'
ns = Collection('helmfile')
ns.add_task(task_helmfile_migraine_apply, 'apply')
from __future__ import absolute_import from invoke import Collection from . import deploy, gpg ns = Collection(deploy, gpg)
"""Read the Docs tasks."""
import os
from invoke import task, Collection
import common.tasks
import common.dockerfiles.tasks
ROOT_PATH = os.path.dirname(__file__)
namespace = Collection()
namespace.add_collection(
Collection(
common.tasks.prepare,
common.tasks.release,
),
name='deploy',
)
namespace.add_collection(
Collection(
common.tasks.setup_labels,
),
name='github',
)
namespace.add_collection(
Collection.from_module(
#!/usr/bin/env python3 """Command line tasks to build and deploy the ACW Battle Data.""" from invoke import Collection from . import data, misc ns = Collection(misc.clean, misc.doc, misc.deploy, misc.check_tables) ns.add_task(data.build, name='build', default=True) ns.add_collection(Collection.from_module(data))
# Licensed under the BSD 3-Clause license.
# For full license text, see the LICENSE file in the repo root
# or https://opensource.org/licenses/BSD-3-Clause
import sys
import os
import logging
from invoke import task, Collection, UnexpectedExit, Failure
sys.path.append(
os.path.abspath(
os.path.join(os.path.dirname(__file__),
os.path.pardir + "/cloudsplaining/")))
logger = logging.getLogger(__name__)
# Create the necessary collections (namespaces)
ns = Collection()
test = Collection("test")
ns.add_collection(test)
integration = Collection("integration")
ns.add_collection(integration)
unit = Collection("unit")
ns.add_collection(unit)
build = Collection("build")
ns.add_collection(build)
docs = Collection("docs")
ns.add_collection(docs)
This may reduce substatially the size of the database and can be run
concurrently with other processes.
Note that decreasing the database size may not always improve
performance (it may degradate writes in some circumstances).
'''
connection = psycopg2.connect('dbname=%s' % database)
cursor = connection.cursor()
cursor.execute(
"SELECT table_name, table_schema FROM information_schema.tables WHERE "
"table_type = 'BASE TABLE' AND table_schema='public'")
for table, schema in cursor.fetchall():
try:
query = 'CLUSTER "%s"."%s" USING "%s_pkey"' % (schema, table,
table)
cursor.execute(query)
connection.commit()
except psycopg2.ProgrammingError:
print(('Error executing: %s' % query))
connection.rollback()
connection.close()
DatabaseCollection = Collection()
DatabaseCollection.add_task(drop)
DatabaseCollection.add_task(dump)
DatabaseCollection.add_task(owner)
DatabaseCollection.add_task(copy)
DatabaseCollection.add_task(cluster)
if input('are you sure? [y/N] ').lower().strip().startswith('y'):
ctx.run(cmd)
__wait_for(ctx, "delete", stack_name)
else:
print("not deleting stack")
def __wait_for(ctx, op, stack_name):
wait_cmd = ("aws {} cloudformation wait stack-{}-complete "
"--stack-name {}").format(profile_arg(), op, stack_name)
print("Waiting for stack {} to complete...".format(op))
ctx.run(wait_cmd)
print("Done")
ns = Collection()
stack_ns = Collection('stack')
stack_ns.add_task(package)
stack_ns.add_task(update_function)
stack_ns.add_task(deploy)
stack_ns.add_task(delete)
stack_ns.add_task(create_dashboard)
ns.add_collection(stack_ns)
config_ns = Collection('config')
config_ns.add_task(list_stacks)
config_ns.add_task(config_check, name='check')
config_ns.add_task(config_save, name='save')
config_ns.add_task(config_pull, name='pull')
ns.add_collection(config_ns)
from invoke import Collection, task
from invocations.packaging import release
from invocations.testing import test, watch_tests, coverage as coverage_
# TODO: add coverage at the Travis level as well sometime. For now this is just
# to help me as I overhaul the release modules
@task
def coverage(c, html=True):
# TODO: can we realistically make use of functools.partial for this sort of
# thing?
# TODO: is it best left to config option overrides (currently the usual
# approach)? Is there stuff we can do to make that even easier?
return coverage_(c, html=html, integration_=False)
ns = Collection(release, test, watch_tests, coverage)
ns.configure({
'tests': {
'package': 'invocations',
},
'packaging': {
'sign': True,
'wheel': True,
},
})
def does_not_seek_tasks_module_if_namespace_was_given(self):
expect('foo',
err="No idea what 'foo' is!\n",
program=Program(namespace=Collection('blank')))
from invoke import Collection
# -- TASK-LIBRARY:
from . import clean
from . import docs
from . import test
# -----------------------------------------------------------------------------
# TASKS:
# -----------------------------------------------------------------------------
# None
# -----------------------------------------------------------------------------
# TASK CONFIGURATION:
# -----------------------------------------------------------------------------
namespace = Collection()
namespace.add_task(clean.clean)
namespace.add_task(clean.clean_all)
namespace.add_task(test.behave, "behave_test")
namespace.add_collection(Collection.from_module(docs))
namespace.add_collection(Collection.from_module(test))
# -- INJECT: clean configuration into this namespace
namespace.configure(clean.namespace.configuration())
namespace.configure(test.namespace.configuration())
if sys.platform.startswith("win"):
# -- OVERRIDE SETTINGS: For platform=win32, ... (Windows)
from ._compat_shutil import which
run_settings = dict(echo=True, pty=False, shell=which("cmd"))
namespace.configure({"run": run_settings})
@task
def get(c):
"""This task synchronizes the server to the local machine"""
inv_logging.task(get.__name__)
settings = inv_base.read_settings("production")
for rsync_task in settings["rsync_get"]:
if "include" in settings["rsync_get"][rsync_task]:
include = settings["rsync_get"][rsync_task]["include"]
else:
include = None
if "exclude" in settings["rsync_get"][rsync_task]:
exclude = settings["rsync_get"][rsync_task]["exclude"]
else:
exclude = None
logging.info(
f"The settings {rsync_task} from the settings.json file are used for the production."
)
rsync_get(c, settings["REMOTE_USER"], settings["REMOTE_HOST"],
settings["rsync_get"][rsync_task]["local_dir"],
settings["rsync_get"][rsync_task]["remote_dir"], include,
exclude)
inv_logging.success(get.__name__)
rsync_ns = Collection("rsync")
rsync_ns.add_task(push)
rsync_ns.add_task(get)
from invoke import Collection
from invocations.testing import test
from invocations.packaging import release
ns = Collection(release, test)
ns.configure({
'packaging': {
'sign': True,
'wheel': True,
},
})
# Please add the "target" parameter if you are adding more @task functions
# to allow custom targets to be used (regardless if your function itself will use it or not)
def get_target(args):
target = args.target
if target not in configs:
message = '\n\tTarget Configuration "{}" is not a valid entry'.format(
TARGET_FLAG
)
message += "\n\tInvalid Entry: " + target
assert target in configs, message
return target
parser = argparse.ArgumentParser(
description='Target parameters for the fab file through the "fab" library'
)
parser.add_argument(
"--target",
default="prod",
help="The Target Configuration key to set the deployment setting",
)
args, unknown = parser.parse_known_args()
target_key = get_target(args)
print("Target Set:", target_key)
ns = Collection(deploy, rollback)
ns.configure(configs[target_key])
from invoke import task, Collection
import sys
sys.path.append('/my/proj/release')
from serv_tasks import serv_tasks
ns = Collection()
serv_tasks(ns, 'serv.n3', 'xidle_dash')
if 0:
TAG_x86 = f'bang6:5000/{JOB.lower()}_x86:latest'
TAG_pi = f'bang6:5000/{JOB.lower()}_pi:latest'
ANSIBLE_TAG = 'homeauto_xidle'
@task
def build_image_x86(ctx):
ctx.run(f'docker build --network=host -t {TAG_x86} .')
@task
def build_image_pi(ctx):
ctx.run(f'docker build --file Dockerfile.pi --network=host -t {TAG_pi} .')
@task(pre=[build_image_x86])
def push_image_x86(ctx):
ctx.run(f'docker push {TAG_x86}')
@task(pre=[build_image_pi])
def push_image_pi(ctx):
ctx.run(f'docker push {TAG_pi}')
@task(pre=[build_image_x86])
def shell(ctx):
ctx.run(f'docker run --rm -it --cap-add SYS_PTRACE -v /tmp/.X11-unix/:/tmp/.X11-unix/ -v /home/drewp/.Xauthority:/root/.Xauthority --net=host {TAG_x86} /bin/bash', pty=True)
from invoke import Collection
from . import call
from . import container
from . import dev
from . import docs
from . import examples
from . import git
from . import mpi_native
ns = Collection(
call,
container,
dev,
docs,
examples,
git,
mpi_native,
)
