def main(argv):
env.user = '******'
env.disable_known_hosts = True
env.no_keys = True
# Disable Output
env.debug = False
try:
opts, args = getopt.getopt(argv,"hH:p:",["hostname=","password="******"--hostname"):
env.hosts = [arg]
elif opt in ("-p", "--password"):
env.password = arg
with settings(
try:
hide('warnings', 'running', 'stdout', 'stderr', 'everything')
):
execute(check_status)
def load(self, config_file):
"""Load the workflow rules from a Mamba .dc Python file
:param config_file: The file where to load the configuration from
:type config_file: str
"""
module_name = filepath.splitext(filepath.basename(config_file))[0]
if self.deployer_object and self.deployer_object.loaded:
raise deployer.DeployerError(
"Tried to load {module} deployer that is " "already loaded!".format(module=module_name)
)
self.deployer_name = module_name
self.deployer_module = deployer.deployer_import(self.deployer_name, config_file)
# load tasks
docstring, new_style, classic, default = fabric_main.load_tasks_from_module(self.deployer_module)
self.tasks = {
"docstring": docstring,
"functions": new_style if state.env.new_style_tasks else classic,
"default": default,
}
state.commands.update(self.tasks.get("functions", {}))
# abort if no commands found
if not state.commands:
log.err("No commands found ...aborting")
else:
for name in state.commands:
execute(name)
def _run_chef_on_hosts(self, hosts):
with hide(*self.hidden_outputs):
execute(self.chef_manager.push_deployment_data, hosts=hosts)
with warn_only():
results = execute(self.chef_manager.run_chef_client, hosts=hosts)
execute(self.chef_manager.pull_node_info, hosts=hosts)
return results
def httpdGracefulAll():
# get host name from hosts file
hosts=[]
f = open('/etc/hosts', 'r')
for line in f:
if line.find('sv') >= 0:
hosts.append(line.split()[1])
f.close()
if len(hosts) == 0:
sys.exit('Host Not Found.')
# show target hosts, and make extra sure
print 'Target host is ...'
for host in hosts:
print host
confirm = raw_input('OK [yes/No]:')
if confirm == 'yes' or confirm == 'Yes' or confirm == 'YES':
pass
else:
sys.exit('execute stop')
# execute httpd graceful serially
for host in hosts :
#execute(checkAccesslogUpdateContinue, hosts=host, accesslog='/var/log/httpd/8080_access_log', egrep='OPTIONS|304', invert_match=True)
execute(httpdConfigtest, hosts=host)
#execute(httpdMaintenanceStart, hosts=host, maintenance_start_command='/root/http-maintenance-for-webserver.sh web01')
#execute(checkAccesslogUpdateStop, hosts=host, accesslog='/var/log/httpd/8080_access_log', egrep='OPTIONS|304', invert_match=True)
execute(httpdGraceful, hosts=host)
def swarm_up_slaves(args):
logging.info("Bringing up {0} swarm slaves".format(args.num_slaves))
_validate_dirs(args)
cfg = get_config(args.config)
master_ip_address = get_master_ip_address(cfg)
if not master_ip_address:
raise Exception("Unable to start slaves without a master. Please "
"bring up a master first.")
for i in xrange(args.num_slaves):
pool.apply_async(create_slave, (cfg,))
pool.close()
pool.join()
update_master_security_group(cfg)
# TODO: Hack for now, should check for ssh-ability
time.sleep(5)
_update_role_defs(get_slave_reservations(cfg), 'slave')
env.user = cfg.get('fabric', 'user', None)
env.key_filename = cfg.get('fabric', 'key_filename', None)
env.parallel = True
execute(_bootstrap_slave, args.directory, master_ip_address)
_disconnect_fabric()
def database_restore_locally(host, user, password, dbname, dba, backup_path):
"""
:type host: str
:param host: Host
:type user: str
:param user: SSH user name.
:type password: str
:param password: SSH user password.
:type dbname: str
:param dbname: Database name.
:type dba: str
:param dba: Database administrator user name.
:type backup_path: str
:param backup_path: Path to backup.
:return: None
"""
env.user = user
env.password = password
execute(task_database_restore_locally,
dbname, dba,
backup_path, '/home/%s/backup/' % user,
hosts=[host])
def initialize(self):
def _initialize():
sudo(self._get_bootstrap_script())
if not self._initialized:
execute(_initialize, hosts=[self._host_string])
self._initialized = True
return self._initialized
def headless():
# How do I enable the "multiverse" repository?
# http://askubuntu.com/questions/89096/how-do-i-enable-the-multiverse-repository
sudo('sed -i "/^# deb.*multiverse/ s/^# //" /etc/apt/sources.list')
timezone()
execute(ntpd)
require_deb_packages( '''
sudo
screen
htop
nmap
iotop
mc
ssh
nano
cpuid
curl
wget
nfs-kernel-server
nfs-common
''')
require.directory('bin')
require.file('.profile', contents=cfg('profile.conf'))
def reboot():
"""Reboot all the etherpad servers."""
cluster_util.ensure_sudo_pass()
with settings(hosts=cluster_hosts.etherpad()):
execute(etherpad.stop)
execute(etherpad.start)
def desktop_basic():
execute(headless)
require_deb_packages('''
baobab
rdesktop
x11vnc
xtightvncviewer
ssvnc
zim
synaptic
gparted
remmina
scrot
shutter
avahi-discover
avahi-ui-utils
avahi-utils
avahi-dnsconfd
xdotool
'''
)
require.directory('shots')
def main():
env['host_tuples'] = []
env['errors'] = []
tasks.execute(get_disk_space)
host_list = env['host_tuples']
error_list = env['errors']
if error_list:
print '-' * 80
print 'ERRORS:'
print
for item in error_list:
print '%s %s %s' % item
print
if not host_list:
print 'ERROR: something bad happened, no host data'
return
low_space_list = []
total_space = 0
for host, freespace in host_list:
total_space += freespace
if freespace <= THRESHOLD:
low_space_list.append((host, freespace))
avg_free_space = total_space / len(host_list)
if low_space_list:
print '-' * 80
print 'LOW DISK SPACE:'
for item in low_space_list:
print '%-32s %s' % item
print
print 'AVERAGE FREE DISK SPACE: %s' % avg_free_space
disconnect_all()
def upgrade():
"""Upgrade all Etherpad servers to the version configured in puppet.
This will:
1. Ask for a password with which to sudo. All servers must have the
same sudo passowrd
2. Perform a git pull on the puppet node to get the latest
configuration
3. On each etherpad host:
a. Forcefully stop puppet on the etherpad hosts and cancel any
runs that are currently happening
b. Stop the etherpad services
c. Back up the etherpad dir to /tmp/etherpad_backup
d. Run puppet, which should notice the missing directory and
repopulate it with the configured (updated) version of the
package
"""
cluster_util.ensure_sudo_pass()
# Pull the updated puppet data
with settings(hosts=[cluster_hosts.puppet()]):
execute(puppet.git_update)
# Run all etherpad upgrades in parallel
with settings(hosts=cluster_hosts.etherpad(), parallel=True):
execute(upgrade_host_internal)
def swarm_up_slaves(args):
logging.info("Bringing up {0} swarm slaves".format(args.num_slaves))
_validate_dirs(args)
cfg = get_config(args.config)
master_ip_address = get_master_ip_address(cfg)
security_group = get_security_group_from_role(cfg, DEFAULT_SLAVE_ROLE_NAME)
if not master_ip_address:
raise Exception("Unable to start slaves without a master. Please "
"bring up a master first.")
for i in xrange(args.num_slaves):
pool.apply_async(create_slave, (cfg, security_group))
pool.close()
pool.join()
_wait_for_slave_reservations(cfg, args.num_slaves)
update_master_security_group(cfg)
_update_role_defs(get_slave_reservations(cfg), 'slave')
env.user = cfg.get('fabric', 'user', None)
env.key_filename = cfg.get('fabric', 'key_filename', None)
env.parallel = True
execute(_bootstrap_slave, args.directory, master_ip_address)
_disconnect_fabric()
def copy_parse_collect(input=None, batch_size=None, parallelism=2, key_id='item_id', content_id='content'):
"""
Wrapper function to split and copy file to servers, parse, and collect
If batch_size is None, it will be automatically calculated based on number of machines
and specified parallelism
"""
if input is None:
print('Please specify input file to parse.')
exit(0)
ensure_hosts()
num_machines = num_lines('.state/HOSTS')
print('Preparing to run on %s machines with PARALLELISM=%s' % (num_machines, parallelism))
if batch_size is None:
batch_size = min(num_lines(input) / (num_machines * int(parallelism)), MAX_BATCH_SIZE)
# Copy the files to the remote machines
execute(copy, input=input, batch_size=batch_size)
# Sometimes coreNLP doesn't download- restage here to ensure compiled
execute(restage)
# Parse
execute(parse, parallelism=parallelism, key_id=key_id, content_id=content_id)
# Collect
execute(collect)
execute(get_status)
def bootstrap(master=False):
"""Provision salt-cloud on the bare box. See saltstack/salt-bootstrap."""
run("apt-get -q update")
run("apt-get -q --yes install git python-dev build-essential python-pip sshpass")
if master:
run("curl -L http://bootstrap.saltstack.org | "
"sh -s -- -M -N git develop")
run("pip install psutil apache-libcloud")
run("pip install git+https://github.com/saltstack/salt-cloud.git"
"#egg=salt_cloud")
conf = SALT_CLOUD_TEMPLATE
conf.update({
'apikey': env.rackspace_api_key,
'minion': {'master': str(env.box_public_ips[4])},
'tenant': str(env.rackspace_tenant_id),
'user': env.rackspace_user,
})
conf = {'rackspace-conf-{0}'.format(env.rackspace_user): conf}
put(StringIO(yaml.dump(conf)), '/etc/salt/cloud.providers')
# dummy a profiles file
put('cloud.profiles', '/etc/salt/cloud.profiles')
else: # minion
run("python -c 'import urllib; print urllib.urlopen("
"\"http://bootstrap.saltstack.org\").read()' | "
"sh -s -- git develop")
execute(shell)
def set_device_read_ahead(read_ahead, devices=None):
"""Set device read ahead.
If devices argument is None, use the 'block_devices' setting from the cluster config."""
if devices is None:
devices = config['block_devices']
execute(cstar.set_device_read_ahead, read_ahead, devices)
def pre_backup_tasks(mysql_root_password):
# allow operations to complete and then gracefully shut down.
with settings(warn_only=True):
run('mysql -uroot -p%s -e"SET GLOBAL innodb_fast_shutdown = 0"'%( mysql_root_password) )
# stop mysql
execute( stop_mysql , hosts = [ '%s'%( env.host ) ] )
def uninstall(self):
self.chef_manager.clear_run_list(self.all_hosts)
if self.roles['clc']:
self.chef_manager.add_to_run_list(self.all_hosts,
['eucalyptus::nuke'])
if self.roles['riak-head']:
self.chef_manager.add_to_run_list(self.all_hosts,
['riakcs-cluster::nuke'])
if self.roles['ceph-mons']:
self.chef_manager.add_to_run_list(self.all_hosts,
['ceph-cluster::nuke'])
if self.roles['haproxy']:
self.chef_manager.add_to_run_list(self.all_hosts,
['haproxy::nuke'])
if self.roles['zookeeper']:
self.chef_manager.add_to_run_list(self.all_hosts,
['zookeeper::nuke'])
if self.roles['cassandra']:
self.chef_manager.add_to_run_list(self.all_hosts,
['cassandra::nuke'])
if self.roles['midonet-cluster']:
self.chef_manager.add_to_run_list(self.all_hosts,
['eucalyptus::midonet-nuke'])
if self.roles['midolman']:
self.chef_manager.add_to_run_list(self.all_hosts,
['eucalyptus::midolman-nuke'])
self._run_chef_on_hosts(self.all_hosts)
with lcd('chef-repo'):
local('knife node bulk delete -z -E {0} -y ".*"'.format(self.environment_name))
execute(self.chef_manager.clear_node_info, hosts=self.all_hosts)
print green('Uninstall has completed successfully. '
'Your cloud is now torn down.')
def ipawho(ipafile=None, variant=None):
"""
show the users in the ipa's file provisioning profile
gets the device list from your ios dev account, then
parses the IPA file's provisioning profile to show
all the user names / emails that can install the ipa.
Usage:
ipawho:/path/to/ipa
alias ipawho = "find . -name *ipa | xargs -I/ fab ipawho:/"
"""
import csv
from StringIO import StringIO
execute('check_iosappdev_credentials', variant)
appdevuser = os.environ.get('IOS_APPDEVUSER_%s_%s' % (app, variant))
with settings(warn_only=True):
list = local('ios devices:list --format csv', capture=True)
csvlist = StringIO(list)
devlist = {}
for device in csv.DictReader(csvlist) if list !='No devices found' else []:
devid = device.get('Device Identifier').strip()
user = device.get('Device Name')
devlist[devid] = user
ipainfo = local('ipa info %s' % ipafile, capture=True)
parsing = False
for line in ipainfo.split('\n'):
if 'ProvisionedDevices' in line:
parsing = True
if 'TeamIdentifier' in line:
parsing = False
if parsing:
uuid = line.replace('ProvisionedDevices', '').split('|')[2].strip()
print uuid, devlist[uuid]
def build_docker_image_from_git_repo_and_push_to_registry(gitRepository,
dockerImageName, **kwargs):
"""A Fabric task which **runs locally**; it builds a Docker image from a git
repository and pushes it to the Docker registry (http://index.docker.io).
This task runs the `build_docker_image_from_git_repo` task followed by the
`push_docker_image_to_registry` task.
**NOTE:** This Fabric task is only run once regardless of the number of
hosts/roles you supply.
Args:
gitRepository(str): Refer to the docstring for the same parameter in
`build_docker_image_from_git_repo` Fabric task
dockerImageName(str): Refer to the docstring for the same parameter in the
`build_docker_image_from_git_repo` Fabric task
\*\*kwargs: Keyword arguments passed to the `build_docker_image_from_git_repo`
Fabric task
Returns:
str: The tag of the built Docker image
"""
retVal = execute(build_docker_image_from_git_repo, gitRepository,
dockerImageName, roles=env.roles, hosts=env.hosts, **kwargs
)
dockerImageTag = \
viki_fab_helpers.get_return_value_from_result_of_execute_runs_once(retVal)
execute(push_docker_image_to_registry, dockerImageName,
dockerImageTag=dockerImageTag, roles=env.roles, hosts=env.hosts
)
return dockerImageTag
def highstate(self, instances):
if instances:
instances = instances.values()
salt_dir = os.path.join(os.getcwd(), "deploy", "salt/")
instances[0].environment.master_server().upload(salt_dir, "/srv/salt")
saltcmd = "salt-call"
if self.environment.opts['all'] is True:
saltcmd = "salt \*"
@parallel
def highstate():
with settings(warn_only=True):
sudo("%s state.highstate" % (saltcmd))
execute(self._prepare_for_highstate)
if self.environment.opts['all'] is False:
env = build_fabric_env(instances)
else:
env = build_fabric_env(self.environment.master_server())
execute(highstate)
else:
print "There was an error finding the instance you're referring"
def post_deploy(deploy_tag, web_tag):
update_current_version(deploy_tag, web_tag, 'set')
update_next_version(deploy_tag, web_tag, 'rm')
env.host_string = get_hosts_for_role('EtcdCluster', limit_to_one=True)[0]
service_keys = run('etcdctl ls /_coreos.com/fleet/states/').split()
status = defaultdict(dict)
for units_service in service_keys:
service_name = os.path.split(units_service)[-1]
unit_keys = run('etcdctl ls {}'.format(units_service)).split()
for unit in unit_keys:
unit_name = os.path.split(unit)[-1]
status[service_name][unit_name] = json.loads(run('etcdctl get {}'.format(unit)))
units_that_should_be_active = make_services_and_get_names(deploy_tag, web_tag)
units_that_should_be_active = [x+'.service' for x in units_that_should_be_active]
for unit, info in status.items():
if unit not in units_that_should_be_active:
execute(
fleetctl, "unload %s" % unit
)
execute(
fleetctl, "destroy %s" % unit
)
def refresh_code_on_machines():
""" Refresh the current version of the code on the machines. """
hostnames = [instance.public_dns_name for instance in get_all_instances()]
with settings(parallel=True, user='******',
key_filename='/Users/aismail/.ssh/ami-keypair.pem'):
execute('refresh_code', hosts=hostnames)
def rewrite_shutdown(self, name, vals):
c = css.Css.get(name)
output['everything'] = False
for srv in c.servers:
addr = '{0}@{1}'.format(srv.taddr.user, srv.taddr.hostname)
env.hosts = [addr]
execute(fabfile.rewrite_shutdown, c, srv.no, vals[addr])
def bump_version():
"""Bump up the version number"""
print yellow("bumping version ...")
write_version_info()
print green("adding file '%s' to next commit." % env.version_file)
local("git add " + env.version_file)
execute(version)
def install(identifier):
init_ssh()
path = get_path(identifier)
allHosts = get_all_hosts()
execute(execute_install_jmeter, identifier=identifier, path=path, force=False, hosts=allHosts)
execute(execute_install_loadtest, identifier=identifier, path=path, force=False, hosts=allHosts)
def format_hdfs():
execute(reset_hdfs_dirs)
local('hdfs namenode -format')
local('start-dfs.sh')
local('hdfs dfs -mkdir /user')
local('hdfs dfs -mkdir /user/ubuntu')
local('stop-dfs.sh')
def job_pserver(jobdir, pids=None):
'''
start all pservers
'''
pargs = " --num_gradient_servers=" + str(len(conf.HOSTS))
pargs += (" --nics=" + conf.PADDLE_NIC)
pargs += " --port=" + str(conf.PADDLE_PORT)
pargs += " --ports_num=" + str(conf.PADDLE_PORTS_NUM)
#always start sparse pserver by default
pargs += " --ports_num_for_sparse=" + str(conf.PADDLE_PORTS_NUM_FOR_SPARSE)
pargs += " --comment=" + "paddle_process_by_paddle"
def start_pserver(jobdir, pargs):
'''
start pserver process with fabric executor
'''
with prefix('export LD_LIBRARY_PATH=' + \
conf.LD_LIBRARY_PATH + \
':$LD_LIBRARY_PATH'):
program = 'paddle pserver'
run('cd ' + jobdir + '; ' + \
'GLOG_logtostderr=0 GLOG_log_dir="./log" ' + \
'nohup ' + \
program + " " + pargs + ' > ./log/server.log 2>&1 < /dev/null & ',
pty=False)
execute(start_pserver, jobdir, pargs, hosts=conf.HOSTS)
def deploy():
""" Pull the changes, update requirements on remote host"""
with cd(Config.project_path):
if not exists('.git'):
print("Cloning GIT repo")
execute(clone)
execute(pull_repo)
def test_package_manager_stuff():
with settings(hosts=[host]):
result = tasks.execute(deploy.package_manager.system)
assert result[host] in ['apt', 'yum', 'pacman']
result = tasks.execute(deploy.package_manager._manager)
assert result[host] in ['apt-get', 'yum', 'pacman']
def make_saltmaster():
"""Spin up and provision a Salt Master on an OpenStack enabled service."""
name = NAMING_SCHEME['saltmaster']
try:
herd(name)
except KeyError:
birth(name, '512', wait=True)
else:
print red("Saltmaster ('{0}') already exists!".format(name))
if euthanise(wait=True): # kill the old one and make a new one
birth(name, '512', wait=True)
time.sleep(10) # takes a little while for SSH to come up...
time.sleep(5) # oddly, need more time here...
execute(bootstrap, master=True)
execute(season)
def run_command(self,
host,
command,
key_file,
command_args=[],
remote_vars={}):
host_string = "{0}@{1}".format(self.user, host)
commands = [command] if isinstance(command, basestring) else command
with ctxt.settings(host_string=host_string,
key_filename=key_file,
output_prefix=False):
with ctxt.hide("running", "aborts"):
for cmd in commands:
if cmd in ["sh", "shell"]:
print(green("Logging in as '%s'" % self.user))
subprocess.call(
["ssh", "-i", key_file,
"%s" % (host_string)])
else:
print(
green("----- Running '{0}' on '{1}' -----".format(
cmd, self.name)))
res = execute(commander.run_remote,
cmd,
command_args=command_args,
remote_vars=remote_vars)
print(
green("----- Done '{0}' on '{1}' -----".format(
cmd, self.name)))
return True
def set_global_system(vmid, params):
logger.debug("set_global_system call!!")
vm = get_vm(vmid)
addr = vm['mgraddr']
env.hosts = [addr]
env.user = vm['sshid']
env.password = vm['sshpw']
env.shell = '/bin/vbash -ic'
results = execute(set_global_system_task, hosts=[addr], systeminfo=params)
if results[addr]['success'] == 'success' or 'already exists' in results[
addr]['errmsg']:
vms = read_repository('vms')
for vm in vms:
if '_id' in vm and vm['_id'] == vmid:
vm['hostname'] = params['hostname']
isChanged = True
break
if isChanged == True:
write_repository('vms', vms)
return results[addr]
def switch_to_second_phase(eng_hosts_1, eng_hosts_2, ws_hosts_1, ws_hosts_2):
# first, enable / disable kraken nodes
#execute(enable_nodes, eng_hosts_1)
execute(enable_kraken_haproxy, eng_hosts_1)
#execute(disable_nodes, eng_hosts_2)
execute(disable_kraken_haproxy, eng_hosts_2)
# then enable / disable jormun nodes
execute(enable_nodes, ws_hosts_1)
execute(disable_nodes, ws_hosts_2)
def getVmAllState(addr, sshid, sshpw, vms):
env.hosts = [addr]
env.user = sshid
env.password = sshpw
env.shell = '/bin/bash -l -c'
results = execute(vm_all_state_task, hosts=[addr], vmList=vms)
return results[addr]
def cqlsh(script, node):
"""Run a cqlsh script on a node"""
global cqlsh_path
script = script.replace('\n', ' ')
cmd = '{cqlsh_path} --no-color {host} -e "{script}"'.format(cqlsh_path=cqlsh_path, host=node, script=script)
with common.fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'), hosts=node):
return execute(fab.run, cmd)[node]
def getDomundefine(addr, sshid, sshpw, name):
env.hosts = [addr]
env.user = sshid
env.password = sshpw
env.shell = '/bin/bash -l -c'
results = execute(domundefine, name, hosts=[addr])
return results[addr]
def logs(dest_dir=TMP_PRESTO_DEBUG):
"""
Gather all the server logs and presto-admin log and create a tar file.
Parameters:
dest_dir - logs destination dir. default /tmp/presto-debug
"""
downloaded_logs_location = os.path.join(dest_dir, "logs")
ensure_directory_exists(downloaded_logs_location)
print 'Downloading logs from all the nodes...'
execute(get_remote_log_files, downloaded_logs_location, roles=env.roles)
copy_admin_log(downloaded_logs_location)
logs_tar_file_location = os.path.join(dest_dir, OUTPUT_FILENAME_FOR_LOGS)
make_tarfile(logs_tar_file_location, downloaded_logs_location)
print 'logs archive created: ' + logs_tar_file_location
def bash(script, nodes=None, user=None):
"""Run a bash script on a set of nodes
script - A bash script written as a string or list.
nodes - The set of nodes to run the command on. If None, all nodes of
the cluster will be used.
user - The user to run the command as. If None, the default user specified
in the cluster configuration
"""
if type(script) in (list, tuple):
script = "\n".join(script)
if nodes is None:
nodes = cstar.fab.env.hosts
if user is None:
user = cstar.fab.env.user
with cstar.fab.settings(user=user, hosts=nodes):
execute(cstar.bash, script)
def check_role_is_up(role: str, task: Callable) -> Tuple[dict, str]:
is_uwsgi_up_results = execute(task, role=role)
per_hosts_success = {
host: res.succeeded
for host, res in is_uwsgi_up_results.items()
}
joint_stderr = '\n'.join(r.stdout for r in is_uwsgi_up_results.values())
return per_hosts_success, joint_stderr
def action(self):
opts = self.get_options()
cmds = opts.get('execute', [])
env.roledefs = opts['roledefs']
for cmd in cmds:
if not cmd.has_key('command') or \
not isinstance(cmd.get('command'), str):
raise ValueError('wrong command')
method = cmd.get('method', default_method)
if not run_methods.has_key(method):
raise ValueError('wrong run method: %s' % (method))
func = run_methods[method]
roles = cmd.get('roles', [])
if not hasattr(roles, '__iter__'):
roles = [roles]
command = cmd.get('command') % opts.get_context()
execute(func, command, roles=roles)
def pingVM(addr, sshid, sshpw):
os.system("rm ~/.ssh/known_hosts")
env.hosts = [addr]
env.user = sshid
env.password = sshpw
env.shell = '/bin/bash -l -c'
results = execute(pingVM_task, hosts=[addr])
return results[addr]
def centos_install_packages():
"""
Installs required packages on CentOS.
"""
execute(centos_install_epel)
run("yum install --assumeyes "
"gcc.x86_64 memcached python-devel.x86_64 zlib-devel.x86_64 "
"memcached memcached-devel libmemcached-devel nginx "
"java-1.6.0-openjdk python-virtualenv")
if not exists("/etc/yum.repos.d/riptano.repo"):
put(
os.path.join(BASE_DIRECTORY, "dev-scripts", "centos",
"riptano.repo"), "/etc/yum.repos.d/riptano.repo")
run("yum install --assumeyes apache-cassandra11")
run("chkconfig --add cassandra")
run("chkconfig cassandra on")
run("service cassandra start")
def test_should_return_dict_for_base_case(self):
"""
Non-network-related tasks should return a dict w/ special key
"""
def task():
return "foo"
eq_(execute(task), {'<local-only>': 'foo'})
def retrieveClusterInformation():
global CLUSTER_PRIVATE_IPS
global CLUSTER_MASTER_IP
private_ips = execute(getPrivateIp)
CLUSTER_PRIVATE_IPS = private_ips
CLUSTER_MASTER_IP = CLUSTER_PRIVATE_IPS[CLUSTER_MASTER]
def check_pfs_dirs(self, name, exists=True):
c = css.Css.get(name)
output['everything'] = False
for srv in c.servers:
addr = '{0}@{1}'.format(srv.taddr.user, srv.taddr.hostname)
env.hosts = [addr]
ret = execute(fabfile.check_pfs_dirs, c, srv.no, exists)
self.assertTrue(ret[addr])
def mark_deployed_backend(pr,
force_hostname=False,
owner='gisce',
repository='erp'):
from apply_pr import fabfile
configure_logging()
mark_deployed_task = WrappedCallableTask(fabfile.mark_deployed)
execute(mark_deployed_task,
pr,
hostname=force_hostname,
owner=owner,
repository=repository)
click.echo(
colors.green(
u"Marking PR#{} as deployed success! \U0001F680".format(pr)))
def deploy_django_application(stage="production"):
"""
Deploy application in selected server(s)
"""
set_stage(stage)
set_user()
execute(Project.push, hosts=env.hosts)
execute(Project.environment, hosts=env.hosts)
execute(Project.install, hosts=env.hosts)
execute(Project.clean, hosts=env.hosts)
def uninstall(self):
self.chef_manager.clear_run_list(self.all_hosts)
if self.roles['clc']:
self.chef_manager.add_to_run_list(self.all_hosts, ['eucalyptus::nuke'])
if self.roles['riak-head']:
self.chef_manager.add_to_run_list(self.all_hosts, ['riakcs-cluster::nuke'])
if self.roles['ceph-mons']:
self.chef_manager.add_to_run_list(self.all_hosts, ['ceph-cluster::nuke'])
if self.roles['haproxy']:
self.chef_manager.add_to_run_list(self.all_hosts, ['haproxy::nuke'])
if (self.roles['midolman'] or self.roles['mido-cassandra'] or
self.roles['mido-zookeeper'] or self.roles['midonet-api']):
self.chef_manager.add_to_run_list(self.all_hosts, ['midokura::nuke'])
self._run_chef_on_hosts(self.all_hosts)
with lcd('chef-repo'):
local('knife node bulk delete -z -E {0} -y ".*"'.format(self.environment_name))
execute(self.chef_manager.clear_node_info, hosts=self.all_hosts)
print green('Uninstall has completed successfully. Your cloud is now torn down.')
def main():
uptimes = tasks.execute(uptime)
avg_ut = sum(uptimes.values()) / len(uptimes)
print '-' * 80
print
print 'Average Uptime: %s' % avg_ut
print
print '-' * 80
disconnect_all()
def main():
# Normal fabric usage - synchronous
execute(get_info)
MLN = 1000 * 1000
# lat/throughput curves
for mean_lat in [
100
]: # FAKE SERVICE TIME (SPINNING): 100usec for each RPC packet
args['mean_lat'] = mean_lat
args['th_rate_base'] = 10 * 1000 # START LOAD IS 10k QPS
args[
'th_rate_step'] = 70 * 1000 # INCREASE THE LOAD BY THE FOLLOWING STEP
args['th_rate_max'] = 350 * 1000 # FINAL LOAD
eval_single_vm()
eval_kata()
def remote_execute_steps(host, recipe_steps):
"""
:param host:
:param recipe_steps:
:return:
"""
env.host_string = host
env.user = "******"
env.key_filename = "assets/adl-performance.pem"
for step in recipe_steps:
if step.startswith("#"):
print "Skipping step: {0}".format(step)
continue
else:
print "Running step: {0}".format(step)
execute(fabric_run_step, step, hosts=[host])
def main():
uts_dict = tasks.execute(uptime)
avg_ut = sum(uts_dict.values()) / len(uts_dict)
print '-' * 80
print
print 'Average Uptime: %s' % avg_ut
print
print '-' * 80
disconnect_all() # Call this when you are done, or get an ugly exception!
def getMeasures():
try:
timestamp = datetime.datetime.utcnow().replace(tzinfo=utc)
now = timestamp - datetime.timedelta(seconds=timestamp.second, microseconds=timestamp.microsecond)
env.skip_bad_hosts = True
env.timeout = 10
env.connection_attempts = 1
servers = Server.objects.filter(is_measuring=False)
if servers:
env.hosts = ["%s@%s" % (server.username, server.ip) for server in servers]
env.passwords = dict([("%s@%s" % (server.username, server.ip), server.password) for server in servers])
with hide('everything'):
execute(launch_command, now)
except:
total_time = datetime.datetime.utcnow().replace(tzinfo=utc) - timestamp
duration = float(int((total_time.seconds * 1000000) + total_time.microseconds) / 1000000.0)
CronLog.objects.create(timestamp=timestamp, action="sensors", server=None, success=False, duration=duration, message=traceback.format_exc())
return 0
def sdk_reset(warn_opt='warn'):
if warn_opt != 'nowarn':
io.warn(
"This step will fetch and reset the public repo to the latest version."
)
if not confirm("Proceed?"):
abort("OK, fine. I understand. :(")
execute(sdk_setup)
with lcd(env.public_repo_path):
local("git checkout master")
# Merge in the internal repo's develop branch.
local("git fetch public --prune")
local("git reset --hard public/master")
local("git clean -x -d -f")
def refresh_vm(addr, sshid, sshpw):
env.hosts = [addr]
env.user = sshid
env.password = sshpw
env.shell = '/bin/vbash -ic'
results = execute(refresh_vm_task, hosts=[addr])
return results[addr]
def test_should_set_all_hosts(self):
"""
should set env.all_hosts to its derived host list
"""
hosts = ['a', 'b']
roledefs = {'r1': ['c', 'd']}
roles = ['r1']
exclude_hosts = ['a']
def command():
eq_(set(env.all_hosts), set(['b', 'c', 'd']))
task = Fake(callable=True, expect_call=True).calls(command)
with settings(hide('everything'), roledefs=roledefs):
execute(task,
hosts=hosts,
roles=roles,
exclude_hosts=exclude_hosts)
def eval_single_vm():
args['exp_prefix'] = 'single_vm'
#for vcpu_num in [1, 4, 16]:
for vcpu_num in [1]:
args['vcpu_n'] = vcpu_num
args['thr_n'] = vcpu_num
for uvm_num in [1, 40, 30, 20, 10]:
args['vm_n'] = uvm_num
execute(server_cleanup)
mnt = dbr.Monitor()
args['should_wait'] = False
mnt.bg_execute(run_single_vm, **args)
time.sleep(15 + uvm_num / 10)
args['should_wait'] = True
mnt.bg_execute(run_lancet, **args)
mnt.monitor()
mnt.killall()
def getInfo():
cmd=request.form['cmd']
result=execute(demo1.runall,cmd)
print(result)
for a in result.values():
print(a)
return render_template('info2.html',result=a[0],cmd=cmd)
def copy_jce_policy_files(cluster):
"""" Copy JCE unlimited strength policy files to all nodes. """
source_folder = cluster.get_config(KEY_JCE_POLICY_FILES_LOCATION)
if not source_folder:
raise KerberosConfigError(
'The location of JCE Unlimited Strength Policy files was not found in {}'
.format(cluster.get_config_file()))
jar_files_pattern = os.path.join(source_folder, "*.jar")
if not glob.glob(jar_files_pattern):
raise KerberosConfigError(
'No policy jar files found in {}'.format(source_folder))
target_dir = os.path.join(cluster.get_config(KEY_JAVA_HOME), 'jre', 'lib',
'security')
execute(copy,
hosts=cluster.get_all_hosts(),
source_file=jar_files_pattern,
remote_file=target_dir)
