def flush_fq_chain():
for table in ('filter', 'nat'):
rules = dump_table(table)
for chain, chain_rules in rules.items():
if chain.startswith('fq_'):
shell.call(
shlex.split('iptables -t %s --flush %s' % (table, chain)))
def init_fq_chains_for_table(table, chains):
rules = dump_table(table)
for chain in chains:
fq_chain = 'fq_%s' % chain
if fq_chain not in rules:
shell.call(shlex.split('iptables -t %s -N %s' % (table, fq_chain)))
ensure_first_target(table, chain, rules.get(chain, []), fq_chain)
def init_fq_chains_for_table(table, chains):
rules = dump_table(table)
for chain in chains:
fq_chain = 'fq_%s' % chain
if fq_chain not in rules:
shell.call(shlex.split('iptables -t %s -N %s' % (table, fq_chain)))
ensure_first_target(table, chain, rules.get(chain, []), fq_chain)
def deploy_puppet_module(module_path, node_express=None, node_file_name=None, puppet_node_path='/etc/puppet/manifests/nodes', puppet_module_path='/etc/puppet/modules/'):
if node_express:
node_file_path = os.path.join(puppet_node_path, node_file_name)
with open(node_file_path, 'w') as fd:
fd.write(node_express)
shell.call('rm -rf %s' % os.path.join(puppet_module_path, os.path.basename(module_path)))
shell.call('cp -r %s %s' % (module_path, puppet_module_path))
def get_build_number():
'''
Get current git repo commits time since last tag.
if no commit, this is an tag
'''
commitid = call('git rev-list --tags --no-walk --max-count=1')
count = call('git rev-list %s.. --count' % commitid.output()[0])
return int(count.output()[0])
def delete_chain(target):
for table in ("filter", "nat"):
rules = dump_table(table)
for chain, chain_rules in rules.items():
for i, rule in enumerate(reversed(chain_rules)):
index = len(chain_rules) - i
if target == rule["target"]:
delete_rule(table, chain, str(index))
shell.call(shlex.split("iptables -t %s --flush %s" % (table, target)))
shell.call(shlex.split("iptables -t %s -X %s" % (table, target)))
def log(launch,
label,
bucket=shell.conf.get_or_prompt_pref('launch_logs_bucket', __file__, message='bucket for launch_logs'),
tail_only=False):
launch = launch.replace('launch=', '')
label = label.replace('label=', '')
prefix = '%(bucket)s/launch_logs/launch=%(launch)s/label=%(label)s' % locals()
suffix = '.tail' if tail_only else ''
key = 's3://%(prefix)s/nohup.out%(suffix)s' % locals()
shell.call('aws s3 cp %(key)s -' % locals())
def insert_rules(rules):
created_chains = set()
for signature, rule_args in reversed(rules): # insert the last one first
table, chain, _ = rule_args
if chain not in ["OUTPUT", "INPUT", "FORWARD", "PREROUTING", "POSTROUTING"] and chain not in created_chains:
shell.call(shlex.split("iptables -t %s -N %s" % (table, chain)))
created_chains.add(chain)
if contains_rule(table, chain, signature):
LOGGER.info("skip insert rule: -t %s -I %s %s" % rule_args)
else:
insert_rule(signature.get("optional"), *rule_args)
def prepare_salt_state(state_path, salt_state_path='/srv/salt'):
try:
subprocess.call(['salt', '--version'])
except Exception as e:
print "Execute `salt --version` failed. Probably there isn't salt installed"
raise e
if not os.path.exists(salt_state_path):
os.makedirs(salt_state_path, 0755)
shell.call('rm -rf %s' % os.path.join(salt_state_path, os.path.basename(state_path)))
shell.call('cp -r %s %s' % (state_path, salt_state_path))
def commit(self, files=[], message='', options=''):
options = parse_options(options)
message = '"-m {0} "'.format(message)
if files == []:
shell.call('git commit {options}{message}'.format(
options = options, message = message))
elif type(files) is list:
files = ' '.join(files)
shell.call('git commit {options}{message}'.format(options = options) + files)
else:
raise TypeError('Files has to be in a list, not {0}'.format(
type_in_str(files)))
def add(self, files=[], options=''):
options = parse_options(options)
if files == []:
shell.call('git add --all {options}'.format(
options = options))
elif type(files) is list:
files = ' '.join(files)
shell.call('git add {options}'.format(
options = options) + files)
else:
raise TypeError('Files has to be list, not {0}'.format(
type_in_str(files)))
def log(launch,
label,
bucket=shell.conf.get_or_prompt_pref('launch_logs_bucket',
__file__,
message='bucket for launch_logs'),
tail_only=False):
launch = launch.replace('launch=', '')
label = label.replace('label=', '')
prefix = '%(bucket)s/launch_logs/launch=%(launch)s/label=%(label)s' % locals(
)
suffix = '.tail' if tail_only else ''
key = 's3://%(prefix)s/nohup.out%(suffix)s' % locals()
shell.call('aws s3 cp %(key)s -' % locals())
def prepare_salt_state(state_path, salt_state_path='/srv/salt'):
try:
subprocess.call(['salt', '--version'])
except Exception as e:
print "Execute `salt --version` failed. Probably there isn't salt installed"
raise e
if not os.path.exists(salt_state_path):
os.makedirs(salt_state_path, 0755)
shell.call('rm -rf %s' %
os.path.join(salt_state_path, os.path.basename(state_path)))
shell.call('cp -r %s %s' % (state_path, salt_state_path))
def deploy_puppet_module(module_path,
node_express=None,
node_file_name=None,
puppet_node_path='/etc/puppet/manifests/nodes',
puppet_module_path='/etc/puppet/modules/'):
if node_express:
node_file_path = os.path.join(puppet_node_path, node_file_name)
with open(node_file_path, 'w') as fd:
fd.write(node_express)
shell.call('rm -rf %s' %
os.path.join(puppet_module_path, os.path.basename(module_path)))
shell.call('cp -r %s %s' % (module_path, puppet_module_path))
def check():
status = call('git status -s --show-stash')
if len(status.output()):
m.error('There umcommit files, plsease commit them.')
m.info("\n".join(status.output()))
return False
return True
def confirm(files):
conanfile = load_conanfile_class(os.path.join(__dir__, 'conanfile.py'))
m.info('''
version: %s bumped. the following files has been modified, please verify.
''' % conanfile.version)
res = prompt('commit this change(yes) or rollback (no)?', ['yes', 'no'])
if res == 'yes':
call('git commit -a -m "bumps to version %s"' % conanfile.version)
call('git tag v{0} -m "bumps to version {0}"'.format(
conanfile.version))
m.success('version %s has been bumped.' % conanfile.version)
m.info('''
Now we will push the commit to remote
''')
os.system('git push')
m.info('''
Now we will push the commit and tags to remote
''')
os.system('git push --tags')
m.warn('DO NOT forget change the branch to dev version.')
else:
for filename in files:
call('git checkout -- %s' % filename)
m.warn('version bump canceled, all changed file reverted.')
def ensure_first_target(table, from_chain, from_chain_rules, to_chain):
if not from_chain_rules:
shell.call(shlex.split('iptables -t %s -I %s -j %s' % (table, from_chain, to_chain)))
return
if to_chain == from_chain_rules[0]['target']:
return
to_be_deleted = []
for i, rule in enumerate(from_chain_rules):
if to_chain == rule['target']:
to_be_deleted.append(i + 1)
for i in reversed(to_be_deleted):
shell.call(shlex.split('iptables -t %s -D %s %s' % (table, from_chain, i)))
shell.call(shlex.split('iptables -t %s -I %s -j %s' % (table, from_chain, to_chain)))
def execute_salt_state(hostname, username, password, state_name, master_name, machine_id=None):
with lock.FileLock(hostname):
ssh.execute('''ip=`env | grep SSH_CLIENT | cut -d '=' -f 2 | cut -d ' ' -f 1`; [ $ip == ::1 ] && ip=127.0.0.1; sed -i "/%s/d" /etc/hosts; sed -i "/$ip/d" /etc/hosts; echo "$ip %s" >> /etc/hosts''' % (master_name, master_name), hostname, username, password)
if not machine_id:
(retcode, machine_id, err) = ssh.execute('cat /sys/class/dmi/id/product_uuid', hostname, username, password, exception_if_error=False)
if not machine_id:
raise SaltError("Can't find machine-id on %s" % hostname)
machine_id = machine_id.strip()
if not wait_for_salt_minion_daemon(machine_id, 1, False):
ssh.execute('which salt-minion; [ $? -ne 0 ] && curl -L http://bootstrap.saltstack.org | sudo sh ;sed -i "^id/d" /etc/salt/minion; sed -i "^master/d" /etc/salt/minion; echo "id: %s" >>/etc/salt/minion; echo "master: %s" >> /etc/salt/minion; rm -f /etc/salt/pki/minion/minion_master.pub ; service salt-minion restart' % (machine_id, master_name), hostname, username, password, exception_if_error=False)
wait_for_salt_minion_daemon(machine_id)
print 'salt %s %s' % (machine_id, state_name)
output = shell.call('salt --out=json %s %s' % (machine_id, state_name))
if not is_salt_failed(output):
print '%s' % output
print "salt has deployed %s" % state_name
else:
raise SaltError('salt execution failure: %s' % output)
def execute_salt_state(hostname,
username,
password,
state_name,
master_name,
machine_id=None):
with lock.FileLock(hostname):
ssh.execute(
'''ip=`env | grep SSH_CLIENT | cut -d '=' -f 2 | cut -d ' ' -f 1`; [ $ip == ::1 ] && ip=127.0.0.1; sed -i "/%s/d" /etc/hosts; sed -i "/$ip/d" /etc/hosts; echo "$ip %s" >> /etc/hosts'''
% (master_name, master_name), hostname, username, password)
if not machine_id:
(retcode, machine_id,
err) = ssh.execute('cat /sys/class/dmi/id/product_uuid',
hostname,
username,
password,
exception_if_error=False)
if not machine_id:
raise SaltError("Can't find machine-id on %s" % hostname)
machine_id = machine_id.strip()
if not wait_for_salt_minion_daemon(machine_id, 1, False):
ssh.execute(
'which salt-minion; [ $? -ne 0 ] && curl -L http://bootstrap.saltstack.org | sudo sh ;sed -i "^id/d" /etc/salt/minion; sed -i "^master/d" /etc/salt/minion; echo "id: %s" >>/etc/salt/minion; echo "master: %s" >> /etc/salt/minion; rm -f /etc/salt/pki/minion/minion_master.pub ; service salt-minion restart'
% (machine_id, master_name),
hostname,
username,
password,
exception_if_error=False)
wait_for_salt_minion_daemon(machine_id)
print 'salt %s %s' % (machine_id, state_name)
output = shell.call('salt --out=json %s %s' % (machine_id, state_name))
if not is_salt_failed(output):
print '%s' % output
print "salt has deployed %s" % state_name
else:
raise SaltError('salt execution failure: %s' % output)
def ensure_first_target(table, from_chain, from_chain_rules, to_chain):
if not from_chain_rules:
shell.call(
shlex.split('iptables -t %s -I %s -j %s' %
(table, from_chain, to_chain)))
return
if to_chain == from_chain_rules[0]['target']:
return
to_be_deleted = []
for i, rule in enumerate(from_chain_rules):
if to_chain == rule['target']:
to_be_deleted.append(i + 1)
for i in reversed(to_be_deleted):
shell.call(
shlex.split('iptables -t %s -D %s %s' % (table, from_chain, i)))
shell.call(
shlex.split('iptables -t %s -I %s -j %s' %
(table, from_chain, to_chain)))
def __enter__(self):
drop_caches()
self.dev = "/dev/%s/%s-root" % (HOST, self.guest)
mapper_name = shell.eval("kpartx", "-l", self.dev).split()[0]
shell.call("kpartx", "-a", self.dev)
mapper = "/dev/mapper/%s" % mapper_name
# this is why bracketing functions and hanging lambdas are a good idea
try:
self.mount = tempfile.mkdtemp("-%s" % self.guest, 'vm-', '/mnt') # no trailing slash
try:
shell.call("mount", "--types", self.types, mapper, self.mount)
except:
os.rmdir(self.mount)
raise
except:
shell.call("kpartx", "-d", self.dev)
raise
return self.mount
def __enter__(self):
drop_caches()
self.dev = "/dev/%s/%s-root" % (HOST, self.guest)
mapper_name = shell.eval("kpartx", "-l", self.dev).split()[0]
shell.call("kpartx", "-a", self.dev)
mapper = "/dev/mapper/%s" % mapper_name
# this is why bracketing functions and hanging lambdas are a good idea
try:
self.mount = tempfile.mkdtemp("-%s" % self.guest, 'vm-', '/mnt') # no trailing slash
try:
shell.call("mount", "--types", self.types, mapper, self.mount)
except:
os.rmdir(self.mount)
raise
except:
shell.call("kpartx", "-d", self.dev)
raise
return self.mount
def test_speed(opts): t = utils.get_current_time() shell.call(opts['test']) return utils.get_elapsed_time(t)
def test_size(opts): shell.call(opts['test']) return os.path.getsize(opts['params'][0])
def clean_test(command):
if command:
if(VERBOSE):
print('Cleaning with command:',command)
shell.call(command)
def cleanup_local_log():
import shell
shell.call(
'''cd /var/log/zstack; tar --ignore-failed-read -zcf zstack-logs-`date +%y%m%d-%H%M%S`.tgz *.log.* *.log; find . -name "*.log"|while read file; do echo "" > $file; done'''
)
def run_playbook(filename, args):
ansiblecmd = ['ansible-playbook', filename] + args
shell.echo('Running', typeOf='info', lr=False)
shell.echo(shell.Color.text(' '.join(ansiblecmd), decoration='bold'))
shell.call(ansiblecmd)
def __init__(self, repo=os.getcwd()):
shell.call('cd {repo}'.format(repo = repo))
self.repo = repo
self.is_bare = self.check_is_bare()
self.remotes = self.get_remotes()
def pull():
call(['git', 'pull'])
def __exit__(self, _type, _value, _traceback):
shell.call("umount", self.mount)
os.rmdir(self.mount)
shell.call("kpartx", "-d", self.dev)
drop_caches()
def cleanup_local_log():
import shell
shell.call('''cd /var/log/zstack; tar --ignore-failed-read -zcf zstack-logs-`date +%y%m%d-%H%M%S`.tgz *.log.* *.log; find . -name "*.log"|while read file; do echo "" > $file; done''')
def flush_fq_chain():
for table in ('filter', 'nat'):
rules = dump_table(table)
for chain, chain_rules in rules.items():
if chain.startswith('fq_'):
shell.call(shlex.split('iptables -t %s --flush %s' % (table, chain)))
#!/usr/bin/env python
import exception
from shell import call, shell
try:
call('git')
except OSError:
err_msg = "is git installed? make sure it's in $PATH."
raise exception.MissingDependency(err_msg)
import os
import logging
from functools import wraps
class GitAPI(object):
@staticmethod
def clone(repo):
call(['git', 'clone', '{}'.format(repo)])
@staticmethod
def tags():
return filter(None, shell(['git', 'tag']).split('\n'))
@staticmethod
def checkout(repo, tag=''):
shell(['git', 'checkout', '{}'.format(tag)])
def clean():
call('rm -rf *', shell=True) # XXX: nix only
def init(self, options=''):
options = parse_options(options)
shell.call('git init {options}'.format(options = options))
def __exit__(self, _type, _value, _traceback):
shell.call("umount", self.mount)
os.rmdir(self.mount)
shell.call("kpartx", "-d", self.dev)
drop_caches()
def clone(repo):
call(['git', 'clone', '{}'.format(repo)])