def convertcurly(self):
if "{" in self.nodes:
self.nodes = self.nodes.replace("{", "[")
self.nodes = self.nodes.replace("}", "]")
self.nodes = self.nodes.replace("..", "-")
return hostlists.expand(self.nodes)
def converttonewline(hostlist):
fw=open("/tmp/hl-l","w")
hostlist=args.expand2line
xl=[]
xl=(hostlists.expand(hostlist))
fw.write('\n'.join(xl))
print type('\n'.join(xl))
def converttocoma(hostlist):
fw=open("/tmp/hl-c","w")
hostlist=args.expand2comma
xl=[]
xl=(hostlists.expand(hostlist))
fw.write(', '.join(xl))
fw.close()
def converttonewline(hostlist):
fw = open("/tmp/hl-l", "w")
hostlist = args.expand2line
xl = []
xl = (hostlists.expand(hostlist))
fw.write('\n'.join(xl))
print type('\n'.join(xl))
def converttocoma(hostlist):
fw = open("/tmp/hl-c", "w")
hostlist = args.expand2comma
xl = []
xl = (hostlists.expand(hostlist))
fw.write(', '.join(xl))
fw.close()
def test_expand_file(self):
with open('test_expand_file.hostlist', 'w') as fh:
fh.write('foo[1-2]\n')
result = hostlists.expand(['file:test_expand_file.hostlist'])
expected_result = ['foo1', 'foo2']
os.remove('test_expand_file.hostlist')
result.sort()
self.assertListEqual(result, expected_result)
def test_expand_file(self):
with open('test_expand_file.hostlist', 'w') as fh:
fh.write('foo[1-2]\n')
result = hostlists.expand(['file:test_expand_file.hostlist'])
expected_result = ['foo1', 'foo2']
os.remove('test_expand_file.hostlist')
result.sort()
self.assertListEqual(result, expected_result)
def test_expand(self):
"""
Expand a list of lists and set operators into a final host lists
>>> hostlists.expand(['foo[01-10]','-','foo[04-06]'])
['foo09', 'foo08', 'foo07', 'foo02', 'foo01', 'foo03', 'foo10']
>>>
"""
result = hostlists.expand(['foo[01-10]', '-', 'foo[04-06]'])
expected_result = [
'foo09', 'foo08', 'foo07', 'foo02', 'foo01', 'foo03', 'foo10']
result.sort()
expected_result.sort()
self.assertLessEqual(result, expected_result)
def test_expand(self):
"""
Expand a list of lists and set operators into a final host lists
>>> hostlists.expand(['foo[01-10]','-','foo[04-06]'])
['foo09', 'foo08', 'foo07', 'foo02', 'foo01', 'foo03', 'foo10']
>>>
"""
result = hostlists.expand(['foo[01-10]', '-', 'foo[04-06]'])
expected_result = [
'foo09', 'foo08', 'foo07', 'foo02', 'foo01', 'foo03', 'foo10'
]
result.sort()
expected_result.sort()
self.assertLessEqual(result, expected_result)
def pingtest(h):
success = list()
fail = list()
#with settings(warn_only=True):
#with settings(hide('stderr', 'warnings','stdout'),warn_only=True):
with settings(hide('everything'),warn_only=True):
nodes = hostlists.expand(h)
for each in nodes:
result = local("ping -c1 "+each)
if result.return_code == 0:
success.append(each)
else:
fail.append(each)
logging.error(fail)
logging.error(success)
logger.addHandler(mail_handler)
print "success:",len(success),success,"failed:",len(fail),fail
def test_expand_invalid_plugin(self):
with self.assertRaises(hostlists.HostListsError):
hostlists.expand(['boozle:bar'])
def expand(self):
return hostlists.expand(self.hl)
def __init__(self):
result = hostlists.expand(self.range_list)
self.assertIsInstance(result, list)
self.assertListEqual(result, self.expected_result)
def run(host_range, command, username=None, password=None, sudo=False,
script=None, timeout=None, sort=False, jobs=0, output_callback=None,
parms=None, shuffle=False, chunksize=None, exit_on_error=False):
"""
Run a command on a hostlists host_range of hosts
:param host_range:
:param command:
:param username:
:param password:
:param sudo:
:param script:
:param timeout:
:param sort:
:param jobs:
:param output_callback:
:param parms:
:param shuffle:
:param chunksize:
:param exit_on_error: Exit as soon as one result comes back with a non 0
return code.
>>> res=run(host_range='localhost',command="echo ok")
>>> print(res[0].dump())
localhost ok 0 0 {'failures': [], 'total_host_count': 1,
'completed_host_count': 1}
"""
if not output_callback:
output_callback = [callback.summarize_failures]
utility.status_info(output_callback, 'Looking up hosts')
# Expand the host range if we were passed a string host list
if 'basestring' not in dir(__builtins__):
# basestring is not in python3.x
basestring = str
if isinstance(host_range, basestring):
hosts = hostlists.expand(hostlists.range_split(host_range))
else:
hosts = host_range
if shuffle:
random.shuffle(hosts)
utility.status_clear()
results = ssh_results()
if parms:
results.parm = parms
else:
results.parm = {}
if sudo and not password:
for host in hosts:
result = ssh_result()
result.host = host
result.err = 'Sudo password required'
result.retcode = defaults.RUN_FAIL_NOPASSWORD
results.append(result)
results.parm['total_host_count'] = len(hosts)
results.parm['completed_host_count'] = 0
results.parm['failures'] = hosts
return results
if jobs < 1:
jobs = 1
if jobs > defaults.JOB_MAX:
jobs = defaults.JOB_MAX
# Set up our ssh client
#status_info(output_callback,'Setting up the SSH client')
client = fastSSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# load_system_host_keys slows things way down
#client.load_system_host_keys()
results.parm['total_host_count'] = len(hosts)
results.parm['completed_host_count'] = 0
utility.status_clear()
utility.status_info(output_callback, 'Spawning processes')
if jobs > len(hosts):
jobs = len(hosts)
pool = multiprocessing.Pool(processes=jobs, initializer=init_worker)
if not chunksize:
if jobs == 1 or jobs >= len(hosts):
chunksize = 1
else:
chunksize = int(len(hosts) / jobs) - 1
if chunksize < 1:
chunksize = 1
if chunksize > 10:
chunksize = 10
results.parm['chunksize'] = chunksize
if sort:
map_command = pool.imap
else:
map_command = pool.imap_unordered
if isinstance(output_callback, list) and \
callback.status_count in output_callback:
callback.status_count(ssh_result(parm=results.parm))
# Create a process pool and pass the parameters to it
utility.status_clear()
utility.status_info(
output_callback, 'Sending %d commands to each process' % chunksize)
if callback.status_count in output_callback:
callback.status_count(ssh_result(parm=results.parm))
try:
for result in map_command(
run_command,
[
(
host, command, username, password, sudo, script, timeout,
results.parm, client
) for host in hosts
],
chunksize
):
results.parm['completed_host_count'] += 1
result.parm = results.parm
if isinstance(output_callback, list):
for cb in output_callback:
result = cb(result)
else:
# noinspection PyCallingNonCallable
result = output_callback(result)
results.parm = result.parm
results.append(result)
if exit_on_error and result.retcode != 0:
break
pool.close()
except KeyboardInterrupt:
print('ctrl-c pressed')
pool.terminate()
#except Exception as e:
# print 'unknown error encountered',Exception,e
# pass
pool.terminate()
if isinstance(output_callback, list) and \
callback.status_count in output_callback:
utility.status_clear()
return results
def test_expand_dnsip(self):
result = hostlists.expand(['dnsip:yahoo.com'])
self.assertGreater(len(result), 0)
def run(host_range,
command,
username=None,
password=None,
sudo=False,
script=None,
timeout=None,
sort=False,
jobs=0,
output_callback=None,
parms=None,
shuffle=False,
chunksize=None,
exit_on_error=False):
"""
Run a command on a hostlists host_range of hosts
:param host_range:
:param command:
:param username:
:param password:
:param sudo:
:param script:
:param timeout:
:param sort:
:param jobs:
:param output_callback:
:param parms:
:param shuffle:
:param chunksize:
:param exit_on_error: Exit as soon as one result comes back with a non 0
return code.
>>> res=run(host_range='localhost',command="echo ok")
>>> print(res[0].dump())
localhost ok 0 0 {'failures': [], 'total_host_count': 1,
'completed_host_count': 1}
"""
if not output_callback:
output_callback = [callback.summarize_failures]
utility.status_info(output_callback, 'Looking up hosts')
# Expand the host range if we were passed a string host list
if 'basestring' not in dir(__builtins__):
# basestring is not in python3.x
basestring = str
if isinstance(host_range, basestring):
hosts = hostlists.expand(hostlists.range_split(host_range))
else:
hosts = host_range
if shuffle:
random.shuffle(hosts)
utility.status_clear()
results = ssh_results()
if parms:
results.parm = parms
else:
results.parm = {}
if sudo and not password:
for host in hosts:
result = ssh_result()
result.host = host
result.err = 'Sudo password required'
result.retcode = defaults.RUN_FAIL_NOPASSWORD
results.append(result)
results.parm['total_host_count'] = len(hosts)
results.parm['completed_host_count'] = 0
results.parm['failures'] = hosts
return results
if jobs < 1:
jobs = 1
if jobs > defaults.JOB_MAX:
jobs = defaults.JOB_MAX
# Set up our ssh client
#status_info(output_callback,'Setting up the SSH client')
client = fastSSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# load_system_host_keys slows things way down
#client.load_system_host_keys()
results.parm['total_host_count'] = len(hosts)
results.parm['completed_host_count'] = 0
utility.status_clear()
utility.status_info(output_callback, 'Spawning processes')
if jobs > len(hosts):
jobs = len(hosts)
pool = multiprocessing.Pool(processes=jobs, initializer=init_worker)
if not chunksize:
if jobs == 1 or jobs >= len(hosts):
chunksize = 1
else:
chunksize = int(len(hosts) / jobs) - 1
if chunksize < 1:
chunksize = 1
if chunksize > 10:
chunksize = 10
results.parm['chunksize'] = chunksize
if sort:
map_command = pool.imap
else:
map_command = pool.imap_unordered
if isinstance(output_callback, list) and \
callback.status_count in output_callback:
callback.status_count(ssh_result(parm=results.parm))
# Create a process pool and pass the parameters to it
utility.status_clear()
utility.status_info(output_callback,
'Sending %d commands to each process' % chunksize)
if callback.status_count in output_callback:
callback.status_count(ssh_result(parm=results.parm))
try:
for result in map_command(run_command,
[(host, command, username, password, sudo,
script, timeout, results.parm, client)
for host in hosts], chunksize):
results.parm['completed_host_count'] += 1
result.parm = results.parm
if isinstance(output_callback, list):
for cb in output_callback:
result = cb(result)
else:
# noinspection PyCallingNonCallable
result = output_callback(result)
results.parm = result.parm
results.append(result)
if exit_on_error and result.retcode != 0:
break
pool.close()
except KeyboardInterrupt:
print('ctrl-c pressed')
pool.terminate()
#except Exception as e:
# print 'unknown error encountered',Exception,e
# pass
pool.terminate()
if isinstance(output_callback, list) and \
callback.status_count in output_callback:
utility.status_clear()
return results
def test_expand_invalid_plugin(self):
with self.assertRaises(hostlists.HostListsError):
hostlists.expand(['boozle:bar'])
def converttonewline(self):
hl = []
hl = hostlists.expand(self.hosts)
print '\n'.join(hl)
def converttocomma(self): hl=[] hl=hostlists.expand(self.hosts) print ', '.join(hl)
def __init__(self):
result = hostlists.expand(self.range_list)
self.assertIsInstance(result, list)
self.assertListEqual(result, self.expected_result)
def test_expand_dnsip(self):
result = hostlists.expand(['dnsip:yahoo.com'])
self.assertGreater(len(result), 0)
def converttonewline(self): hl=[] hl=hostlists.expand(self.hosts) print '\n'.join(hl)
def converttocomma(self):
hl = []
hl = hostlists.expand(self.hosts)
print ', '.join(hl)
