class TemplateTest(unittest.TestCase):
def setUp(self):
account = Account('sab', '')
self.queue = Queue(verbose = 0, max_threads = 1)
self.logger = Logger()
self.queue.add_account(account)
def tearDown(self):
self.queue.destroy()
def testTemplates(self):
callback = bind(log_to(self.logger)(dummy_cb), self)
for test in os.listdir(test_dir):
pseudo = os.path.join(test_dir, test, 'pseudodev.py')
if os.path.exists(pseudo):
self.queue.run('pseudo://' + pseudo, callback)
else:
self.queue.run('ios://' + test, callback)
self.queue.shutdown()
# Unfortunately, unittest.TestCase does not fail if self.assert()
# was called from a subthread, so this is our workaround...
failed = self.logger.get_aborted_logs()
report = format(self.logger, show_successful = False)
self.assert_(not failed, report)
class TemplateTest(unittest.TestCase):
def setUp(self):
account = Account('sab', '')
self.queue = Queue(verbose=0, max_threads=1)
self.logger = Logger()
self.queue.add_account(account)
def tearDown(self):
self.queue.destroy()
def testTemplates(self):
callback = bind(log_to(self.logger)(dummy_cb), self)
for test in os.listdir(test_dir):
pseudo = os.path.join(test_dir, test, 'pseudodev.py')
if os.path.exists(pseudo):
self.queue.run('pseudo://' + pseudo, callback)
else:
self.queue.run('ios://' + test, callback)
self.queue.shutdown()
# Unfortunately, unittest.TestCase does not fail if self.assert()
# was called from a subthread, so this is our workaround...
failed = self.logger.get_aborted_logs()
report = format(self.logger, show_successful=False)
self.assert_(not failed, report)
def createQueue(self, logdir=None, **kwargs):
if self.queue:
self.queue.destroy()
self.out = self.manager.Log()
self.err = self.manager.Log()
self.queue = Queue(mode=self.mode,
stdout=self.out,
stderr=self.err,
**kwargs)
self.accm = self.queue.account_manager
if logdir is not None:
self.logger = FileLogger(logdir)
def run(users, hosts, func, **kwargs):
"""
Convenience function that creates an Exscript.Queue instance, adds
the given accounts, and calls Queue.run() with the given
hosts and function as an argument.
If you also want to pass arguments to the given function, you may use
util.decorator.bind() like this::
def my_callback(job, host, conn, my_arg, **kwargs):
print my_arg, kwargs.get('foo')
run(account,
host,
bind(my_callback, 'hello', foo = 'world'),
max_threads = 10)
@type users: Account|list[Account]
@param users: The account(s) to use for logging in.
@type hosts: Host|list[Host]
@param hosts: A list of Host objects.
@type func: function
@param func: The callback function.
@type kwargs: dict
@param kwargs: Passed to the Exscript.Queue constructor.
"""
queue = Queue(**kwargs)
queue.add_account(users)
queue.run(hosts, func)
queue.destroy()
def routerLogin():
# This function prompts the user to provide their login credentials and logs into each
# of the routers before calling the buildIndex function to extract relevant portions of
# the router config. As designed, this function actually has the capability to login to
# multiple routers simultaneously. I chose to not allow it to multi-thread given possibility
# of undesirable results from multiple threads writing to the same index file simultaneously
try:# Check for existence of routerFile; If exists, continue with program
with open(routerFile, "r"): pass
# Read hosts from specified file & remove duplicate entries, set protocol to SSH2
hosts = get_hosts_from_file(routerFile,default_protocol="ssh2",remove_duplicates=True)
if username == "": # If username is blank
print
account = read_login() # Prompt the user for login credentials
elif password == "": # If password is blank
print
account = read_login() # Prompt the user for login credentials
else: # Else use username/password from configFile
account = Account(name=username, password=b64decode(password))
# Minimal message from queue, 1 threads, redirect errors to null
queue = Queue(verbose=0, max_threads=1, stderr=(open(os.devnull, "w")))
queue.add_account(account) # Use supplied user credentials
print
stdout.write("--> Building index...") # Print without trailing newline
queue.run(hosts, buildIndex) # Create queue using provided hosts
queue.shutdown() # End all running threads and close queue
# If logFileDirectory does not exist, create it.
if not path.exists(logFileDirectory): makedirs(logFileDirectory)
# Define log filename based on date
logFilename = logFileDirectory+"VRFSearchAndBackup_"+date+".log"
# Check to see if logFilename currently exists. If it does, append an
# integer onto the end of the filename until logFilename no longer exists
incrementLogFilename = 1
while fileExist(logFilename):
logFilename = logFileDirectory+"VRFSearchAndBackup_"+date+"_"+str(incrementLogFilename)+".log"
incrementLogFilename = incrementLogFilename + 1
# Write log results to logFile
with open(logFilename, "w") as outputLogFile:
try:
outputLogFile.write(summarize(logger))
# Exception: router file was not able to be opened
except IOError:
print
print "--> An error occurred opening "+logFileDirectory+logFile+"."
# Exception: router file could not be opened
except IOError:
print
print "--> An error occurred opening "+routerFile+"."
def handle(self, *args, **options):
if options['all'] and options['device_id']:
print ("Error: --all and --deviceid detected. Use only one")
return
if options['all']:
devices = Device.objects.all()
if options['device_id']:
try:
devices = [ Device.objects.get(pk=int(options['device_id'])) ]
except:
raise CommandError('Device ID "%s" does not exist' % options['device_id'])
for device in devices:
found_credential = False
for credential in Credential.objects.all():
if credential.match_ip(device.access_ip):
found_credential = True
host = Host(device.protocol + device.access_ip)
account = (Account(credential.username,
credential.password))
if credential.enable_username and credential.enable_password:
account.set_authorization_password(credential.enable_password)
host.set_account(account)
host.set('device', device)
hosts.append(host)
break
if not found_credential:
device.backup_status = "No credentials found." % device
device.save()
queue = Queue(max_threads = 5, verbose = -1, stderr=open(os.devnull, 'w'))
queue.run(hosts, do_backup)
queue.shutdown()
for log in logger.get_logs():
device = Device.objects.get(access_ip=log.get_name())
if log.has_error():
device.backup_status = log.get_error(False)
else:
device.backup_status = 'ok'
device.backup_last_ran = now()
device.save()
def run(users, hosts, func, **kwargs):
"""
Convenience function that creates an Exscript.Queue instance, adds
the given accounts, and calls Queue.run() with the given
hosts and function as an argument.
If you also want to pass arguments to the given function, you may use
util.decorator.bind() like this::
def my_callback(job, host, conn, my_arg, **kwargs):
print my_arg, kwargs.get('foo')
run(account,
host,
bind(my_callback, 'hello', foo = 'world'),
max_threads = 10)
@type users: Account|list[Account]
@param users: The account(s) to use for logging in.
@type hosts: Host|list[Host]
@param hosts: A list of Host objects.
@type func: function
@param func: The callback function.
@type kwargs: dict
@param kwargs: Passed to the Exscript.Queue constructor.
"""
queue = Queue(**kwargs)
queue.add_account(users)
queue.run(hosts, func)
queue.destroy()
def _init_queue_from_name(self, name):
# Create the queue first.
element = self.cfgtree.find('queue[@name="%s"]' % name)
max_threads = element.find('max-threads').text
queue = Queue(verbose = 0, max_threads = max_threads)
# Assign account pools to the queue.
for pool_elem in element.iterfind('account-pool'):
pool = self._init_account_pool_from_name(pool_elem.text)
cond = pool_elem.get('for')
if cond is None:
queue.add_account_pool(pool)
continue
condition = compile(cond, 'config', 'eval')
def match_cb(host):
return eval(condition, host.get_dict())
queue.add_account_pool(pool, match_cb)
return queue
def createQueue(self, logdir=None, **kwargs):
if self.queue:
self.queue.destroy()
self.out = self.manager.Log()
self.err = self.manager.Log()
self.queue = Queue(mode=self.mode,
stdout=self.out,
stderr=self.err,
**kwargs)
self.accm = self.queue.account_manager
if logdir is not None:
self.logger = FileLogger(logdir)
def _init_queue_from_name(self, name):
# Create the queue first.
element = self.cfgtree.find('queue[@name="%s"]' % name)
max_threads = element.find('max-threads').text
queue = Queue(verbose=0, max_threads=max_threads)
# Assign account pools to the queue.
def match_cb(condition, host):
return eval(condition, host.get_dict())
for pool_elem in element.iterfind('account-pool'):
pname = pool_elem.text
pool = self._init_account_pool_from_name(pname)
cond = pool_elem.get('for')
if cond is None:
print 'Assigning default account pool "%s" to "%s"...' % (
pname, name)
queue.add_account_pool(pool)
continue
print 'Assigning account pool "%s" to "%s"...' % (pname, name)
condition = compile(cond, 'config', 'eval')
queue.add_account_pool(pool, partial(match_cb, condition))
return queue
def _init_queue_from_name(self, name):
# Create the queue first.
element = self.cfgtree.find('queue[@name="%s"]' % name)
max_threads = element.find('max-threads').text
queue = Queue(verbose = 0, max_threads = max_threads)
# Assign account pools to the queue.
def match_cb(condition, host):
return eval(condition, host.get_dict())
for pool_elem in element.iterfind('account-pool'):
pname = pool_elem.text
pool = self._init_account_pool_from_name(pname)
cond = pool_elem.get('for')
if cond is None:
print 'Assigning default account pool "%s" to "%s"...' % (pname, name)
queue.add_account_pool(pool)
continue
print 'Assigning account pool "%s" to "%s"...' % (pname, name)
condition = compile(cond, 'config', 'eval')
queue.add_account_pool(pool, partial(match_cb, condition))
return queue
def _init_queue_from_name(self, name):
# Create the queue first.
element = self.cfgtree.find('queue[@name="%s"]' % name)
max_threads = element.find('max-threads').text
queue = Queue(verbose=0, max_threads=max_threads)
# Assign account pools to the queue.
for pool_elem in element.iterfind('account-pool'):
pool = self._init_account_pool_from_name(pool_elem.text)
cond = pool_elem.get('for')
if cond is None:
queue.add_account_pool(pool)
continue
condition = compile(cond, 'config', 'eval')
def match_cb(host):
return eval(condition, host.get_dict())
queue.add_account_pool(pool, match_cb)
return queue
import time, Exscript.util.sigintcatcher
from Exscript import Queue, Account
from Exscript.util.decorator import bind
from TkExscript import QueueWindow
def do_something(conn, wait):
conn.connect()
conn.authenticate()
for i in range(100):
conn.execute('test%d' % i)
time.sleep(wait)
conn.close()
queue = Queue(max_threads = 4, verbose = 0)
queue.add_account(Account('test', 'test'))
window = QueueWindow(queue)
queue.run('dummy://dummy1', bind(do_something, .02))
queue.run('dummy://dummy2', bind(do_something, .2))
queue.run('dummy://dummy3', bind(do_something, .3))
window.mainloop()
queue.shutdown()
from Exscript import Queue, Logger
from Exscript.util.log import log_to
from Exscript.util.decorator import autologin
from Exscript.util.file import get_hosts_from_file, get_accounts_from_file
from Exscript.util.report import status, summarize
logger = Logger() # Logs everything to memory.
@log_to(logger)
@autologin()
def do_something(job, host, conn):
conn.execute('show ip int brie')
# Read input data.
accounts = get_accounts_from_file('accounts.cfg')
hosts = get_hosts_from_file('hostlist.txt')
# Run do_something on each of the hosts. The given accounts are used
# round-robin. "verbose=0" instructs the queue to not generate any
# output on stdout.
queue = Queue(verbose=5, max_threads=5)
queue.add_account(accounts) # Adds one or more accounts.
queue.run(hosts, do_something) # Asynchronously enqueues all hosts.
queue.shutdown() # Waits until all hosts are completed.
# Print a short report.
print status(logger)
print summarize(logger)
import gc
import re
from Exscript.protocols import connect
from Exscript.util.decorator import bind
from Exscript import Queue, Account, Host
objnames = ('count_calls',)
follow_modules = False
def count_calls(conn, thedata, **kwargs):
thedata['n_calls'] += 1
def foobar():
pass
queue = Queue()
data = {'n_calls': 0}
func = bind(count_calls, data)
task = queue.run(['t1', 't2', 't3', 't4', 't5', 't6'], func)
task.wait()
queue.shutdown()
queue.destroy()
del func
# Test memory consumption.
from meliae import scanner
gc.collect()
scanner.dump_all_objects("test.dump")
from meliae import loader
om = loader.load('test.dump')
f.write(description + "\t\t\t" + intf + "\t\t\t" + "Serial: " + serial + "\n")
f.close()
# Read input data.
#accounts = get_accounts_from_file('accounts.cfg')
accounts = read_login()
try:
hosts = get_hosts_from_file('hostlist.txt')
except:
print """
Devi creare un file che si chiama hostlist.txt e dentro
ci sono linee di questo tipo:
ssh://r.rm2.garr.net
"""
sys.exit(0)
# Run do_something on each of the hosts. The given accounts are used
# round-robin. "verbose = 0" instructs the queue to not generate any
# output on stdout.
queue = Queue(verbose = 3, max_threads = 5)
queue.add_account(accounts) # Adds one or more accounts.
queue.run(hosts, do_something) # Asynchronously enqueues all hosts.
queue.shutdown() # Waits until all hosts are completed.
# Print a short report.
print status(logger)
print summarize(logger)
class QueueTest(unittest.TestCase):
CORRELATE = Queue
mode = 'threading'
def createQueue(self, logdir=None, **kwargs):
if self.queue:
self.queue.destroy()
self.out = self.manager.Log()
self.err = self.manager.Log()
self.queue = Queue(mode=self.mode,
stdout=self.out,
stderr=self.err,
**kwargs)
self.accm = self.queue.account_manager
if logdir is not None:
self.logger = FileLogger(logdir)
def setUp(self):
self.tempdir = mkdtemp()
self.queue = None
self.logger = None
self.manager = LogManager()
self.manager.start()
self.createQueue(verbose=-1, logdir=self.tempdir)
def tearDown(self):
shutil.rmtree(self.tempdir)
try:
self.queue.destroy()
except:
pass # queue already destroyed
self.manager.shutdown()
def assertVerbosity(self, channel, expected):
data = channel.read()
if expected == 'no_tb':
self.assert_('error' in data, data)
self.assert_('Traceback' not in data)
elif expected == 'tb':
self.assert_('error' in data, data)
self.assert_('Traceback' in data)
elif expected == '':
self.assertEqual(data, '')
else:
msg = repr(expected) + ' not in ' + repr(data)
self.assert_(expected in data, msg)
def testConstructor(self):
self.assertEqual(1, self.queue.get_max_threads())
# Test all verbosity levels.
levels = (
(-1, 1, ('', ''), ('', ''), ('', 'tb')),
(-1, 2, ('', ''), ('', ''), ('', 'tb')),
(0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')),
(0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')),
(1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')),
(1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')),
(2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
)
for level, max_threads, with_simple, with_error, with_fatal in levels:
#print "S:", level, max_threads, with_simple, with_error, with_fatal
stdout, stderr = with_simple
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', say_hello)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
#print "E:", level, max_threads, with_simple, with_error, with_fatal
stdout, stderr = with_error
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
#print "F:", level, max_threads, with_simple, with_error, with_fatal
stdout, stderr = with_fatal
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', fatal_error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
def testCreatePipe(self):
account = Account('user', 'test')
self.accm.add_account(account)
pipe = self.queue._create_pipe()
pipe.send(('acquire-account', None))
response = pipe.recv()
expected = (account.__hash__(), account.get_name(),
account.get_password(),
account.get_authorization_password(), account.get_key())
self.assertEqual(response, expected)
pipe.send(('release-account', account.__hash__()))
response = pipe.recv()
self.assertEqual(response, 'ok')
pipe.close()
def testSetMaxThreads(self):
self.assertEqual(1, self.queue.get_max_threads())
self.queue.set_max_threads(2)
self.assertEqual(2, self.queue.get_max_threads())
def testGetMaxThreads(self):
pass # Already tested in testSetMaxThreads().
def testGetProgress(self):
self.assertEqual(0.0, self.queue.get_progress())
self.testIsCompleted()
self.assertEqual(100.0, self.queue.get_progress())
def testAddAccount(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def testAddAccountPool(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def match_cb(data, host):
data['match-called'].value = True
return True
def start_cb(data, job, host, conn):
account = conn.account_factory(None)
data['start-called'].value = True
data['account-hash'].value = account.__hash__()
account.release()
# Replace the default pool.
pool1 = AccountPool()
self.queue.add_account_pool(pool1)
self.assertEqual(self.accm.default_pool, pool1)
# Add another pool, making sure that it does not replace
# the default pool.
pool2 = AccountPool()
account2 = Account('user', 'test')
pool2.add_account(account2)
match_called = Value(ctypes.c_bool, False)
start_called = Value(ctypes.c_bool, False)
account_hash = Value(ctypes.c_long, 0)
data = {
'match-called': match_called,
'start-called': start_called,
'account-hash': account_hash
}
self.queue.add_account_pool(pool2, partial(match_cb, data))
self.assertEqual(self.accm.default_pool, pool1)
# Make sure that pool2 is chosen (because the match function
# returns True).
self.queue.run('dummy://dummy', partial(start_cb, data))
self.queue.shutdown()
data = dict((k, v.value) for (k, v) in data.iteritems())
self.assertEqual(
data, {
'match-called': True,
'start-called': True,
'account-hash': account2.__hash__()
})
def startTask(self):
self.testAddAccount()
hosts = ['dummy://dummy1', 'dummy://dummy2']
task = self.queue.run(hosts, log_to(self.logger)(do_nothing))
self.assert_(task is not None)
return task
def testIsCompleted(self):
self.assert_(self.queue.is_completed())
task = self.startTask()
self.failIf(self.queue.is_completed())
task.wait()
self.assert_(task.is_completed())
self.assert_(self.queue.is_completed())
def testJoin(self):
task = self.startTask()
self.queue.join()
self.assert_(task.is_completed())
self.assert_(self.queue.is_completed())
def testShutdown(self):
task = self.startTask() # this also adds an account
self.queue.shutdown()
self.assert_(task.is_completed())
self.assert_(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 1)
def testDestroy(self):
task = self.startTask() # this also adds an account
self.queue.destroy()
self.assert_(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testReset(self):
self.testAddAccount()
self.queue.reset()
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testExceptionCallback(self):
self.exc = {}
def my_exc_cb(jobname, exc_info):
self.exc[jobname] = exc_info
self.createQueue(exc_cb=my_exc_cb)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assert_("mytest" in self.exc)
self.assert_(isinstance(self.exc["mytest"][1], FailException))
def testRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
self.queue.run(hosts, func)
self.queue.run('dummy://dummy3', func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
self.queue.run('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 4)
def testRunOrIgnore(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.run_or_ignore(hosts, func)
self.queue.run_or_ignore('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.run_or_ignore('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testPriorityRun(self):
def write(data, value, *args):
data.value = value
data = Value('i', 0)
self.queue.workqueue.pause()
self.queue.enqueue(partial(write, data, 1))
self.queue.priority_run('dummy://dummy', partial(write, data, 2))
self.queue.workqueue.unpause()
self.queue.destroy()
# The 'dummy' job should run first, so the value must
# be overwritten by the other process.
self.assertEqual(data.value, 1)
def testPriorityRunOrRaise(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.priority_run_or_raise(hosts, func)
self.queue.priority_run_or_raise('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.priority_run_or_raise('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testForceRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
# By setting max_threads to 0 we ensure that the 'force' part is
# actually tested; the thread should run regardless.
self.queue.set_max_threads(0)
self.queue.force_run(hosts, func)
self.queue.destroy()
self.assertEqual(data.value, 2)
def testEnqueue(self):
data = Value('i', 0)
func = bind(count_calls, data, testarg=1)
self.queue.enqueue(func)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
func = bind(count_and_fail, data, testarg=1)
self.queue.enqueue(func, attempts=7)
self.queue.destroy()
self.assertEqual(data.value, 10)
#FIXME: Not a method test; this should probably be elsewhere.
def testLogging(self):
task = self.startTask()
while not task.is_completed():
time.sleep(.1)
# The following function is not synchronous with the above, so add
# a timeout to avoid races.
time.sleep(.1)
self.assert_(self.queue.is_completed())
logfiles = os.listdir(self.tempdir)
self.assertEqual(2, len(logfiles))
self.assert_('dummy1.log' in logfiles)
self.assert_('dummy2.log' in logfiles)
for file in logfiles:
content = open(os.path.join(self.tempdir, file)).read()
elif task == 'NO_DIVERT' and valid_ipv4(ipaddr):
conn.execute('term len 0')
conn.execute('conf t')
conn.execute('ip access-list extended NORMAL')
conn.execute('no permit ip any any')
conn.execute(f"no deny ip {ipaddr} {wildcard}")
conn.execute(f"permit ip any any")
conn.execute('ip access-list extended DIVERT')
conn.execute('no deny ip any any')
conn.execute(f"no permit ip {ipaddr} {wildcard}")
conn.execute('deny ip any any')
conn.execute('end')
conn.execute('clear ip bgp * soft out')
else:
RED = '\033[31m'
print(RED + "\nWARNING: \tPLEASE INPUT A CORRECT TASK AND IP ADDRESS!")
print(f"NOTE:\t\tNO CHANGES HAS BEEN MADE!")
print(Style.RESET_ALL)
# Read input data.
accounts = get_accounts_from_file('/root/crash_course/project/accounts.cfg')
hosts = get_hosts_from_file('/root/crash_course/project/hostlist.txt',default_protocol='ssh2')
queue = Queue(max_threads=5)
queue.add_account(accounts)
queue.run(hosts, do_something,)
queue.shutdown()
def setUp(self):
account = Account('sab', '')
self.queue = Queue(verbose=0, max_threads=1)
self.logger = Logger()
self.queue.add_account(account)
import time, Exscript.util.sigintcatcher
from Exscript import Queue, Account
from Exscript.util.decorator import bind
from TkExscript import QueueWindow
def do_something(conn, wait):
conn.connect()
conn.authenticate()
for i in range(100):
conn.execute('test%d' % i)
time.sleep(wait)
conn.close()
queue = Queue(max_threads=4, verbose=0)
queue.add_account(Account('test', 'test'))
window = QueueWindow(queue)
queue.run('dummy://dummy1', bind(do_something, .02))
queue.run('dummy://dummy2', bind(do_something, .2))
queue.run('dummy://dummy3', bind(do_something, .3))
window.mainloop()
queue.shutdown()
account = read_login() # Prompt the user for login credentials
elif password == "": # If password is blank
print
account = read_login() # Prompt the user for login credentials
else: # Else use username/password from configFile
account = Account(name=username, password=b64decode(password))
# Read hosts from specified file & remove duplicate entries, set protocol to SSH2
hosts = get_hosts_from_file(routerFile, default_protocol="ssh2", remove_duplicates=True)
print
# Verbose & # threads taken from configFile, redirect errors to null
queue = Queue(verbose=int(verboseOutput), max_threads=int(maxThreads), stderr=(open(os.devnull, "w")))
queue.add_account(account) # Use supplied user credentials
queue.run(hosts, downloadRouterConfig) # Create queue using provided hosts
queue.shutdown() # End all running threads and close queue
print status(logger) # Print current % status of operation to screen
# If logFileDirectory does not exist, create it.
if not path.exists(logFileDirectory): makedirs(logFileDirectory)
# Define log filename based on date
logFilename = logFileDirectory+"DownloadRouterConfig_"+date+".log"
# Check to see if logFilename currently exists. If it does, append an
# integer onto the end of the filename until logFilename no longer exists
incrementLogFilename = 1
class QueueTest(unittest.TestCase):
CORRELATE = Queue
mode = 'threading'
def createQueue(self, logdir=None, **kwargs):
if self.queue:
self.queue.destroy()
self.out = self.manager.Log()
self.err = self.manager.Log()
self.queue = Queue(mode=self.mode,
stdout=self.out,
stderr=self.err,
**kwargs)
self.accm = self.queue.account_manager
if logdir is not None:
self.logger = FileLogger(logdir)
def setUp(self):
self.tempdir = mkdtemp()
self.queue = None
self.logger = None
self.manager = LogManager()
self.manager.start()
self.createQueue(verbose=-1, logdir=self.tempdir)
def tearDown(self):
shutil.rmtree(self.tempdir)
try:
self.queue.destroy()
except:
pass # queue already destroyed
self.manager.shutdown()
def assertVerbosity(self, channel, expected):
data = channel.read()
if expected == 'no_tb':
self.assertTrue('error' in data, data)
self.assertNotIn('Traceback', data)
elif expected == 'tb':
self.assertTrue('error' in data, data)
self.assertIn('Traceback', data)
elif expected == '':
self.assertEqual(data, '')
else:
msg = repr(expected) + ' not in ' + repr(data)
self.assertTrue(expected in data, msg)
def testConstructor(self):
self.assertEqual(1, self.queue.get_max_threads())
# Test all verbosity levels.
levels = (
(-1, 1, ('', ''), ('', ''), ('', 'tb')),
(-1, 2, ('', ''), ('', ''), ('', 'tb')),
(0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')),
(0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')),
(1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')),
(1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')),
(2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
)
for level, max_threads, with_simple, with_error, with_fatal in levels:
# print("S:", level, max_threads, with_simple, with_error,)
# with_fatal
stdout, stderr = with_simple
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', say_hello)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
# print("E:", level, max_threads, with_simple, with_error,)
# with_fatal
stdout, stderr = with_error
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
# print("F:", level, max_threads, with_simple, with_error,)
# with_fatal
stdout, stderr = with_fatal
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', fatal_error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
def testCreatePipe(self):
account = Account('user', 'test')
self.accm.add_account(account)
pipe = self.queue._create_pipe()
pipe.send(('acquire-account', None))
response = pipe.recv()
expected = (account.__hash__(),
account.get_name(),
account.get_password(),
account.get_authorization_password(),
account.get_key())
self.assertEqual(response, expected)
pipe.send(('release-account', account.__hash__()))
response = pipe.recv()
self.assertEqual(response, 'ok')
pipe.close()
def testSetMaxThreads(self):
self.assertEqual(1, self.queue.get_max_threads())
self.queue.set_max_threads(2)
self.assertEqual(2, self.queue.get_max_threads())
def testGetMaxThreads(self):
pass # Already tested in testSetMaxThreads().
def testGetProgress(self):
self.assertEqual(0.0, self.queue.get_progress())
self.testIsCompleted()
self.assertEqual(100.0, self.queue.get_progress())
def testAddAccount(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def testAddAccountPool(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def match_cb(data, host):
data['match-called'].value = True
return True
def start_cb(data, job, host, conn):
account = conn.account_factory(None)
data['start-called'].value = True
data['account-hash'].value = account.__hash__()
account.release()
# Replace the default pool.
pool1 = AccountPool()
self.queue.add_account_pool(pool1)
self.assertEqual(self.accm.default_pool, pool1)
# Add another pool, making sure that it does not replace
# the default pool.
pool2 = AccountPool()
account2 = Account('user', 'test')
pool2.add_account(account2)
match_called = Value(ctypes.c_bool, False)
start_called = Value(ctypes.c_bool, False)
account_hash = Value(ctypes.c_long, 0)
data = {'match-called': match_called,
'start-called': start_called,
'account-hash': account_hash}
self.queue.add_account_pool(pool2, partial(match_cb, data))
self.assertEqual(self.accm.default_pool, pool1)
# Make sure that pool2 is chosen (because the match function
# returns True).
self.queue.run('dummy://dummy', partial(start_cb, data))
self.queue.shutdown()
data = dict((k, v.value) for (k, v) in list(data.items()))
self.assertEqual(data, {'match-called': True,
'start-called': True,
'account-hash': account2.__hash__()})
def startTask(self):
self.testAddAccount()
hosts = ['dummy://dummy1', 'dummy://dummy2']
task = self.queue.run(hosts, log_to(self.logger)(do_nothing))
self.assertTrue(task is not None)
return task
def testIsCompleted(self):
self.assertTrue(self.queue.is_completed())
task = self.startTask()
self.assertFalse(self.queue.is_completed())
task.wait()
self.assertTrue(task.is_completed())
self.assertTrue(self.queue.is_completed())
def testJoin(self):
task = self.startTask()
self.queue.join()
self.assertTrue(task.is_completed())
self.assertTrue(self.queue.is_completed())
def testShutdown(self):
task = self.startTask() # this also adds an account
self.queue.shutdown()
self.assertTrue(task.is_completed())
self.assertTrue(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 1)
def testDestroy(self):
task = self.startTask() # this also adds an account
self.queue.destroy()
self.assertTrue(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testReset(self):
self.testAddAccount()
self.queue.reset()
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testExceptionCallback(self):
self.exc = {}
def my_exc_cb(jobname, exc_info):
self.exc[jobname] = exc_info
self.createQueue(exc_cb=my_exc_cb)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assertIn("mytest", self.exc)
self.assertIsInstance(self.exc["mytest"][1], FailException)
def testRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
self.queue.run(hosts, func)
self.queue.run('dummy://dummy3', func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
self.queue.run('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 4)
def testRunOrIgnore(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.run_or_ignore(hosts, func)
self.queue.run_or_ignore('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.run_or_ignore('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testPriorityRun(self):
def write(data, value, *args):
data.value = value
data = Value('i', 0)
self.queue.workqueue.pause()
self.queue.enqueue(partial(write, data, 1))
self.queue.priority_run('dummy://dummy', partial(write, data, 2))
self.queue.workqueue.unpause()
self.queue.destroy()
# The 'dummy' job should run first, so the value must
# be overwritten by the other process.
self.assertEqual(data.value, 1)
def testPriorityRunOrRaise(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.priority_run_or_raise(hosts, func)
self.queue.priority_run_or_raise('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.priority_run_or_raise('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testForceRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
# By setting max_threads to 0 we ensure that the 'force' part is
# actually tested; the thread should run regardless.
self.queue.set_max_threads(0)
self.queue.force_run(hosts, func)
self.queue.destroy()
self.assertEqual(data.value, 2)
def testEnqueue(self):
data = Value('i', 0)
func = bind(count_calls, data, testarg=1)
self.queue.enqueue(func)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
func = bind(count_and_fail, data, testarg=1)
self.queue.enqueue(func, attempts=7)
self.queue.destroy()
self.assertEqual(data.value, 10)
# FIXME: Not a method test; this should probably be elsewhere.
def testLogging(self):
task = self.startTask()
while not task.is_completed():
time.sleep(.1)
# The following function is not synchronous with the above, so add
# a timeout to avoid races.
time.sleep(.1)
self.assertTrue(self.queue.is_completed())
logfiles = os.listdir(self.tempdir)
self.assertEqual(2, len(logfiles))
self.assertIn('dummy1.log', logfiles)
self.assertIn('dummy2.log', logfiles)
for file in logfiles:
with open(os.path.join(self.tempdir, file)) as fp:
content = fp.read()
def setUp(self):
account = Account('sab', '')
self.queue = Queue(verbose = 0, max_threads = 1)
self.logger = Logger()
self.queue.add_account(account)
from Exscript.util.decorator import bind
from Exscript import Queue, Account, Host
objnames = ('count_calls', )
follow_modules = False
def count_calls(conn, thedata, **kwargs):
thedata['n_calls'] += 1
def foobar():
pass
queue = Queue()
data = {'n_calls': 0}
func = bind(count_calls, data)
task = queue.run(['t1', 't2', 't3', 't4', 't5', 't6'], func)
task.wait()
queue.shutdown()
queue.destroy()
del func
# Test memory consumption.
from meliae import scanner
gc.collect()
scanner.dump_all_objects("test.dump")
from meliae import loader
om = loader.load('test.dump')
