class TemplateTest(unittest.TestCase):
def setUp(self):
account = Account('sab', '')
self.queue = Queue(verbose=0, max_threads=1)
self.logger = Logger()
self.queue.add_account(account)
def tearDown(self):
self.queue.destroy()
def testTemplates(self):
callback = bind(log_to(self.logger)(dummy_cb), self)
for test in os.listdir(test_dir):
pseudo = os.path.join(test_dir, test, 'pseudodev.py')
if os.path.exists(pseudo):
self.queue.run('pseudo://' + pseudo, callback)
else:
self.queue.run('ios://' + test, callback)
self.queue.shutdown()
# Unfortunately, unittest.TestCase does not fail if self.assert()
# was called from a subthread, so this is our workaround...
failed = self.logger.get_aborted_logs()
report = format(self.logger, show_successful=False)
self.assert_(not failed, report)
class TemplateTest(unittest.TestCase):
def setUp(self):
account = Account('sab', '')
self.queue = Queue(verbose = 0, max_threads = 1)
self.logger = Logger()
self.queue.add_account(account)
def tearDown(self):
self.queue.destroy()
def testTemplates(self):
callback = bind(log_to(self.logger)(dummy_cb), self)
for test in os.listdir(test_dir):
pseudo = os.path.join(test_dir, test, 'pseudodev.py')
if os.path.exists(pseudo):
self.queue.run('pseudo://' + pseudo, callback)
else:
self.queue.run('ios://' + test, callback)
self.queue.shutdown()
# Unfortunately, unittest.TestCase does not fail if self.assert()
# was called from a subthread, so this is our workaround...
failed = self.logger.get_aborted_logs()
report = format(self.logger, show_successful = False)
self.assert_(not failed, report)
def routerLogin():
# This function prompts the user to provide their login credentials and logs into each
# of the routers before calling the buildIndex function to extract relevant portions of
# the router config. As designed, this function actually has the capability to login to
# multiple routers simultaneously. I chose to not allow it to multi-thread given possibility
# of undesirable results from multiple threads writing to the same index file simultaneously
try:# Check for existence of routerFile; If exists, continue with program
with open(routerFile, "r"): pass
# Read hosts from specified file & remove duplicate entries, set protocol to SSH2
hosts = get_hosts_from_file(routerFile,default_protocol="ssh2",remove_duplicates=True)
if username == "": # If username is blank
print
account = read_login() # Prompt the user for login credentials
elif password == "": # If password is blank
print
account = read_login() # Prompt the user for login credentials
else: # Else use username/password from configFile
account = Account(name=username, password=b64decode(password))
# Minimal message from queue, 1 threads, redirect errors to null
queue = Queue(verbose=0, max_threads=1, stderr=(open(os.devnull, "w")))
queue.add_account(account) # Use supplied user credentials
print
stdout.write("--> Building index...") # Print without trailing newline
queue.run(hosts, buildIndex) # Create queue using provided hosts
queue.shutdown() # End all running threads and close queue
# If logFileDirectory does not exist, create it.
if not path.exists(logFileDirectory): makedirs(logFileDirectory)
# Define log filename based on date
logFilename = logFileDirectory+"VRFSearchAndBackup_"+date+".log"
# Check to see if logFilename currently exists. If it does, append an
# integer onto the end of the filename until logFilename no longer exists
incrementLogFilename = 1
while fileExist(logFilename):
logFilename = logFileDirectory+"VRFSearchAndBackup_"+date+"_"+str(incrementLogFilename)+".log"
incrementLogFilename = incrementLogFilename + 1
# Write log results to logFile
with open(logFilename, "w") as outputLogFile:
try:
outputLogFile.write(summarize(logger))
# Exception: router file was not able to be opened
except IOError:
print
print "--> An error occurred opening "+logFileDirectory+logFile+"."
# Exception: router file could not be opened
except IOError:
print
print "--> An error occurred opening "+routerFile+"."
def handle(self, *args, **options):
if options['all'] and options['device_id']:
print ("Error: --all and --deviceid detected. Use only one")
return
if options['all']:
devices = Device.objects.all()
if options['device_id']:
try:
devices = [ Device.objects.get(pk=int(options['device_id'])) ]
except:
raise CommandError('Device ID "%s" does not exist' % options['device_id'])
for device in devices:
found_credential = False
for credential in Credential.objects.all():
if credential.match_ip(device.access_ip):
found_credential = True
host = Host(device.protocol + device.access_ip)
account = (Account(credential.username,
credential.password))
if credential.enable_username and credential.enable_password:
account.set_authorization_password(credential.enable_password)
host.set_account(account)
host.set('device', device)
hosts.append(host)
break
if not found_credential:
device.backup_status = "No credentials found." % device
device.save()
queue = Queue(max_threads = 5, verbose = -1, stderr=open(os.devnull, 'w'))
queue.run(hosts, do_backup)
queue.shutdown()
for log in logger.get_logs():
device = Device.objects.get(access_ip=log.get_name())
if log.has_error():
device.backup_status = log.get_error(False)
else:
device.backup_status = 'ok'
device.backup_last_ran = now()
device.save()
objnames = ('count_calls',)
follow_modules = False
def count_calls(conn, thedata, **kwargs):
thedata['n_calls'] += 1
def foobar():
pass
queue = Queue()
data = {'n_calls': 0}
func = bind(count_calls, data)
task = queue.run(['t1', 't2', 't3', 't4', 't5', 't6'], func)
task.wait()
queue.shutdown()
queue.destroy()
del func
# Test memory consumption.
from meliae import scanner
gc.collect()
scanner.dump_all_objects("test.dump")
from meliae import loader
om = loader.load('test.dump')
om.remove_expensive_references()
om.collapse_instance_dicts()
om.compute_referrers()
om.compute_total_size()
#print om.summarize()
class QueueTest(unittest.TestCase):
CORRELATE = Queue
mode = 'threading'
def createQueue(self, logdir=None, **kwargs):
if self.queue:
self.queue.destroy()
self.out = self.manager.Log()
self.err = self.manager.Log()
self.queue = Queue(mode=self.mode,
stdout=self.out,
stderr=self.err,
**kwargs)
self.accm = self.queue.account_manager
if logdir is not None:
self.logger = FileLogger(logdir)
def setUp(self):
self.tempdir = mkdtemp()
self.queue = None
self.logger = None
self.manager = LogManager()
self.manager.start()
self.createQueue(verbose=-1, logdir=self.tempdir)
def tearDown(self):
shutil.rmtree(self.tempdir)
try:
self.queue.destroy()
except:
pass # queue already destroyed
self.manager.shutdown()
def assertVerbosity(self, channel, expected):
data = channel.read()
if expected == 'no_tb':
self.assert_('error' in data, data)
self.assert_('Traceback' not in data)
elif expected == 'tb':
self.assert_('error' in data, data)
self.assert_('Traceback' in data)
elif expected == '':
self.assertEqual(data, '')
else:
msg = repr(expected) + ' not in ' + repr(data)
self.assert_(expected in data, msg)
def testConstructor(self):
self.assertEqual(1, self.queue.get_max_threads())
# Test all verbosity levels.
levels = (
(-1, 1, ('', ''), ('', ''), ('', 'tb')),
(-1, 2, ('', ''), ('', ''), ('', 'tb')),
(0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')),
(0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')),
(1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')),
(1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')),
(2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
)
for level, max_threads, with_simple, with_error, with_fatal in levels:
#print "S:", level, max_threads, with_simple, with_error, with_fatal
stdout, stderr = with_simple
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', say_hello)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
#print "E:", level, max_threads, with_simple, with_error, with_fatal
stdout, stderr = with_error
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
#print "F:", level, max_threads, with_simple, with_error, with_fatal
stdout, stderr = with_fatal
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', fatal_error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
def testCreatePipe(self):
account = Account('user', 'test')
self.accm.add_account(account)
pipe = self.queue._create_pipe()
pipe.send(('acquire-account', None))
response = pipe.recv()
expected = (account.__hash__(), account.get_name(),
account.get_password(),
account.get_authorization_password(), account.get_key())
self.assertEqual(response, expected)
pipe.send(('release-account', account.__hash__()))
response = pipe.recv()
self.assertEqual(response, 'ok')
pipe.close()
def testSetMaxThreads(self):
self.assertEqual(1, self.queue.get_max_threads())
self.queue.set_max_threads(2)
self.assertEqual(2, self.queue.get_max_threads())
def testGetMaxThreads(self):
pass # Already tested in testSetMaxThreads().
def testGetProgress(self):
self.assertEqual(0.0, self.queue.get_progress())
self.testIsCompleted()
self.assertEqual(100.0, self.queue.get_progress())
def testAddAccount(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def testAddAccountPool(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def match_cb(data, host):
data['match-called'].value = True
return True
def start_cb(data, job, host, conn):
account = conn.account_factory(None)
data['start-called'].value = True
data['account-hash'].value = account.__hash__()
account.release()
# Replace the default pool.
pool1 = AccountPool()
self.queue.add_account_pool(pool1)
self.assertEqual(self.accm.default_pool, pool1)
# Add another pool, making sure that it does not replace
# the default pool.
pool2 = AccountPool()
account2 = Account('user', 'test')
pool2.add_account(account2)
match_called = Value(ctypes.c_bool, False)
start_called = Value(ctypes.c_bool, False)
account_hash = Value(ctypes.c_long, 0)
data = {
'match-called': match_called,
'start-called': start_called,
'account-hash': account_hash
}
self.queue.add_account_pool(pool2, partial(match_cb, data))
self.assertEqual(self.accm.default_pool, pool1)
# Make sure that pool2 is chosen (because the match function
# returns True).
self.queue.run('dummy://dummy', partial(start_cb, data))
self.queue.shutdown()
data = dict((k, v.value) for (k, v) in data.iteritems())
self.assertEqual(
data, {
'match-called': True,
'start-called': True,
'account-hash': account2.__hash__()
})
def startTask(self):
self.testAddAccount()
hosts = ['dummy://dummy1', 'dummy://dummy2']
task = self.queue.run(hosts, log_to(self.logger)(do_nothing))
self.assert_(task is not None)
return task
def testIsCompleted(self):
self.assert_(self.queue.is_completed())
task = self.startTask()
self.failIf(self.queue.is_completed())
task.wait()
self.assert_(task.is_completed())
self.assert_(self.queue.is_completed())
def testJoin(self):
task = self.startTask()
self.queue.join()
self.assert_(task.is_completed())
self.assert_(self.queue.is_completed())
def testShutdown(self):
task = self.startTask() # this also adds an account
self.queue.shutdown()
self.assert_(task.is_completed())
self.assert_(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 1)
def testDestroy(self):
task = self.startTask() # this also adds an account
self.queue.destroy()
self.assert_(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testReset(self):
self.testAddAccount()
self.queue.reset()
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testExceptionCallback(self):
self.exc = {}
def my_exc_cb(jobname, exc_info):
self.exc[jobname] = exc_info
self.createQueue(exc_cb=my_exc_cb)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assert_("mytest" in self.exc)
self.assert_(isinstance(self.exc["mytest"][1], FailException))
def testRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
self.queue.run(hosts, func)
self.queue.run('dummy://dummy3', func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
self.queue.run('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 4)
def testRunOrIgnore(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.run_or_ignore(hosts, func)
self.queue.run_or_ignore('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.run_or_ignore('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testPriorityRun(self):
def write(data, value, *args):
data.value = value
data = Value('i', 0)
self.queue.workqueue.pause()
self.queue.enqueue(partial(write, data, 1))
self.queue.priority_run('dummy://dummy', partial(write, data, 2))
self.queue.workqueue.unpause()
self.queue.destroy()
# The 'dummy' job should run first, so the value must
# be overwritten by the other process.
self.assertEqual(data.value, 1)
def testPriorityRunOrRaise(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.priority_run_or_raise(hosts, func)
self.queue.priority_run_or_raise('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.priority_run_or_raise('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testForceRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
# By setting max_threads to 0 we ensure that the 'force' part is
# actually tested; the thread should run regardless.
self.queue.set_max_threads(0)
self.queue.force_run(hosts, func)
self.queue.destroy()
self.assertEqual(data.value, 2)
def testEnqueue(self):
data = Value('i', 0)
func = bind(count_calls, data, testarg=1)
self.queue.enqueue(func)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
func = bind(count_and_fail, data, testarg=1)
self.queue.enqueue(func, attempts=7)
self.queue.destroy()
self.assertEqual(data.value, 10)
#FIXME: Not a method test; this should probably be elsewhere.
def testLogging(self):
task = self.startTask()
while not task.is_completed():
time.sleep(.1)
# The following function is not synchronous with the above, so add
# a timeout to avoid races.
time.sleep(.1)
self.assert_(self.queue.is_completed())
logfiles = os.listdir(self.tempdir)
self.assertEqual(2, len(logfiles))
self.assert_('dummy1.log' in logfiles)
self.assert_('dummy2.log' in logfiles)
for file in logfiles:
content = open(os.path.join(self.tempdir, file)).read()
f.write(description + "\t\t\t" + intf + "\t\t\t" + "Serial: " + serial + "\n")
f.close()
# Read input data.
#accounts = get_accounts_from_file('accounts.cfg')
accounts = read_login()
try:
hosts = get_hosts_from_file('hostlist.txt')
except:
print """
Devi creare un file che si chiama hostlist.txt e dentro
ci sono linee di questo tipo:
ssh://r.rm2.garr.net
"""
sys.exit(0)
# Run do_something on each of the hosts. The given accounts are used
# round-robin. "verbose = 0" instructs the queue to not generate any
# output on stdout.
queue = Queue(verbose = 3, max_threads = 5)
queue.add_account(accounts) # Adds one or more accounts.
queue.run(hosts, do_something) # Asynchronously enqueues all hosts.
queue.shutdown() # Waits until all hosts are completed.
# Print a short report.
print status(logger)
print summarize(logger)
print
account = read_login() # Prompt the user for login credentials
else: # Else use username/password from configFile
account = Account(name=username, password=b64decode(password))
# Read hosts from specified file & remove duplicate entries, set protocol to SSH2
hosts = get_hosts_from_file(routerFile, default_protocol="ssh2", remove_duplicates=True)
print
# Verbose & # threads taken from configFile, redirect errors to null
queue = Queue(verbose=int(verboseOutput), max_threads=int(maxThreads), stderr=(open(os.devnull, "w")))
queue.add_account(account) # Use supplied user credentials
queue.run(hosts, downloadRouterConfig) # Create queue using provided hosts
queue.shutdown() # End all running threads and close queue
print status(logger) # Print current % status of operation to screen
# If logFileDirectory does not exist, create it.
if not path.exists(logFileDirectory): makedirs(logFileDirectory)
# Define log filename based on date
logFilename = logFileDirectory+"DownloadRouterConfig_"+date+".log"
# Check to see if logFilename currently exists. If it does, append an
# integer onto the end of the filename until logFilename no longer exists
incrementLogFilename = 1
while fileExist(logFilename):
logFilename = logFileDirectory+"DownloadRouterConfig_"+date+"_"+str(incrementLogFilename)+".log"
incrementLogFilename = incrementLogFilename + 1
class QueueTest(unittest.TestCase):
CORRELATE = Queue
mode = 'threading'
def createQueue(self, logdir=None, **kwargs):
if self.queue:
self.queue.destroy()
self.out = self.manager.Log()
self.err = self.manager.Log()
self.queue = Queue(mode=self.mode,
stdout=self.out,
stderr=self.err,
**kwargs)
self.accm = self.queue.account_manager
if logdir is not None:
self.logger = FileLogger(logdir)
def setUp(self):
self.tempdir = mkdtemp()
self.queue = None
self.logger = None
self.manager = LogManager()
self.manager.start()
self.createQueue(verbose=-1, logdir=self.tempdir)
def tearDown(self):
shutil.rmtree(self.tempdir)
try:
self.queue.destroy()
except:
pass # queue already destroyed
self.manager.shutdown()
def assertVerbosity(self, channel, expected):
data = channel.read()
if expected == 'no_tb':
self.assertTrue('error' in data, data)
self.assertNotIn('Traceback', data)
elif expected == 'tb':
self.assertTrue('error' in data, data)
self.assertIn('Traceback', data)
elif expected == '':
self.assertEqual(data, '')
else:
msg = repr(expected) + ' not in ' + repr(data)
self.assertTrue(expected in data, msg)
def testConstructor(self):
self.assertEqual(1, self.queue.get_max_threads())
# Test all verbosity levels.
levels = (
(-1, 1, ('', ''), ('', ''), ('', 'tb')),
(-1, 2, ('', ''), ('', ''), ('', 'tb')),
(0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')),
(0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')),
(1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')),
(1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')),
(2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
(5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')),
(5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')),
)
for level, max_threads, with_simple, with_error, with_fatal in levels:
# print("S:", level, max_threads, with_simple, with_error,)
# with_fatal
stdout, stderr = with_simple
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', say_hello)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
# print("E:", level, max_threads, with_simple, with_error,)
# with_fatal
stdout, stderr = with_error
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
# print("F:", level, max_threads, with_simple, with_error,)
# with_fatal
stdout, stderr = with_fatal
self.createQueue(verbose=level, max_threads=max_threads)
self.queue.run('dummy://mytest', fatal_error)
self.queue.join()
self.assertVerbosity(self.out, stdout)
self.assertVerbosity(self.err, stderr)
def testCreatePipe(self):
account = Account('user', 'test')
self.accm.add_account(account)
pipe = self.queue._create_pipe()
pipe.send(('acquire-account', None))
response = pipe.recv()
expected = (account.__hash__(),
account.get_name(),
account.get_password(),
account.get_authorization_password(),
account.get_key())
self.assertEqual(response, expected)
pipe.send(('release-account', account.__hash__()))
response = pipe.recv()
self.assertEqual(response, 'ok')
pipe.close()
def testSetMaxThreads(self):
self.assertEqual(1, self.queue.get_max_threads())
self.queue.set_max_threads(2)
self.assertEqual(2, self.queue.get_max_threads())
def testGetMaxThreads(self):
pass # Already tested in testSetMaxThreads().
def testGetProgress(self):
self.assertEqual(0.0, self.queue.get_progress())
self.testIsCompleted()
self.assertEqual(100.0, self.queue.get_progress())
def testAddAccount(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def testAddAccountPool(self):
self.assertEqual(0, self.accm.default_pool.n_accounts())
account = Account('user', 'test')
self.queue.add_account(account)
self.assertEqual(1, self.accm.default_pool.n_accounts())
def match_cb(data, host):
data['match-called'].value = True
return True
def start_cb(data, job, host, conn):
account = conn.account_factory(None)
data['start-called'].value = True
data['account-hash'].value = account.__hash__()
account.release()
# Replace the default pool.
pool1 = AccountPool()
self.queue.add_account_pool(pool1)
self.assertEqual(self.accm.default_pool, pool1)
# Add another pool, making sure that it does not replace
# the default pool.
pool2 = AccountPool()
account2 = Account('user', 'test')
pool2.add_account(account2)
match_called = Value(ctypes.c_bool, False)
start_called = Value(ctypes.c_bool, False)
account_hash = Value(ctypes.c_long, 0)
data = {'match-called': match_called,
'start-called': start_called,
'account-hash': account_hash}
self.queue.add_account_pool(pool2, partial(match_cb, data))
self.assertEqual(self.accm.default_pool, pool1)
# Make sure that pool2 is chosen (because the match function
# returns True).
self.queue.run('dummy://dummy', partial(start_cb, data))
self.queue.shutdown()
data = dict((k, v.value) for (k, v) in list(data.items()))
self.assertEqual(data, {'match-called': True,
'start-called': True,
'account-hash': account2.__hash__()})
def startTask(self):
self.testAddAccount()
hosts = ['dummy://dummy1', 'dummy://dummy2']
task = self.queue.run(hosts, log_to(self.logger)(do_nothing))
self.assertTrue(task is not None)
return task
def testIsCompleted(self):
self.assertTrue(self.queue.is_completed())
task = self.startTask()
self.assertFalse(self.queue.is_completed())
task.wait()
self.assertTrue(task.is_completed())
self.assertTrue(self.queue.is_completed())
def testJoin(self):
task = self.startTask()
self.queue.join()
self.assertTrue(task.is_completed())
self.assertTrue(self.queue.is_completed())
def testShutdown(self):
task = self.startTask() # this also adds an account
self.queue.shutdown()
self.assertTrue(task.is_completed())
self.assertTrue(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 1)
def testDestroy(self):
task = self.startTask() # this also adds an account
self.queue.destroy()
self.assertTrue(self.queue.is_completed())
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testReset(self):
self.testAddAccount()
self.queue.reset()
self.assertEqual(self.accm.default_pool.n_accounts(), 0)
def testExceptionCallback(self):
self.exc = {}
def my_exc_cb(jobname, exc_info):
self.exc[jobname] = exc_info
self.createQueue(exc_cb=my_exc_cb)
self.queue.run('dummy://mytest', error)
self.queue.join()
self.assertIn("mytest", self.exc)
self.assertIsInstance(self.exc["mytest"][1], FailException)
def testRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
self.queue.run(hosts, func)
self.queue.run('dummy://dummy3', func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
self.queue.run('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 4)
def testRunOrIgnore(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.run_or_ignore(hosts, func)
self.queue.run_or_ignore('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.run_or_ignore('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testPriorityRun(self):
def write(data, value, *args):
data.value = value
data = Value('i', 0)
self.queue.workqueue.pause()
self.queue.enqueue(partial(write, data, 1))
self.queue.priority_run('dummy://dummy', partial(write, data, 2))
self.queue.workqueue.unpause()
self.queue.destroy()
# The 'dummy' job should run first, so the value must
# be overwritten by the other process.
self.assertEqual(data.value, 1)
def testPriorityRunOrRaise(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.priority_run_or_raise(hosts, func)
self.queue.priority_run_or_raise('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.priority_run_or_raise('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testForceRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
# By setting max_threads to 0 we ensure that the 'force' part is
# actually tested; the thread should run regardless.
self.queue.set_max_threads(0)
self.queue.force_run(hosts, func)
self.queue.destroy()
self.assertEqual(data.value, 2)
def testEnqueue(self):
data = Value('i', 0)
func = bind(count_calls, data, testarg=1)
self.queue.enqueue(func)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
func = bind(count_and_fail, data, testarg=1)
self.queue.enqueue(func, attempts=7)
self.queue.destroy()
self.assertEqual(data.value, 10)
# FIXME: Not a method test; this should probably be elsewhere.
def testLogging(self):
task = self.startTask()
while not task.is_completed():
time.sleep(.1)
# The following function is not synchronous with the above, so add
# a timeout to avoid races.
time.sleep(.1)
self.assertTrue(self.queue.is_completed())
logfiles = os.listdir(self.tempdir)
self.assertEqual(2, len(logfiles))
self.assertIn('dummy1.log', logfiles)
self.assertIn('dummy2.log', logfiles)
for file in logfiles:
with open(os.path.join(self.tempdir, file)) as fp:
content = fp.read()
import time, Exscript.util.sigintcatcher
from Exscript import Queue, Account
from Exscript.util.decorator import bind
from TkExscript import QueueWindow
def do_something(conn, wait):
conn.connect()
conn.authenticate()
for i in range(100):
conn.execute('test%d' % i)
time.sleep(wait)
conn.close()
queue = Queue(max_threads=4, verbose=0)
queue.add_account(Account('test', 'test'))
window = QueueWindow(queue)
queue.run('dummy://dummy1', bind(do_something, .02))
queue.run('dummy://dummy2', bind(do_something, .2))
queue.run('dummy://dummy3', bind(do_something, .3))
window.mainloop()
queue.shutdown()
from Exscript import Queue, Logger
from Exscript.util.log import log_to
from Exscript.util.decorator import autologin
from Exscript.util.file import get_hosts_from_file, get_accounts_from_file
from Exscript.util.report import status, summarize
logger = Logger() # Logs everything to memory.
@log_to(logger)
@autologin()
def do_something(job, host, conn):
conn.execute('show ip int brie')
# Read input data.
accounts = get_accounts_from_file('accounts.cfg')
hosts = get_hosts_from_file('hostlist.txt')
# Run do_something on each of the hosts. The given accounts are used
# round-robin. "verbose=0" instructs the queue to not generate any
# output on stdout.
queue = Queue(verbose=5, max_threads=5)
queue.add_account(accounts) # Adds one or more accounts.
queue.run(hosts, do_something) # Asynchronously enqueues all hosts.
queue.shutdown() # Waits until all hosts are completed.
# Print a short report.
print status(logger)
print summarize(logger)
