class TemplateTest(unittest.TestCase): def setUp(self): account = Account('sab', '') self.queue = Queue(verbose=0, max_threads=1) self.logger = Logger() self.queue.add_account(account) def tearDown(self): self.queue.destroy() def testTemplates(self): callback = bind(log_to(self.logger)(dummy_cb), self) for test in os.listdir(test_dir): pseudo = os.path.join(test_dir, test, 'pseudodev.py') if os.path.exists(pseudo): self.queue.run('pseudo://' + pseudo, callback) else: self.queue.run('ios://' + test, callback) self.queue.shutdown() # Unfortunately, unittest.TestCase does not fail if self.assert() # was called from a subthread, so this is our workaround... failed = self.logger.get_aborted_logs() report = format(self.logger, show_successful=False) self.assert_(not failed, report)
class TemplateTest(unittest.TestCase): def setUp(self): account = Account('sab', '') self.queue = Queue(verbose = 0, max_threads = 1) self.logger = Logger() self.queue.add_account(account) def tearDown(self): self.queue.destroy() def testTemplates(self): callback = bind(log_to(self.logger)(dummy_cb), self) for test in os.listdir(test_dir): pseudo = os.path.join(test_dir, test, 'pseudodev.py') if os.path.exists(pseudo): self.queue.run('pseudo://' + pseudo, callback) else: self.queue.run('ios://' + test, callback) self.queue.shutdown() # Unfortunately, unittest.TestCase does not fail if self.assert() # was called from a subthread, so this is our workaround... failed = self.logger.get_aborted_logs() report = format(self.logger, show_successful = False) self.assert_(not failed, report)
def routerLogin(): # This function prompts the user to provide their login credentials and logs into each # of the routers before calling the buildIndex function to extract relevant portions of # the router config. As designed, this function actually has the capability to login to # multiple routers simultaneously. I chose to not allow it to multi-thread given possibility # of undesirable results from multiple threads writing to the same index file simultaneously try:# Check for existence of routerFile; If exists, continue with program with open(routerFile, "r"): pass # Read hosts from specified file & remove duplicate entries, set protocol to SSH2 hosts = get_hosts_from_file(routerFile,default_protocol="ssh2",remove_duplicates=True) if username == "": # If username is blank print account = read_login() # Prompt the user for login credentials elif password == "": # If password is blank print account = read_login() # Prompt the user for login credentials else: # Else use username/password from configFile account = Account(name=username, password=b64decode(password)) # Minimal message from queue, 1 threads, redirect errors to null queue = Queue(verbose=0, max_threads=1, stderr=(open(os.devnull, "w"))) queue.add_account(account) # Use supplied user credentials print stdout.write("--> Building index...") # Print without trailing newline queue.run(hosts, buildIndex) # Create queue using provided hosts queue.shutdown() # End all running threads and close queue # If logFileDirectory does not exist, create it. if not path.exists(logFileDirectory): makedirs(logFileDirectory) # Define log filename based on date logFilename = logFileDirectory+"VRFSearchAndBackup_"+date+".log" # Check to see if logFilename currently exists. If it does, append an # integer onto the end of the filename until logFilename no longer exists incrementLogFilename = 1 while fileExist(logFilename): logFilename = logFileDirectory+"VRFSearchAndBackup_"+date+"_"+str(incrementLogFilename)+".log" incrementLogFilename = incrementLogFilename + 1 # Write log results to logFile with open(logFilename, "w") as outputLogFile: try: outputLogFile.write(summarize(logger)) # Exception: router file was not able to be opened except IOError: print print "--> An error occurred opening "+logFileDirectory+logFile+"." # Exception: router file could not be opened except IOError: print print "--> An error occurred opening "+routerFile+"."
def handle(self, *args, **options): if options['all'] and options['device_id']: print ("Error: --all and --deviceid detected. Use only one") return if options['all']: devices = Device.objects.all() if options['device_id']: try: devices = [ Device.objects.get(pk=int(options['device_id'])) ] except: raise CommandError('Device ID "%s" does not exist' % options['device_id']) for device in devices: found_credential = False for credential in Credential.objects.all(): if credential.match_ip(device.access_ip): found_credential = True host = Host(device.protocol + device.access_ip) account = (Account(credential.username, credential.password)) if credential.enable_username and credential.enable_password: account.set_authorization_password(credential.enable_password) host.set_account(account) host.set('device', device) hosts.append(host) break if not found_credential: device.backup_status = "No credentials found." % device device.save() queue = Queue(max_threads = 5, verbose = -1, stderr=open(os.devnull, 'w')) queue.run(hosts, do_backup) queue.shutdown() for log in logger.get_logs(): device = Device.objects.get(access_ip=log.get_name()) if log.has_error(): device.backup_status = log.get_error(False) else: device.backup_status = 'ok' device.backup_last_ran = now() device.save()
objnames = ('count_calls',) follow_modules = False def count_calls(conn, thedata, **kwargs): thedata['n_calls'] += 1 def foobar(): pass queue = Queue() data = {'n_calls': 0} func = bind(count_calls, data) task = queue.run(['t1', 't2', 't3', 't4', 't5', 't6'], func) task.wait() queue.shutdown() queue.destroy() del func # Test memory consumption. from meliae import scanner gc.collect() scanner.dump_all_objects("test.dump") from meliae import loader om = loader.load('test.dump') om.remove_expensive_references() om.collapse_instance_dicts() om.compute_referrers() om.compute_total_size() #print om.summarize()
class QueueTest(unittest.TestCase): CORRELATE = Queue mode = 'threading' def createQueue(self, logdir=None, **kwargs): if self.queue: self.queue.destroy() self.out = self.manager.Log() self.err = self.manager.Log() self.queue = Queue(mode=self.mode, stdout=self.out, stderr=self.err, **kwargs) self.accm = self.queue.account_manager if logdir is not None: self.logger = FileLogger(logdir) def setUp(self): self.tempdir = mkdtemp() self.queue = None self.logger = None self.manager = LogManager() self.manager.start() self.createQueue(verbose=-1, logdir=self.tempdir) def tearDown(self): shutil.rmtree(self.tempdir) try: self.queue.destroy() except: pass # queue already destroyed self.manager.shutdown() def assertVerbosity(self, channel, expected): data = channel.read() if expected == 'no_tb': self.assert_('error' in data, data) self.assert_('Traceback' not in data) elif expected == 'tb': self.assert_('error' in data, data) self.assert_('Traceback' in data) elif expected == '': self.assertEqual(data, '') else: msg = repr(expected) + ' not in ' + repr(data) self.assert_(expected in data, msg) def testConstructor(self): self.assertEqual(1, self.queue.get_max_threads()) # Test all verbosity levels. levels = ( (-1, 1, ('', ''), ('', ''), ('', 'tb')), (-1, 2, ('', ''), ('', ''), ('', 'tb')), (0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')), (0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')), (1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')), (1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')), (2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), ) for level, max_threads, with_simple, with_error, with_fatal in levels: #print "S:", level, max_threads, with_simple, with_error, with_fatal stdout, stderr = with_simple self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', say_hello) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) #print "E:", level, max_threads, with_simple, with_error, with_fatal stdout, stderr = with_error self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', error) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) #print "F:", level, max_threads, with_simple, with_error, with_fatal stdout, stderr = with_fatal self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', fatal_error) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) def testCreatePipe(self): account = Account('user', 'test') self.accm.add_account(account) pipe = self.queue._create_pipe() pipe.send(('acquire-account', None)) response = pipe.recv() expected = (account.__hash__(), account.get_name(), account.get_password(), account.get_authorization_password(), account.get_key()) self.assertEqual(response, expected) pipe.send(('release-account', account.__hash__())) response = pipe.recv() self.assertEqual(response, 'ok') pipe.close() def testSetMaxThreads(self): self.assertEqual(1, self.queue.get_max_threads()) self.queue.set_max_threads(2) self.assertEqual(2, self.queue.get_max_threads()) def testGetMaxThreads(self): pass # Already tested in testSetMaxThreads(). def testGetProgress(self): self.assertEqual(0.0, self.queue.get_progress()) self.testIsCompleted() self.assertEqual(100.0, self.queue.get_progress()) def testAddAccount(self): self.assertEqual(0, self.accm.default_pool.n_accounts()) account = Account('user', 'test') self.queue.add_account(account) self.assertEqual(1, self.accm.default_pool.n_accounts()) def testAddAccountPool(self): self.assertEqual(0, self.accm.default_pool.n_accounts()) account = Account('user', 'test') self.queue.add_account(account) self.assertEqual(1, self.accm.default_pool.n_accounts()) def match_cb(data, host): data['match-called'].value = True return True def start_cb(data, job, host, conn): account = conn.account_factory(None) data['start-called'].value = True data['account-hash'].value = account.__hash__() account.release() # Replace the default pool. pool1 = AccountPool() self.queue.add_account_pool(pool1) self.assertEqual(self.accm.default_pool, pool1) # Add another pool, making sure that it does not replace # the default pool. pool2 = AccountPool() account2 = Account('user', 'test') pool2.add_account(account2) match_called = Value(ctypes.c_bool, False) start_called = Value(ctypes.c_bool, False) account_hash = Value(ctypes.c_long, 0) data = { 'match-called': match_called, 'start-called': start_called, 'account-hash': account_hash } self.queue.add_account_pool(pool2, partial(match_cb, data)) self.assertEqual(self.accm.default_pool, pool1) # Make sure that pool2 is chosen (because the match function # returns True). self.queue.run('dummy://dummy', partial(start_cb, data)) self.queue.shutdown() data = dict((k, v.value) for (k, v) in data.iteritems()) self.assertEqual( data, { 'match-called': True, 'start-called': True, 'account-hash': account2.__hash__() }) def startTask(self): self.testAddAccount() hosts = ['dummy://dummy1', 'dummy://dummy2'] task = self.queue.run(hosts, log_to(self.logger)(do_nothing)) self.assert_(task is not None) return task def testIsCompleted(self): self.assert_(self.queue.is_completed()) task = self.startTask() self.failIf(self.queue.is_completed()) task.wait() self.assert_(task.is_completed()) self.assert_(self.queue.is_completed()) def testJoin(self): task = self.startTask() self.queue.join() self.assert_(task.is_completed()) self.assert_(self.queue.is_completed()) def testShutdown(self): task = self.startTask() # this also adds an account self.queue.shutdown() self.assert_(task.is_completed()) self.assert_(self.queue.is_completed()) self.assertEqual(self.accm.default_pool.n_accounts(), 1) def testDestroy(self): task = self.startTask() # this also adds an account self.queue.destroy() self.assert_(self.queue.is_completed()) self.assertEqual(self.accm.default_pool.n_accounts(), 0) def testReset(self): self.testAddAccount() self.queue.reset() self.assertEqual(self.accm.default_pool.n_accounts(), 0) def testExceptionCallback(self): self.exc = {} def my_exc_cb(jobname, exc_info): self.exc[jobname] = exc_info self.createQueue(exc_cb=my_exc_cb) self.queue.run('dummy://mytest', error) self.queue.join() self.assert_("mytest" in self.exc) self.assert_(isinstance(self.exc["mytest"][1], FailException)) def testRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] func = bind(count_calls2, data, testarg=1) self.queue.run(hosts, func) self.queue.run('dummy://dummy3', func) self.queue.shutdown() self.assertEqual(data.value, 3) self.queue.run('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 4) def testRunOrIgnore(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1'] func = bind(count_calls2, data, testarg=1) self.queue.workqueue.pause() self.queue.run_or_ignore(hosts, func) self.queue.run_or_ignore('dummy://dummy2', func) self.queue.workqueue.unpause() self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.run_or_ignore('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 3) def testPriorityRun(self): def write(data, value, *args): data.value = value data = Value('i', 0) self.queue.workqueue.pause() self.queue.enqueue(partial(write, data, 1)) self.queue.priority_run('dummy://dummy', partial(write, data, 2)) self.queue.workqueue.unpause() self.queue.destroy() # The 'dummy' job should run first, so the value must # be overwritten by the other process. self.assertEqual(data.value, 1) def testPriorityRunOrRaise(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1'] func = bind(count_calls2, data, testarg=1) self.queue.workqueue.pause() self.queue.priority_run_or_raise(hosts, func) self.queue.priority_run_or_raise('dummy://dummy2', func) self.queue.workqueue.unpause() self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.priority_run_or_raise('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 3) def testForceRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] func = bind(count_calls2, data, testarg=1) # By setting max_threads to 0 we ensure that the 'force' part is # actually tested; the thread should run regardless. self.queue.set_max_threads(0) self.queue.force_run(hosts, func) self.queue.destroy() self.assertEqual(data.value, 2) def testEnqueue(self): data = Value('i', 0) func = bind(count_calls, data, testarg=1) self.queue.enqueue(func) self.queue.enqueue(func) self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.enqueue(func) self.queue.shutdown() self.assertEqual(data.value, 3) func = bind(count_and_fail, data, testarg=1) self.queue.enqueue(func, attempts=7) self.queue.destroy() self.assertEqual(data.value, 10) #FIXME: Not a method test; this should probably be elsewhere. def testLogging(self): task = self.startTask() while not task.is_completed(): time.sleep(.1) # The following function is not synchronous with the above, so add # a timeout to avoid races. time.sleep(.1) self.assert_(self.queue.is_completed()) logfiles = os.listdir(self.tempdir) self.assertEqual(2, len(logfiles)) self.assert_('dummy1.log' in logfiles) self.assert_('dummy2.log' in logfiles) for file in logfiles: content = open(os.path.join(self.tempdir, file)).read()
f.write(description + "\t\t\t" + intf + "\t\t\t" + "Serial: " + serial + "\n") f.close() # Read input data. #accounts = get_accounts_from_file('accounts.cfg') accounts = read_login() try: hosts = get_hosts_from_file('hostlist.txt') except: print """ Devi creare un file che si chiama hostlist.txt e dentro ci sono linee di questo tipo: ssh://r.rm2.garr.net """ sys.exit(0) # Run do_something on each of the hosts. The given accounts are used # round-robin. "verbose = 0" instructs the queue to not generate any # output on stdout. queue = Queue(verbose = 3, max_threads = 5) queue.add_account(accounts) # Adds one or more accounts. queue.run(hosts, do_something) # Asynchronously enqueues all hosts. queue.shutdown() # Waits until all hosts are completed. # Print a short report. print status(logger) print summarize(logger)
print account = read_login() # Prompt the user for login credentials else: # Else use username/password from configFile account = Account(name=username, password=b64decode(password)) # Read hosts from specified file & remove duplicate entries, set protocol to SSH2 hosts = get_hosts_from_file(routerFile, default_protocol="ssh2", remove_duplicates=True) print # Verbose & # threads taken from configFile, redirect errors to null queue = Queue(verbose=int(verboseOutput), max_threads=int(maxThreads), stderr=(open(os.devnull, "w"))) queue.add_account(account) # Use supplied user credentials queue.run(hosts, downloadRouterConfig) # Create queue using provided hosts queue.shutdown() # End all running threads and close queue print status(logger) # Print current % status of operation to screen # If logFileDirectory does not exist, create it. if not path.exists(logFileDirectory): makedirs(logFileDirectory) # Define log filename based on date logFilename = logFileDirectory+"DownloadRouterConfig_"+date+".log" # Check to see if logFilename currently exists. If it does, append an # integer onto the end of the filename until logFilename no longer exists incrementLogFilename = 1 while fileExist(logFilename): logFilename = logFileDirectory+"DownloadRouterConfig_"+date+"_"+str(incrementLogFilename)+".log" incrementLogFilename = incrementLogFilename + 1
class QueueTest(unittest.TestCase): CORRELATE = Queue mode = 'threading' def createQueue(self, logdir=None, **kwargs): if self.queue: self.queue.destroy() self.out = self.manager.Log() self.err = self.manager.Log() self.queue = Queue(mode=self.mode, stdout=self.out, stderr=self.err, **kwargs) self.accm = self.queue.account_manager if logdir is not None: self.logger = FileLogger(logdir) def setUp(self): self.tempdir = mkdtemp() self.queue = None self.logger = None self.manager = LogManager() self.manager.start() self.createQueue(verbose=-1, logdir=self.tempdir) def tearDown(self): shutil.rmtree(self.tempdir) try: self.queue.destroy() except: pass # queue already destroyed self.manager.shutdown() def assertVerbosity(self, channel, expected): data = channel.read() if expected == 'no_tb': self.assertTrue('error' in data, data) self.assertNotIn('Traceback', data) elif expected == 'tb': self.assertTrue('error' in data, data) self.assertIn('Traceback', data) elif expected == '': self.assertEqual(data, '') else: msg = repr(expected) + ' not in ' + repr(data) self.assertTrue(expected in data, msg) def testConstructor(self): self.assertEqual(1, self.queue.get_max_threads()) # Test all verbosity levels. levels = ( (-1, 1, ('', ''), ('', ''), ('', 'tb')), (-1, 2, ('', ''), ('', ''), ('', 'tb')), (0, 1, ('', ''), ('', 'no_tb'), ('', 'tb')), (0, 2, ('', ''), ('', 'no_tb'), ('', 'tb')), (1, 1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')), (1, 2, ('[', ''), ('[', 'no_tb'), ('[', 'tb')), (2, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (2, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (3, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (3, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (4, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (4, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), (5, 1, ('hello', ''), ('hello', 'tb'), ('hello', 'tb')), (5, 2, ('[', ''), ('[', 'tb'), ('[', 'tb')), ) for level, max_threads, with_simple, with_error, with_fatal in levels: # print("S:", level, max_threads, with_simple, with_error,) # with_fatal stdout, stderr = with_simple self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', say_hello) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) # print("E:", level, max_threads, with_simple, with_error,) # with_fatal stdout, stderr = with_error self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', error) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) # print("F:", level, max_threads, with_simple, with_error,) # with_fatal stdout, stderr = with_fatal self.createQueue(verbose=level, max_threads=max_threads) self.queue.run('dummy://mytest', fatal_error) self.queue.join() self.assertVerbosity(self.out, stdout) self.assertVerbosity(self.err, stderr) def testCreatePipe(self): account = Account('user', 'test') self.accm.add_account(account) pipe = self.queue._create_pipe() pipe.send(('acquire-account', None)) response = pipe.recv() expected = (account.__hash__(), account.get_name(), account.get_password(), account.get_authorization_password(), account.get_key()) self.assertEqual(response, expected) pipe.send(('release-account', account.__hash__())) response = pipe.recv() self.assertEqual(response, 'ok') pipe.close() def testSetMaxThreads(self): self.assertEqual(1, self.queue.get_max_threads()) self.queue.set_max_threads(2) self.assertEqual(2, self.queue.get_max_threads()) def testGetMaxThreads(self): pass # Already tested in testSetMaxThreads(). def testGetProgress(self): self.assertEqual(0.0, self.queue.get_progress()) self.testIsCompleted() self.assertEqual(100.0, self.queue.get_progress()) def testAddAccount(self): self.assertEqual(0, self.accm.default_pool.n_accounts()) account = Account('user', 'test') self.queue.add_account(account) self.assertEqual(1, self.accm.default_pool.n_accounts()) def testAddAccountPool(self): self.assertEqual(0, self.accm.default_pool.n_accounts()) account = Account('user', 'test') self.queue.add_account(account) self.assertEqual(1, self.accm.default_pool.n_accounts()) def match_cb(data, host): data['match-called'].value = True return True def start_cb(data, job, host, conn): account = conn.account_factory(None) data['start-called'].value = True data['account-hash'].value = account.__hash__() account.release() # Replace the default pool. pool1 = AccountPool() self.queue.add_account_pool(pool1) self.assertEqual(self.accm.default_pool, pool1) # Add another pool, making sure that it does not replace # the default pool. pool2 = AccountPool() account2 = Account('user', 'test') pool2.add_account(account2) match_called = Value(ctypes.c_bool, False) start_called = Value(ctypes.c_bool, False) account_hash = Value(ctypes.c_long, 0) data = {'match-called': match_called, 'start-called': start_called, 'account-hash': account_hash} self.queue.add_account_pool(pool2, partial(match_cb, data)) self.assertEqual(self.accm.default_pool, pool1) # Make sure that pool2 is chosen (because the match function # returns True). self.queue.run('dummy://dummy', partial(start_cb, data)) self.queue.shutdown() data = dict((k, v.value) for (k, v) in list(data.items())) self.assertEqual(data, {'match-called': True, 'start-called': True, 'account-hash': account2.__hash__()}) def startTask(self): self.testAddAccount() hosts = ['dummy://dummy1', 'dummy://dummy2'] task = self.queue.run(hosts, log_to(self.logger)(do_nothing)) self.assertTrue(task is not None) return task def testIsCompleted(self): self.assertTrue(self.queue.is_completed()) task = self.startTask() self.assertFalse(self.queue.is_completed()) task.wait() self.assertTrue(task.is_completed()) self.assertTrue(self.queue.is_completed()) def testJoin(self): task = self.startTask() self.queue.join() self.assertTrue(task.is_completed()) self.assertTrue(self.queue.is_completed()) def testShutdown(self): task = self.startTask() # this also adds an account self.queue.shutdown() self.assertTrue(task.is_completed()) self.assertTrue(self.queue.is_completed()) self.assertEqual(self.accm.default_pool.n_accounts(), 1) def testDestroy(self): task = self.startTask() # this also adds an account self.queue.destroy() self.assertTrue(self.queue.is_completed()) self.assertEqual(self.accm.default_pool.n_accounts(), 0) def testReset(self): self.testAddAccount() self.queue.reset() self.assertEqual(self.accm.default_pool.n_accounts(), 0) def testExceptionCallback(self): self.exc = {} def my_exc_cb(jobname, exc_info): self.exc[jobname] = exc_info self.createQueue(exc_cb=my_exc_cb) self.queue.run('dummy://mytest', error) self.queue.join() self.assertIn("mytest", self.exc) self.assertIsInstance(self.exc["mytest"][1], FailException) def testRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] func = bind(count_calls2, data, testarg=1) self.queue.run(hosts, func) self.queue.run('dummy://dummy3', func) self.queue.shutdown() self.assertEqual(data.value, 3) self.queue.run('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 4) def testRunOrIgnore(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1'] func = bind(count_calls2, data, testarg=1) self.queue.workqueue.pause() self.queue.run_or_ignore(hosts, func) self.queue.run_or_ignore('dummy://dummy2', func) self.queue.workqueue.unpause() self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.run_or_ignore('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 3) def testPriorityRun(self): def write(data, value, *args): data.value = value data = Value('i', 0) self.queue.workqueue.pause() self.queue.enqueue(partial(write, data, 1)) self.queue.priority_run('dummy://dummy', partial(write, data, 2)) self.queue.workqueue.unpause() self.queue.destroy() # The 'dummy' job should run first, so the value must # be overwritten by the other process. self.assertEqual(data.value, 1) def testPriorityRunOrRaise(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1'] func = bind(count_calls2, data, testarg=1) self.queue.workqueue.pause() self.queue.priority_run_or_raise(hosts, func) self.queue.priority_run_or_raise('dummy://dummy2', func) self.queue.workqueue.unpause() self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.priority_run_or_raise('dummy://dummy4', func) self.queue.destroy() self.assertEqual(data.value, 3) def testForceRun(self): data = Value('i', 0) hosts = ['dummy://dummy1', 'dummy://dummy2'] func = bind(count_calls2, data, testarg=1) # By setting max_threads to 0 we ensure that the 'force' part is # actually tested; the thread should run regardless. self.queue.set_max_threads(0) self.queue.force_run(hosts, func) self.queue.destroy() self.assertEqual(data.value, 2) def testEnqueue(self): data = Value('i', 0) func = bind(count_calls, data, testarg=1) self.queue.enqueue(func) self.queue.enqueue(func) self.queue.shutdown() self.assertEqual(data.value, 2) self.queue.enqueue(func) self.queue.shutdown() self.assertEqual(data.value, 3) func = bind(count_and_fail, data, testarg=1) self.queue.enqueue(func, attempts=7) self.queue.destroy() self.assertEqual(data.value, 10) # FIXME: Not a method test; this should probably be elsewhere. def testLogging(self): task = self.startTask() while not task.is_completed(): time.sleep(.1) # The following function is not synchronous with the above, so add # a timeout to avoid races. time.sleep(.1) self.assertTrue(self.queue.is_completed()) logfiles = os.listdir(self.tempdir) self.assertEqual(2, len(logfiles)) self.assertIn('dummy1.log', logfiles) self.assertIn('dummy2.log', logfiles) for file in logfiles: with open(os.path.join(self.tempdir, file)) as fp: content = fp.read()
import time, Exscript.util.sigintcatcher from Exscript import Queue, Account from Exscript.util.decorator import bind from TkExscript import QueueWindow def do_something(conn, wait): conn.connect() conn.authenticate() for i in range(100): conn.execute('test%d' % i) time.sleep(wait) conn.close() queue = Queue(max_threads=4, verbose=0) queue.add_account(Account('test', 'test')) window = QueueWindow(queue) queue.run('dummy://dummy1', bind(do_something, .02)) queue.run('dummy://dummy2', bind(do_something, .2)) queue.run('dummy://dummy3', bind(do_something, .3)) window.mainloop() queue.shutdown()
from Exscript import Queue, Logger from Exscript.util.log import log_to from Exscript.util.decorator import autologin from Exscript.util.file import get_hosts_from_file, get_accounts_from_file from Exscript.util.report import status, summarize logger = Logger() # Logs everything to memory. @log_to(logger) @autologin() def do_something(job, host, conn): conn.execute('show ip int brie') # Read input data. accounts = get_accounts_from_file('accounts.cfg') hosts = get_hosts_from_file('hostlist.txt') # Run do_something on each of the hosts. The given accounts are used # round-robin. "verbose=0" instructs the queue to not generate any # output on stdout. queue = Queue(verbose=5, max_threads=5) queue.add_account(accounts) # Adds one or more accounts. queue.run(hosts, do_something) # Asynchronously enqueues all hosts. queue.shutdown() # Waits until all hosts are completed. # Print a short report. print status(logger) print summarize(logger)