def upload(jsondata,coorddata):
def do_addnode(job,host,conn,node):
conn.execute('configure')
conn.execute('set devices authgroups group auth-cisco default-map remote-name cisco remote-password cisco remote-secondary-password cisco')
conn.execute('commit')
conn.execute('set devices device '+node['NodeName']+' address '+node['managementIP']+' port '+portmapping[node['managementProtocol']]+' authgroup auth-cisco device-type cli ned-id '+iosmapping[node["NodeSubtype"]]+' protocol '+node['managementProtocol'] )
conn.execute('commit')
conn.execute('set devices device '+node['NodeName']+' state admin-state unlocked ')
conn.execute('commit')
conn.execute('request devices device '+node['NodeName']+' sync-from')
def get_int_info(device,connex):
for node in jsondata:
if node["NodeName"]==device :
for int in node["Interface"]:
if(connex==int['network']):
return int['name'], int['ip-address']
def do_addconnection(job,host,conn,node1,node2,interface):
info1=get_int_info(node1,interface)
info2=get_int_info(node2,interface)
conn.execute('configure')
try:
if info1[1]=="None":
conn.execute('set topology connection '+node1+"_"+node2+' endpoint-1 device '+node1+" interface "+info1[0])
conn.execute('set topology connection '+node1+"_"+node2+' endpoint-2 device '+node2+" interface "+info2[0])
else:
conn.execute('set topology connection '+node1+"_"+node2+' endpoint-1 device '+node1+" interface "+info1[0]+ " ip-address "+info2[1])
conn.execute('set topology connection '+node1+"_"+node2+' endpoint-2 device '+node2+" interface "+info2[0]+ " ip-address "+info2[1])
conn.execute('commit')
except Exception as e:
print e
print info1,info2
def do_addicons(job,host,conn,node,coordx,coordy):
conn.execute("configure")
conn.execute('set webui icons device '+node['NodeName']+' disabled large icon cisco-disabled')
conn.execute('set webui icons device '+node['NodeName']+' enabled large icon cisco-enabled')
conn.execute('set webui data-stores static-map device '+node['NodeName']+' coord x '+str(coordx)+' y '+str(coordy)+'')
conn.execute('commit')
for node in jsondata :
for i in range (0,len(coorddata)):
if coorddata[i][0]==node['NodeName']:
coordx=coorddata[i][1]
coordy=coorddata[i][2]
break
start(account, host, bind(do_addnode,node))
start(account, host, bind(do_addicons,node,coordx,coordy))
for interfaces in node['Interface']:
split=interfaces['network'].split("-to-",2)
if len(split) ==2:
start(account, host, bind(do_addconnection,split[0],split[1],interfaces['network']))
def setUp(self):
from Exscript import Account
from Exscript.util.decorator import bind
self.data = Value('i', 0)
self.callback = bind(count_calls, self.data, testarg=1)
self.account = Account('test', 'test')
def testEnqueue(self):
data = Value('i', 0)
func = bind(count_calls, data, testarg=1)
self.queue.enqueue(func)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
func = bind(count_and_fail, data, testarg=1)
self.queue.enqueue(func, attempts=7)
self.queue.destroy()
self.assertEqual(data.value, 10)
def testEnqueue(self):
data = Value('i', 0)
func = bind(count_calls, data, testarg=1)
self.queue.enqueue(func)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.enqueue(func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
func = bind(count_and_fail, data, testarg=1)
self.queue.enqueue(func, attempts=7)
self.queue.destroy()
self.assertEqual(data.value, 10)
def setUp(self):
from Exscript import Account
from Exscript.util.decorator import bind
self.data = Value('i', 0)
self.callback = bind(count_calls, self.data, testarg = 1)
self.account = Account('test', 'test')
def testForceRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
# By setting max_threads to 0 we ensure that the 'force' part is
# actually tested; the thread should run regardless.
self.queue.set_max_threads(0)
self.queue.force_run(hosts, func)
self.queue.destroy()
self.assertEqual(data.value, 2)
def testForceRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
# By setting max_threads to 0 we ensure that the 'force' part is
# actually tested; the thread should run regardless.
self.queue.set_max_threads(0)
self.queue.force_run(hosts, func)
self.queue.destroy()
self.assertEqual(data.value, 2)
def testRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
self.queue.run(hosts, func)
self.queue.run('dummy://dummy3', func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
self.queue.run('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 4)
def testRun(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2']
func = bind(count_calls2, data, testarg=1)
self.queue.run(hosts, func)
self.queue.run('dummy://dummy3', func)
self.queue.shutdown()
self.assertEqual(data.value, 3)
self.queue.run('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 4)
def testPriorityRunOrRaise(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.priority_run_or_raise(hosts, func)
self.queue.priority_run_or_raise('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.priority_run_or_raise('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testPriorityRunOrRaise(self):
data = Value('i', 0)
hosts = ['dummy://dummy1', 'dummy://dummy2', 'dummy://dummy1']
func = bind(count_calls2, data, testarg=1)
self.queue.workqueue.pause()
self.queue.priority_run_or_raise(hosts, func)
self.queue.priority_run_or_raise('dummy://dummy2', func)
self.queue.workqueue.unpause()
self.queue.shutdown()
self.assertEqual(data.value, 2)
self.queue.priority_run_or_raise('dummy://dummy4', func)
self.queue.destroy()
self.assertEqual(data.value, 3)
def testTemplates(self):
callback = bind(log_to(self.logger)(dummy_cb), self)
for test in os.listdir(test_dir):
pseudo = os.path.join(test_dir, test, 'pseudodev.py')
if os.path.exists(pseudo):
self.queue.run('pseudo://' + pseudo, callback)
else:
self.queue.run('ios://' + test, callback)
self.queue.shutdown()
# Unfortunately, unittest.TestCase does not fail if self.assert()
# was called from a subthread, so this is our workaround...
failed = self.logger.get_aborted_logs()
report = format(self.logger, show_successful = False)
self.assert_(not failed, report)
def testTemplates(self):
callback = bind(log_to(self.logger)(dummy_cb), self)
for test in os.listdir(test_dir):
pseudo = os.path.join(test_dir, test, 'pseudodev.py')
if os.path.exists(pseudo):
self.queue.run('pseudo://' + pseudo, callback)
else:
self.queue.run('ios://' + test, callback)
self.queue.shutdown()
# Unfortunately, unittest.TestCase does not fail if self.assert()
# was called from a subthread, so this is our workaround...
failed = self.logger.get_aborted_logs()
report = format(self.logger, show_successful=False)
self.assert_(not failed, report)
def testBind(self):
from Exscript.util.decorator import bind
bound = bind(self.bind_cb, 'one', 'two', three = 3)
result = bound(FakeJob())
self.assert_(result == 123, result)
def main():
global config, opts
"""
Main runtime routine
"""
load_config()
load_options()
# Read system configuration from file
groups = config.get("system", "groups")
if opts.quiet:
sys.stderr = open(os.devnull, "w")
for group in groups.split():
if opts.verbose > 0:
print "Group %s found" % group
section = "_%s" % group
g = {}
if config.has_option(section, "name"):
g["_name"] = config.get(section, "name", group)
else:
g["_name"] = config.get("group_default", "name", group)
if config.has_option(section, "devfile"):
g["devfile"] = config.get(section, "devfile")
else:
g["devfile"] = config.get("group_default", "devfile")
if config.has_option(section, "filename"):
g["filename"] = config.get(section, "filename")
else:
g["filename"] = config.get("group_default", "filename")
g["name"] = Template(g["_name"]).safe_substitute({"group": group})
# g['filename'] = Template(g["_filename"]).safe_substitute({'group': group, 'name': 'testfilename' , 'address': '127.0.0.1'})
g["git_dir"] = os.path.join(os.path.realpath(config.get("system", "gitrepo")), g["name"])
g["routerdb"] = os.path.join(os.path.realpath(config.get("system", "gitrepo")), g["name"], g["devfile"])
if config.has_option(section, "loginrc"):
g["_loginrc"] = config.get(section, "loginrc")
else:
g["_loginrc"] = config.get("group_default", "loginrc")
g["loginrc"] = os.path.realpath(Template(g["_loginrc"]).safe_substitute({"basedir": basedir, "group": group}))
if opts.group: # IF parameter group spesified, only run on devices in that group
if not mfnmatch.longest(group, opts.group.split(",")) and not mfnmatch.longest(
g["name"], opts.group.split(",")
):
continue
config_groups[group] = g
# Reading router.db file for all groups
for _g in config_groups:
g = config_groups[_g]
config_groups[_g]["devices"] = {}
devices = {}
# print "%s : %s" % (_g, g)
if not os.path.isfile(g["routerdb"]):
sys.stderr.write("ERROR: device database for group %s not found\n" % _g)
continue
# Load LoginRC file for group
try:
loginrc = pycid.loginrc(g["loginrc"])
# print loginrc.get_all()
except Exception as e:
sys.stderr.write("ERROR: Unable to load loginrc for group %s (%s)\n" % (_g, e))
continue
with open(g["routerdb"], "r") as f:
for l in f.readlines():
dev = {}
line = l.strip()
if not line.startswith("#") and line:
e = line.split(";")
try:
if not e[2] in pycid.workflow.driver_map:
sys.stderr.write("ERROR: workflow %s not found on node %s:%s\n" % (e[2], _g, e[0]))
continue
if opts.device: # IF parameter "--device" spesified, only run on device with matching name
if not mfnmatch.longest(e[0], opts.device.split(",")):
continue
dev["name"] = e[0]
dev["host"] = Host(e[1])
dev["workflow_name"] = e[2]
# dev['workflow'] = pycid.workflow.Workflow(pycid.workflow.driver_map[e[2]])
opt = e[3] if e[3].startswith("{") else "{%s}" % e[3]
# print opt
dev["options"] = json.loads(opt)
dev["configfile"] = os.path.join(
g["git_dir"],
Template(g["filename"]).safe_substitute({"group": _g, "name": e[0], "address": e[1]}),
)
except Exception as e:
sys.stderr.write("ERROR: error parsing line: %s (%s)\n" % (line, e))
continue
# print dev['host'].get_account()
# print "Account"
if not dev["host"].get_account(): # Account not spesifies on device, get best match from loginrc
dev["host"].set_account(
Account(
loginrc.get("username", dev["host"].get_address()),
loginrc.get("password", dev["host"].get_address()),
)
)
if not "authuser" in dev["options"].keys():
dev["options"]["authuser"] = loginrc.get("authuser", dev["host"].get_address())
if not "authpass" in dev["options"].keys():
dev["options"]["authpass"] = loginrc.get("authpass", dev["host"].get_address())
dev["options"]["auth_account"] = Account(dev["options"]["authuser"], dev["options"]["authpass"])
devices[e[0]] = dev
config_groups[_g]["devices"] = devices
# Creating a Exscript queue and redirects all errors to systems devnull, errors are shown in log instead
threads = config.get("system", "max_threads")
if opts.threads:
threads = opts.threads
try:
threads = int(threads)
except Exception as e:
sys.stderr.write("ERROR: maximum threads, value(%s) is not a number: %s\n" % (threads, e))
threads = 1
queue = Queue(
verbose=1 if opts.verbose > 1 else 0,
max_threads=threads,
stderr=sys.stderr
# stderr = open(os.devnull,'w') if opts.quiet else sys.stderr
)
filelog = log_to_file("../log")
# running workflow on all devices in all groups
jobid = 0
for _g in config_groups:
for _d in config_groups[_g]["devices"]:
jobid += 1
d = config_groups[_g]["devices"][_d]
d["job_id"] = jobid
host = d["host"]
workflow = d["workflow_name"]
# Activate logging and save log to memory, log will be connected to device in config_groups
log = Logger()
logdecorator = log_to(log)
if opts.verbose > 1:
print "RUN: queueing %s:%-20s (jobid: %s)" % (_g, _d, jobid)
d["log"] = log
sleep(
0.2
) # fixup! Some devices failes when threads are started to fast, sleeps 200ms between each thread-start
d["task"] = queue.run(
host,
logdecorator(
bind(
pycid.workflow.driver_map[workflow]._command_runner, # Command to execute
(_g, _d, jobid, d["options"]), # jobinfo
excallback, # Callback function
)
),
)
# Waiting for all queues to finish
if opts.verbose > 1:
print "Waiting for all threads to finish!"
queue.join()
# Print logs and save configfiles
for _g in config_groups:
for _d in config_groups[_g]["devices"]:
d = config_groups[_g]["devices"][_d]
file = d["configfile"]
status = d["log"].get_succeeded_actions()
# print "%s %s" % (d['log'].get_succeeded_actions(), d['log'].get_aborted_actions())
if d["log"].get_succeeded_actions() > 0:
try:
with open(file, "w") as out:
out.write(d["result"])
except Exception as e:
sys.stderr.write(
"ERROR: Unable to save configuration for node %s on group %s to file %s (%s)\n"
% (_d, _g, file, e)
)
else:
if opts.verbose > 0:
print "Save %s:%s to %s" % (_g, _d, file)
elif d["log"].get_aborted_actions() > 0:
sys.stderr.write(
"ERROR: No collected data from %s:%s (%s)\n" % (_g, _d, d["log"].get_aborted_logs()[0].exc_info[1])
)
else:
sys.stderr.write("ERROR: Device not processed %s:%s\n" % (_g, _d))
if opts.ipyshell:
IPython.embed()
# Destroys work queue and cleans up variables
queue.destroy()
if opts.verbose > 2:
pprint(config_groups)
from Exscript.protocols import connect
from Exscript.util.decorator import bind
from Exscript import Queue, Account, Host
objnames = ('count_calls',)
follow_modules = False
def count_calls(conn, thedata, **kwargs):
thedata['n_calls'] += 1
def foobar():
pass
queue = Queue()
data = {'n_calls': 0}
func = bind(count_calls, data)
task = queue.run(['t1', 't2', 't3', 't4', 't5', 't6'], func)
task.wait()
queue.shutdown()
queue.destroy()
del func
# Test memory consumption.
from meliae import scanner
gc.collect()
scanner.dump_all_objects("test.dump")
from meliae import loader
om = loader.load('test.dump')
om.remove_expensive_references()
om.collapse_instance_dicts()
def testBind(self):
from Exscript.util.decorator import bind
bound = bind(self.bind_cb, 'one', 'two', three=3)
result = bound(FakeJob())
self.assertEqual(result, 123)
import time, Exscript.util.sigintcatcher
from Exscript import Queue, Account
from Exscript.util.decorator import bind
from TkExscript import QueueWindow
def do_something(conn, wait):
conn.connect()
conn.authenticate()
for i in range(100):
conn.execute('test%d' % i)
time.sleep(wait)
conn.close()
queue = Queue(max_threads=4, verbose=0)
queue.add_account(Account('test', 'test'))
window = QueueWindow(queue)
queue.run('dummy://dummy1', bind(do_something, .02))
queue.run('dummy://dummy2', bind(do_something, .2))
queue.run('dummy://dummy3', bind(do_something, .3))
window.mainloop()
queue.shutdown()
def spawn_subtask(job, host, conn, queue, data, **kwargs):
count_calls2(job, host, conn, data, **kwargs)
func = bind(count_calls2, data, testarg=1)
task = queue.priority_run('subtask', func)
task.wait()
def spawn_subtask(job, host, conn, queue, data, **kwargs):
count_calls2(job, host, conn, data, **kwargs)
func = bind(count_calls2, data, testarg=1)
task = queue.priority_run('subtask', func)
task.wait()
objnames = ('count_calls', )
follow_modules = False
def count_calls(conn, thedata, **kwargs):
thedata['n_calls'] += 1
def foobar():
pass
queue = Queue()
data = {'n_calls': 0}
func = bind(count_calls, data)
task = queue.run(['t1', 't2', 't3', 't4', 't5', 't6'], func)
task.wait()
queue.shutdown()
queue.destroy()
del func
# Test memory consumption.
from meliae import scanner
gc.collect()
scanner.dump_all_objects("test.dump")
from meliae import loader
om = loader.load('test.dump')
om.remove_expensive_references()
om.collapse_instance_dicts()
def testBind(self):
from Exscript.util.decorator import bind
bound = bind(self.bind_cb, "one", "two", three=3)
result = bound(FakeJob())
self.assert_(result == 123, result)
import time, Exscript.util.sigintcatcher
from Exscript import Queue, Account
from Exscript.util.decorator import bind
from TkExscript import QueueWindow
def do_something(conn, wait):
conn.connect()
conn.authenticate()
for i in range(100):
conn.execute('test%d' % i)
time.sleep(wait)
conn.close()
queue = Queue(max_threads = 4, verbose = 0)
queue.add_account(Account('test', 'test'))
window = QueueWindow(queue)
queue.run('dummy://dummy1', bind(do_something, .02))
queue.run('dummy://dummy2', bind(do_something, .2))
queue.run('dummy://dummy3', bind(do_something, .3))
window.mainloop()
queue.shutdown()