def wait(self):
"""
Wait until all proxy retrieval/collection threads finish running.
"""
futures.wait(self._executing_getters, return_when=futures.ALL_COMPLETED)
futures.wait(self._executing_queues, return_when=futures.ALL_COMPLETED)
def proxy_checker(proxies):
'''
proxies is a list of {key:value}, where the key is the ip of the proxy (including port), e.g., 192.168.1.1:8080, and the value is the type of the proxy (http/https)
'''
logger.info('%d proxies to check'%(len(proxies)))
import multiprocessing as mp
results = []
with futures.ProcessPoolExecutor(max_workers=mp.cpu_count()*10) as executor:
future_to_proxy = {executor.submit(check_proxy, proxy, 30): proxy for proxy in proxies if proxy.values()[0] == 'http'}
for future in future_to_proxy:
future.add_done_callback(lambda f: results.append(f.result()))
logger.info('%d http proxies to check'%(len(future_to_proxy)))
futures.wait(future_to_proxy)
# for future in futures.as_completed(future_to_proxy):
# proxy = future_to_proxy[future]
# try:
# good, proxy_dict = future.result()
# except Exception as exc:
# logger.info('%r generated an exception: %s'%(proxy, exc))
# else:
# if (good):
# good_proxies.append(proxy_dict)
return [p for (good, p) in results if good]
def update_depths(self):
depths = {}
futures = []
for market in self.markets:
futures.append(self.threadpool.submit(self.__get_market_depth,
market, depths))
wait(futures, timeout=20)
return depths
def get_context_data(self, **kwargs):
context = super(ProfileView, self).get_context_data(**kwargs)
widgets = [import_by_path(classname)(self.object) for classname in settings.PROFILE_WIDGETS]
context['widgets'] = [widget for widget in widgets if widget.can_display(self.request.user)]
# Parallelize queries
with futures.ThreadPoolExecutor(max_workers=4) as executor:
workers = [executor.submit(widget.fill_context, context) for widget in widgets]
futures.wait(workers)
return context
def do_update(self, line):
if self.myconfig.CONFIG['routerloopbackips'] == []:
print("Current configuration is empty. First enter configuration mode and load a file or run the wizard.")
else:
print("Using current configuration, \"" + self.myconfig.CONFIG['info']) + "\""
numtoupdate = str(len(self.myconfig.CONFIG['routerloopbackips']))
print("Fetching configuration from " + numtoupdate + " routers...")
def updateThread(ip):
client = paramiko.SSHClient()
client.load_system_host_keys()
key = paramiko.DSSKey.from_private_key_file(self.myconfig.CONFIG['jumpboxsshkey'])
print(ip + '\t: connecting...')
client.connect(self.myconfig.CONFIG['jumpboxhost'], username=self.myconfig.CONFIG['jumpboxsshuser'], pkey=key)
command = 'ssh ' + self.myconfig.CONFIG['routersshuser'] + '@' + ip + ' "show configuration | display xml | except \\"rpc-reply|\<*cli\>|\<banner\>\\" | no-more"'
print(ip + '\t: executing ' + '`' + command + '`')
stdin, stdout, stderr = client.exec_command(str(command), -1)
print(ip + '\t: waiting for data...')
output = stdout.read()
sanitised = '<configuration>\n' + '\n'.join(output.split('\n')[1:])
f = open("./cache/routerconfigs/" + ip + "_config", "w", 1)
f.write(sanitised)
f.close()
print(ip + '\t: retrieved configuration and wrote to file: ./cache/routerconfigs/' + ip + '_config' + '\t(' + str(len(output))) + ' ' + 'bytes)'
executor = concurrentfutures.ThreadPoolExecutor(10)
start = time.time()
futures = [executor.submit(updateThread, ip) for ip in self.myconfig.CONFIG['routerloopbackips']]
concurrentfutures.wait(futures)
end = time.time()
print
print("\n...done!")
print
print('Updated\t\t: ' + str(numtoupdate) + ' configuration files')
diff = end - start
print('Wall time\t: ' + "%.2f" % diff + ' seconds')
print
print
print('Now building XML data structures...')
print
for rtr in self.myconfig.CONFIG['routerloopbackips']:
self.myconfig.CONFIG[rtr] = xml2datacustom.xml_jcfg2data("./cache/routerconfigs/" + rtr + "_config")
print(rtr + '\t: created XML struct')
print
print('...Done')
print
def run(self):
LOGGER.debug("starting the fetcher with max {} workers"
.format(self._max_workers))
kwargs = {'max_workers': self._max_workers}
with futures.ThreadPoolExecutor(**kwargs) as executor:
pending = list()
while not self._shutdown:
# Update pending jobs.
pending = filter(lambda f: not f.done(), pending)
if len(pending) < self._max_workers:
try:
key, url = self._in.get(timeout=self.TIMEOUT)
LOGGER.debug("processing job {} ({})".format(key, url))
future = executor.submit(self._fetch, key, url)
pending.append(future)
except Queue.Empty:
pass
if self._rate is not None:
time.sleep(1.0 / self._rate)
else:
futures.wait(pending, return_when=futures.FIRST_COMPLETED)
def collect(self):
try:
LOGGER.debug('collect, start')
enabled = self.config.get('enable', True)
if not enabled:
LOGGER.warn('collect, data collection is disabled')
return
tp_size = self.config.get('tp_size', 5)
with futures.ThreadPoolExecutor(tp_size) as tp_executor:
for device in self.device_mgr.get_devices():
future = tp_executor.submit(self.collect_data, device)
self.pending_request.append(future)
future.add_done_callback(self.collect_datacb)
futures.wait(self.pending_request)
if LOGGER.isEnabledFor(logging.DEBUG):
self.dump()
for listener in self.listeners:
listener.notify(self.stats, self.poll_time())
self.clear()
except Exception as e:
LOGGER.exception('collect, failed %s', e)
finally:
LOGGER.debug('collect, end')
def launch_workers(n, size="small", engine="sense-ipython-engine", startup_script="", startup_code="", env={}):
"""launch_workers
Description
-----------
Launches worker dashboards into the current cluster.
Parameters
----------
n: int
The number of dashboards to launch.
size: str, optional
The dashboard size, for example "small", "medium" or "large".
engine: str, optional
The name of the `npm <http://npmjs.org>`_ module to use as the engine.
startup_script: str, optional
The name of a Python source file the dashboard should execute as soon as it starts up.
startup_code: str, optional
Python code the dashboard should execute as soon as it starts up. If startup_script is specified, startup_code will be ignored.
env: dict, optional
Environment variables to set in the dashboard.
Returns
-------
list
A list of dashboard dicts of the form described in the `REST API. <http://help.senseplatform.com/api/rest#retrieve-dashboard>`_
"""
request_body = {
"engine": engine,
"size": size,
"startup_script": startup_script,
"startup_code": startup_code,
"env": env,
"master_id": get_master_id()
}
url = get_base_url()
auth = get_auth()
headers = {'Content-type': 'application/json','Accept': 'application/json'}
# The n launch requests are done concurrently in a thread pool for lower
# latency.
def launch_worker(i):
return requests.post(url, data=simplejson.dumps(request_body), auth=(auth["user"], auth["password"]), headers=headers).json()
pool = futures.ThreadPoolExecutor(THREAD_POOL_SIZE)
responses = [pool.submit(launch_worker, i) for i in xrange(n)]
return map(lambda x: x.result(), futures.wait(responses)[0])
def collect(self):
try:
LOGGER.debug('collect, start')
enabled = self.config.get('enable', True)
if not enabled:
LOGGER.warn(
'collect, data collection is disabled')
return
tp_size = self.config.get('tp_size', 5)
with futures.ThreadPoolExecutor(tp_size) as tp_executor:
for device in self.device_mgr.get_devices():
future = tp_executor.submit(self.collect_data, device)
self.pending_request.append(future)
future.add_done_callback(self.collect_datacb)
futures.wait(self.pending_request)
if LOGGER.isEnabledFor(logging.DEBUG):
self.dump()
for listener in self.listeners:
listener.notify(self.stats, self.poll_time())
self.clear()
except Exception as e:
LOGGER.exception('collect, failed %s', e)
finally:
LOGGER.debug('collect, end')
def dispatch(self, message, *args, **extras):
# used for querying dns servers in parallel:
@profile.howfast
def _resolver(message, resolver):
log.debug('sending %s to %s ' % (message,resolver))
return dns.query.udp(message, resolver, timeout=self.timeout).to_wire()
fs = [ self.pool.submit( _resolver, message, resolver) for resolver in self.resolvers ]
result = futures.wait(fs,return_when=futures.FIRST_COMPLETED).done.pop()
if not result.exception():
return result.result()
else:
log.error(result.exception())
raise errors.NetworkError('could not resolve query %s using %s' % (message, self.resolvers))
def dispatch(self, message, *args, **extras):
# return a CNAME plus A/AAAA records for destination of redirect
# used for querying dns servers in parallel:
@profile.howfast
def _resolver(message, resolver):
log.debug('sending %s to %s ' % (message, resolver.nameservers))
return resolver.query(self.dst, "A")
# First get the A/AAA records:
fs = [
self.pool.submit(_resolver, message, resolver)
for resolver in self.resolvers
]
result = futures.wait(fs,
return_when=futures.FIRST_COMPLETED).done.pop()
if not result.exception():
# First make the CNAME part of the response
response = dns.message.make_response(message)
resp_data = dns.rrset.from_text(message.question[0].name,
DEFAULT_TTL, dns.rdataclass.IN,
dns.rdatatype.CNAME, self.dst)
response.answer.append(resp_data)
# Add A/AAAA records to it
r = result.result()
for a in r:
if a.rdclass == dns.rdataclass.IN and (
a.rdtype == dns.rdatatype.A
or a.rdtype == dns.rdatatype.AAAA):
address = a.address
a_rec = dns.rrset.from_text(self.dst, DEFAULT_TTL,
dns.rdataclass.IN, a.rdtype,
address)
response.answer.append(a_rec)
return response.to_wire()
else:
log.error(result.exception())
raise errors.NetworkError('could not resolve query %s using %s' %
(message, self.resolvers))
def stop_workers(*ids):
"""stop_workers
Description
-----------
Stops worker engines.
Parameters
----------
ids: int or list of worker descriptions, optional
The id's of the worker engines to stop. If not provided, all
worker engines in the cluster will be stopped.
Returns
-------
list
A list of dicts describing the engines.
"""
if len(ids) == 0:
ids = [worker["id"] for worker in list_workers()]
if len(ids) == 0:
return
return stop_workers(*ids)
else:
ids_to_stop = [get_id(worker_or_id) for worker_or_id in ids]
base_url = get_base_url()
auth = get_auth()
# The stop requests are done concurrently in a thread pool for
# lower latency.
def stop_worker(id):
return requests.put(
base_url + "/" + str(id) + "/stop",
auth=(auth["user"], auth["password"])
)
pool = futures.ThreadPoolExecutor(THREAD_POOL_SIZE)
responses = [pool.submit(stop_worker, id) for id in ids]
return [x.result() for x in futures.wait(responses)[0]]
def stop_workers(*ids):
"""stop_workers
Description
----------
Stops worker dashboards in the current cluster.
Parameters
----------
ids: int, optional
The id numbers of the worker dashboards to stop. If not provided, all worker
dashboards in the cluster will be stopped.
Returns
-------
list
A list of dicts of the form described in `the REST API. <http://help.senseplatform.com/api/rest#retrieve-dashboard>`_
"""
if len(ids) == 0:
ids = [worker["id"] for worker in list_workers()]
stop_workers(*ids)
else:
base_url = get_base_url()
request_body = {"status": "stopped"}
auth = get_auth()
# The stop requests are done concurrently in a thread pool for
# lower latency.
def stop_worker(id):
return requests.patch(base_url + str(id),
data=request_body,
auth=(auth["user"],
auth["password"])).json()
pool = futures.ThreadPoolExecutor(THREAD_POOL_SIZE)
responses = [pool.submit(stop_worker, id) for id in ids]
return map(lambda x: x.result(), futures.wait(responses)[0])
def dispatch(self, message, *args, **extras):
# used for querying dns servers in parallel:
@profile.howfast
def _resolver(message, resolver):
log.debug('sending %s to %s ' % (message, resolver))
return dns.query.udp(message, resolver,
timeout=self.timeout).to_wire()
fs = [
self.pool.submit(_resolver, message, resolver)
for resolver in self.resolvers
]
result = futures.wait(fs,
return_when=futures.FIRST_COMPLETED).done.pop()
if not result.exception():
return result.result()
else:
log.error(result.exception())
raise errors.NetworkError('could not resolve query %s using %s' %
(message, self.resolvers))
def stop_workers(*ids):
"""stop_workers
Description
----------
Stops worker dashboards in the current cluster.
Parameters
----------
ids: int, optional
The id numbers of the worker dashboards to stop. If not provided, all worker
dashboards in the cluster will be stopped.
Returns
-------
list
A list of dicts of the form described in `the REST API. <http://help.senseplatform.com/api/rest#retrieve-dashboard>`_
"""
if len(ids) == 0:
ids = [worker["id"] for worker in list_workers()]
stop_workers(*ids)
else:
base_url = get_base_url()
request_body = {"status": "stopped"}
auth = get_auth()
# The stop requests are done concurrently in a thread pool for
# lower latency.
def stop_worker(id):
return requests.patch(base_url + str(id), data=request_body, auth=(auth["user"], auth["password"])).json()
pool = futures.ThreadPoolExecutor(THREAD_POOL_SIZE)
responses = [pool.submit(stop_worker, id) for id in ids]
return map(lambda x: x.result(), futures.wait(responses)[0])
V.add_plot({'type': 'raster',
'ids': {0: neu_pub},
#'yticks': range(1, 1+len(neu_out)),
#'yticklabels': range(len(neu_out))
},
'Generic LPU %s' % i, 'Output')
V._update_interval = 50
V.rows = 3
V.cols = 1
V.fontsize = 18
V.out_filename = '%s.avi' % out_name
V.codec = 'libtheora'
V.dt = 0.0001
V.xlim = [0, 1.0]
V.run()
# Run the visualizations in parallel:
with futures.ProcessPoolExecutor() as executor:
fs_dict = {}
for out_name in ['un', 'co']:
res = executor.submit(run, out_name)
fs_dict[out_name] = res
futures.wait(fs_dict.values())
# Report any exceptions that may have occurred:
for k in fs_dict:
e = fs_dict[k].exception()
if e:
print '%s: %s' % (k, e)
def launch_workers(n, cpu, memory, nvidia_gpu=0, kernel="python3", script="",
code="", env={}):
"""
Description
-----------
Launches worker engines into the cluster.
Parameters
----------
n: int
The number of engines to launch.
cpu: float
The number of CPU cores to allocate to the engine.
memory: float
The number of gigabytes of memory to allocate to the engine.
nvidia_gpu: int, optional
The number of GPU's to allocate to the engine.
kernel: str, optional
The kernel. Can be "r", "python2", "python3" or "scala".
script: str, optional
The name of a Python source file the engine should execute as soon as
it starts up.
code: str, optional
Python code the engine should execute as soon as it starts up. If
script is specified, code will be ignored.
env: dict, optional
Environment variables to set in the engine.
Returns
-------
list
A list of dicts describing the engines.
"""
request_body = {
"kernel": kernel,
"cpu": cpu,
"memory": memory,
"nvidia_gpu": nvidia_gpu,
"script": script,
"code": code,
"env": env,
"master_id": get_master_id()
}
url = get_base_url()
auth = get_auth()
headers = {
'Content-type': 'application/json',
'Accept': 'application/json'
}
# The n launch requests are done concurrently in a thread pool for lower
# latency.
def launch_worker(i):
r = requests.post(
url, data=simplejson.dumps(request_body),
auth=(auth["user"], auth["password"]), headers=headers
)
return r.json()
pool = futures.ThreadPoolExecutor(THREAD_POOL_SIZE)
responses = [pool.submit(launch_worker, i) for i in range(n)]
return [x.result() for x in futures.wait(responses)[0]]
def launch_workers(n,
size="small",
engine="sense-ipython-engine",
startup_script="",
startup_code="",
env={}):
"""launch_workers
Description
-----------
Launches worker dashboards into the current cluster.
Parameters
----------
n: int
The number of dashboards to launch.
size: str, optional
The dashboard size, for example "small", "medium" or "large".
engine: str, optional
The name of the `npm <http://npmjs.org>`_ module to use as the engine.
startup_script: str, optional
The name of a Python source file the dashboard should execute as soon as it starts up.
startup_code: str, optional
Python code the dashboard should execute as soon as it starts up. If startup_script is specified, startup_code will be ignored.
env: dict, optional
Environment variables to set in the dashboard.
Returns
-------
list
A list of dashboard dicts of the form described in the `REST API. <http://help.senseplatform.com/api/rest#retrieve-dashboard>`_
"""
request_body = {
"engine": engine,
"size": size,
"startup_script": startup_script,
"startup_code": startup_code,
"env": env,
"master_id": get_master_id()
}
url = get_base_url()
auth = get_auth()
headers = {
'Content-type': 'application/json',
'Accept': 'application/json'
}
# The n launch requests are done concurrently in a thread pool for lower
# latency.
def launch_worker(i):
return requests.post(url,
data=simplejson.dumps(request_body),
auth=(auth["user"], auth["password"]),
headers=headers).json()
pool = futures.ThreadPoolExecutor(THREAD_POOL_SIZE)
responses = [pool.submit(launch_worker, i) for i in xrange(n)]
return map(lambda x: x.result(), futures.wait(responses)[0])
V.add_plot({'type': 'raster',
'ids': {0: neu_pub},
#'yticks': range(1, 1+len(neu_out)),
#'yticklabels': range(len(neu_out))
},
'Generic LPU %s' % i, 'Output')
V._update_interval = 50
V.rows = 3
V.cols = 1
V.fontsize = 18
V.out_filename = 'generic_output_%s.avi' % out_name
V.codec = 'libtheora'
V.dt = 0.0001
V.xlim = [0, 1.0]
V.run()
# Run the visualizations in parallel:
with futures.ProcessPoolExecutor() as executor:
fs_dict = {}
for out_name in ['un', 'co']:
res = executor.submit(run, out_name)
fs_dict[out_name] = res
futures.wait(fs_dict.values())
# Report any exceptions that may have occurred:
for k in fs_dict:
e = fs_dict[k].exception()
if e:
print '%s: %s' % (k, e)
def do_autoscan(self, line):
"""Automatically scan the network and identify any routers present."""
if self.myconfig.CONFIG == {}:
print("Current configuration is empty. First enter configuration mode and load a file or run the wizard.")
else:
print
print(ruler("*"))
print("NETWORK SCAN")
print(ruler("*"))
print("Using current configuration, \"" + self.myconfig.CONFIG['info']) + "\""
print("Scanning for routers in prefixes " + self.myconfig.CONFIG['routerprefixes'])
print(ruler("*"))
print
prefixes = self.myconfig.CONFIG['routerprefixes'].split()
self.myconfig.CONFIG['routerips'] = []
for prefix in prefixes:
network = ipaddr.IPNetwork(prefix)
for ip in network.iterhosts():
self.myconfig.CONFIG['routerips'].append(str(ip))
numscanned = len(self.myconfig.CONFIG['routerips'])
def autoscanThread(ip):
client = paramiko.SSHClient()
client.load_system_host_keys()
key = paramiko.DSSKey.from_private_key_file(self.myconfig.CONFIG['jumpboxsshkey'])
print(ip + '\t: connecting...')
client.connect(self.myconfig.CONFIG['jumpboxhost'], username=self.myconfig.CONFIG['jumpboxsshuser'], pkey=key)
command = 'snmpget -v 2c -c ' + self.myconfig.CONFIG['routersnmpcommunity'] + ' ' + str(ip) + ' 1.3.6.1.4.1.2636.3.1.2.0'
print(ip + '\t: executing ' + "`" + command + "`")
stdin, stdout, stderr = client.exec_command(str(command))
print(ip + '\t: waiting...')
output = stdout.read()
if "Router" in output:
print(ip + '\t: is a router (' + output.split('"')[1] + ')')
else:
print(ip + '\t: is not a router')
# prune from list
self.myconfig.CONFIG['routerips'].remove(ip)
executor = concurrentfutures.ThreadPoolExecutor(10)
start = time.time()
futures = [executor.submit(autoscanThread, ip) for ip in self.myconfig.CONFIG['routerips']]
concurrentfutures.wait(futures)
end = time.time()
numfound = len(self.myconfig.CONFIG['routerips'])
print
print('Scanned\t\t: ' + str(numscanned) + ' IPs')
print('Found\t\t: ' + str(numfound) + ' routers')
diff = end - start
print('Wall time\t: ' + "%.2f" % diff + ' seconds')
print
print("...done!")
print
# Now we've found which IPs belong to routers, we need to determine the unique loopback IPs, ignoring physical interface IPs
self.myconfig.CONFIG['routerloopbackips'] = []
# Tell user what we're doing...
print
print('Now determining unique loopback interface IPs using these physical interface IPs...')
print
def autoscanCleanupThread(ip):
client = paramiko.SSHClient()
client.load_system_host_keys()
key = paramiko.DSSKey.from_private_key_file(self.myconfig.CONFIG['jumpboxsshkey'])
print(ip + '\t: connecting...')
client.connect(self.myconfig.CONFIG['jumpboxhost'], username=self.myconfig.CONFIG['jumpboxsshuser'], pkey=key)
command = 'ssh ' + self.myconfig.CONFIG['routersshuser'] + '@' + ip + ' -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "show configuration interfaces lo0 unit 0 family inet | grep address"'
print(ip + '\t: executing ' + "`" + command + "`")
stdin, stdout, stderr = client.exec_command(str(command))
print(ip + '\t: waiting...')
output = stdout.read()
if "address" in output:
loopback = output.split()[1].split('/')[0]
fqdn = check_output(["host", loopback]).split()[4]
print(ip + '\t: is a physical interface on ' + fqdn[:-1] + ' with loopback address ' + loopback)
self.myconfig.CONFIG['routerloopbackips'].append(str(loopback))
else:
print(ip + '\t: unknown response: ' + output + ' (ignoring...possible phy->lo0 firewall issue)')
start = time.time()
futures = [executor.submit(autoscanCleanupThread, ip) for ip in self.myconfig.CONFIG['routerips']]
concurrentfutures.wait(futures)
end = time.time()
numscanned = len(self.myconfig.CONFIG['routerips'])
foolist = self.myconfig.CONFIG['routerloopbackips']
self.myconfig.CONFIG['routerloopbackips'] = list(set(foolist))
numfound = len(self.myconfig.CONFIG['routerloopbackips'])
print
print('Scanned\t\t: ' + str(numscanned) + ' physical interface IPs')
print('Found\t\t: ' + str(numfound) + ' loopback IPs')
diff = end - start
print('Wall time\t: ' + "%.2f" % diff + ' seconds')