def __init__(self, host, port, onreceive, threadpool_size):
# udp socket binding
self.udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# tcp socket binding
self.tcp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.host = host
self.port = port
self.onreceive = onreceive
log.debug('using a threadpool of size: %d' % threadpool_size)
self.threadpool = futures.ThreadPoolExecutor(
max_workers=threadpool_size)
self.pollers = futures.ThreadPoolExecutor(max_workers=1)
self.active_threads = 0
self.udp_socket.bind((host, port))
self.tcp_socket.bind((host, port))
log.info("Server started on %s:%s" % (self.host, self.port))
def benchmark(conf):
loaders = get_running_lcnodes(conf, role='loader')
cass = get_running_lcnodes(conf, role='cass')
cassips = [c.private_ips[-1] for c in cass]
print 'Running with keys=1 to establish keyspace....'
cmd = get_benchcmd(conf, loaders[0], cassips, 'keyspace')
run_cmd(conf, loaders[0], cmd)
print 'Sleeping for 20 seconds to allow keyspace propogation.'
time.sleep(20)
print 'Done!'
print ''
pt = PrettyTable(['name', 'status', 'detail'])
with futures.ThreadPoolExecutor(max_workers=len(loaders)) as e:
returns = {}
for loader in loaders:
cmd = get_benchcmd(conf, loader, cassips, 'benchmark')
returns[loader.name] = e.submit(run_cmd, conf, loader, cmd)
for key in returns.keys():
try:
n = returns[key].result()
pt.add_row([key, 'OK', ''])
except Exception as exc:
traceback.print_exc(file=sys.stdout)
pt.add_row([key, 'EXCEPTION', str(exc)])
print pt
def getresults(conf):
loaders = get_running_lcnodes(conf, role='loader')
pt = PrettyTable(['name', 'status', 'detail'])
with futures.ThreadPoolExecutor(max_workers=len(loaders)) as e:
returns = {}
for loader in loaders:
fname = '/opt/cassandra/' + loader.name + '.results2'
cmd = [
'ssh', '-o', 'StrictHostKeyChecking=no', '-o',
'UserKnownHostsFile=/dev/null',
os_login(conf.loader.image) + '@' + loader.public_ips[-1],
'cat', fname
]
returns[loader.name] = e.submit(run_cmd, conf, loader, cmd)
for key in returns.keys():
try:
n = returns[key].result()
pt.add_row([key, 'OK', ''])
with open(key + '.results', 'w') as fp:
fp.write(n)
except Exception as exc:
traceback.print_exc(file=sys.stdout)
pt.add_row([key, 'EXCEPTION', str(exc)])
print pt
def sshtest(conf):
nodes = get_running_lcnodes(conf)
pt = PrettyTable(['name', 'status', 'detail'])
with futures.ThreadPoolExecutor(max_workers=len(nodes)) as e:
returns = {}
for node in nodes:
cmd = [
'ssh',
'-o',
'StrictHostKeyChecking=no',
'-o',
'UserKnownHostsFile=/dev/null',
os_login(conf.loader.image) + '@' + node.public_ips[-1],
'uptime',
]
returns[node.name] = e.submit(run_cmd, conf, node, cmd)
for key in returns.keys():
try:
n = returns[key].result()
pt.add_row([key, 'OK', n.strip()])
except Exception as exc:
traceback.print_exc(file=sys.stdout)
pt.add_row([key, 'EXCEPTION', str(exc)])
print pt.get_string(sortby="status")
def upload_multiple(file_list, max_workers=5):
"""Upload a bunch of CSV files to Sailthru.
Using Python's concurrent.futures library, asynchronously uploads
the files to Sailthru, a few at a time.
Arguments:
file_list: List of filenames to be uploaded.
max_workers: The numbers of asynchronous uploads to allow at once.
"""
with futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
future_to_file = {
executor.submit(upload, filename): filename
for filename in file_list
}
count = 0
total = len(file_list)
for future in futures.as_completed(future_to_file):
file_name = future_to_file[future]
if future.exception() is not None:
print '%r generated an exception: %s' % (file_name,
future.exception())
else:
# Successful upload
count += 1
print('%s/%s files have been successfully uploaded' %
(count, total))
def __init__(self, settings, domain, args):
self.dst = str(args['destination'])
if not self.dst.endswith('.'):
self.dst += '.'
resolvers = args.get('resolvers', None)
# how long we wait on upstream dns servers before puking
self.timeout = settings.get('resolution_timeout', DEFAULT_TIMEOUT)
log.debug('timeout: %d' % self.timeout)
# Add way to obtain A record for dst
def get_resolver(nameserver):
res = dns.resolver.Resolver(filename=None, configure=False)
res.nameservers = [nameserver]
res.timeout = self.timeout
return res
self.resolvers = map(lambda x: get_resolver(x.strip()),
resolvers.split(','))
log.debug('resolvers: %s' % self.resolvers)
self.pool = futures.ThreadPoolExecutor(max_workers=len(self.resolvers))
super(CNameRule, self).__init__(settings, domain, args)
def main():
user_txt = open(USER_FILE)
for username in user_txt:
task.put(username.rstrip("\n"))
executor = futures.ThreadPoolExecutor(max_workers=THREAD_NUM)
for i in range(THREAD_NUM):
executor.submit(attack)
executor.shutdown()
def solveImages(imagePaths,
channel=None,
maskingFn=maskStarfield,
solveTimeout=60 * 5,
parallel=True,
debugOutputFolder=None,
noAstrometryPlots=False,
pixelError=10,
oddsToSolve=None,
sigma=None,
maxWorkers=None,
astrometryBinPath=None,
useModifiedPath=False,
verbose=False):
"""
Solves multiple images in parallel. See :func:`solveImage` for parameter documentation.
Note: This only works because the actual solving is done in an external
process (astrometry.net), otherwise nothing would have been gained
due to Pythons GIL.
"""
if maxWorkers is None:
maxWorkers = multiprocessing.cpu_count()
if parallel and len(imagePaths) > 1:
workerCount = min(multiprocessing.cpu_count(), maxWorkers)
with futures.ThreadPoolExecutor(max_workers=workerCount) as executor:
for wcsHeader in executor.map(
lambda imagePath: solveImage(
imagePath,
channel=channel,
maskingFn=maskingFn,
solveTimeout=solveTimeout,
debugOutputFolder=debugOutputFolder,
noAstrometryPlots=noAstrometryPlots,
astrometryBinPath=astrometryBinPath,
useModifiedPath=useModifiedPath,
pixelError=pixelError,
oddsToSolve=oddsToSolve,
sigma=sigma,
verbose=verbose), imagePaths):
yield wcsHeader
else:
for wcsHeader in map(
lambda imagePath: solveImage(
imagePath,
channel=channel,
maskingFn=maskingFn,
solveTimeout=solveTimeout,
debugOutputFolder=debugOutputFolder,
noAstrometryPlots=noAstrometryPlots,
astrometryBinPath=astrometryBinPath,
useModifiedPath=useModifiedPath,
pixelError=pixelError,
oddsToSolve=oddsToSolve,
sigma=sigma,
verbose=verbose), imagePaths):
yield wcsHeader
def handler(*args, **kwargs):
logging.info(
'asyncmethod: Method "%s" invoked with args "%s" and kwargs "%s"',
method.__name__, args, kwargs)
with futures.ThreadPoolExecutor(max_workers=1) as executor:
future_result = executor.submit(method, *args, **kwargs)
future_result.add_done_callback(callback)
return future_result
def main():
# fs = {}
with futures.ThreadPoolExecutor(max_workers=100) as executor: #默认10线程
time.clock()
executor.submit(refreshCode)
time.sleep(1)
for i in xrange(130000, 134800):
future = executor.submit(burpCode, str(i))
print time.clock()
def get_page_names(base_url, category):
if isinstance(category, basestring):
category = [category]
with futures.ThreadPoolExecutor(50) as executor:
all_page_names = executor.map(
partial(get_page_names_for_category, base_url),
get_all_categories(base_url, category))
return reduce(set.union, all_page_names)
def make_one(self, pkg='dummy', index_path=None):
from cheeseprism import index
executor = futures.ThreadPoolExecutor(1)
if index_path is None:
index_path = self.new_path('test-index')
idx = index.IndexManager(index_path, executor=executor)
pkg = getattr(self, pkg)
pkg.copy(idx.path)
self.dummypath = idx.path / pkg.name
return idx
def Audit(services):
if services.has_key('url') and services.has_key(
'webserver') and services['webserver'] == 'Tomcat':
url = services['url']
host = None
m = re.match('(http[s]?)://([^:^/]+):?([^/]*)/', url)
if m:
host = m.group(2)
pwddicts = getPwds(host)
# pprint(pwddicts)
fs = {}
time.clock()
# use ProcessPoolExecutor will faster
with futures.ThreadPoolExecutor(max_workers=20) as executor: #默认10线程
time.clock()
for eachname in pwddicts.keys():
for eachpwd in pwddicts[eachname]:
# print 'starting\t',eachname+':'+eachpwd
future = executor.submit(
tomcatcrack,
url,
eachname,
eachpwd,
)
fs[future] = eachname + ':' + eachpwd
# print eachname+':'+eachpwd +' '+str(f.result())
# print time.clock()
logger(time.clock())
# 如何抓取到一个就优雅的退出？
# print len(fs)
# uncompleted_fs = fs
# for future in futures.as_completed(fs):
# url = fs[future]
# if future.exception() is not None:
# print('%r generated an exception: %s' % (url,future.exception()))
# #Regardless of the value of wait, the entire Python program will not exit until all pending futures are done executing.
# # executor.shutdown(wait=False)
# break
# else:
# # print url,'\t',future.result()
# uncompleted_fs.pop(future)
# print len(fs)
# print time.clock()
# for future in uncompleted_fs:
# url = uncompleted_fs[future]
# fg = future.cancel()
# # print 'canceling',url,'\t',fg
# donefs,notdonefs = futures.wait(fs)
# # print notdonefs
# print time.clock()
# 找到一个不立即停止，把所有子进程都跑完，最后返回，所花时间更长
# print time.clock()
logger(time.clock())
def main():
test = httpscan(ADDRESS, TIMES)
#start = time.time()
try:
with futures.ThreadPoolExecutor(max_workers=THREADS) as executor:
tasks = dict((executor.submit(test.scan, port), port)
for port in map(str, xrange(START, STOP)))
futures.as_completed(tasks)
except KeyboardInterrupt:
print "Scan done\n"
finally:
print "Have a nice day"
def __init__(self, settings, domain, args):
resolvers = args.get('resolvers', None)
self.resolvers = map(lambda x: x.strip(), resolvers.split(','))
log.debug('resolvers: %s' % self.resolvers)
# how long we wait on upstream dns servers before puking
self.timeout = settings.get('resolution_timeout', DEFAULT_TIMEOUT)
log.debug('timeout: %d' % self.timeout)
self.pool = futures.ThreadPoolExecutor(max_workers=len(self.resolvers))
super(ResolveRule, self).__init__(settings, domain, args)
def download_pages(page=1):
with futures.ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
# We submit NUM_THREADS tasks at a time since we don't know how many
# pages we will need to download in advance
while True:
l = []
for i in range(NUM_THREADS):
f = executor.submit(download_and_save_page, page)
l.append(f)
page += 1
# Block and stop if we're done downloading the page
if not all(f.result() for f in l):
break
def get_plants(category, base_url):
if category == [] or category is None:
category = SEED_CATEGORIES
if base_url is None:
base_url = WIKIPEDIA_URL
page_names = get_page_names(base_url, category)
logging.info("Got %s page names", len(page_names))
with futures.ThreadPoolExecutor(50) as executor:
pages = executor.map(partial(get_page, base_url), page_names)
plants = map(parse_plant_info, pages)
return filter(None, plants)
def search_results(request):
query = request.params.get('q', '')
with futures.ThreadPoolExecutor(max_workers=3) as executor:
workers = [
executor.submit(dataset.search, query, request),
executor.submit(organization.search, query),
executor.submit(wiki.search, query),
]
results = dict(worker.result() for worker in futures.as_completed(workers))
return templates.render_site('search.html', request, search_query=query, has_ckan=False, **results)
def Audit(services):
global ret
retinfo = {}
output = ''
#print'ok'
if services.has_key('url'):
#print'ok'
output += 'plugin run' + os.linesep
urls = generateUrls(services['url'])
# pprint(urls)
# threads
lock = threading.Lock()
threads = []
maxthreads = 20
# for url in urls:
# th = threading.Thread(target=httpcrack,args=(url,lock))
# threads.append(th)
# i = 0
# while i<len(threads):
# if i+maxthreads >len(threads):
# numthreads = len(threads) - i
# else:
# numthreads = maxthreads
# print 'threads:',i,' - ', i + numthreads
# # start threads
# for j in range(numthreads):
# threads[i+j].start()
# # wait for threads
# for j in range(numthreads):
# threads[i+j].join()
# i += maxthreads
# 改用futures模块
with futures.ThreadPoolExecutor(max_workers=maxthreads) as executor: #默认10线程
future_to_url = dict((executor.submit(httpcrack, url, lock), url)
for url in urls)
if ret != '':
retinfo = {'level':'low','content':ret}
security_warning(str(ret))
#
ret = ''
return (retinfo,output)
def multiScan(self):
with futures.ThreadPoolExecutor(max_workers=self.threads) as executor:
future_to_url = dict((executor.submit(self.scan, target), target)
for target in self.targets)
for future in futures.as_completed(future_to_url):
target = future_to_url[future]
try:
ret = future.result()
except Exception as exc:
print('%r generated an exception: %s' % (target, exc))
# logger('%r generated an exception: %s' % (url, exc))
else:
print('%r returns: %s' % (target, str(ret)))
self.results.append((target[0], target[1], ret))
def test_concurrent_dwt():
# dwt on 1D data calls the Cython dwt_single
# other cases call dwt_axis
for dwt_func, x in zip([pywt.dwt, pywt.dwt2, pywt.dwtn],
[np.ones(8), np.eye(16), np.eye(16)]):
transform = partial(dwt_func, wavelet='haar')
for _ in range(10):
arrs = [x.copy() for _ in range(100)]
with futures.ThreadPoolExecutor(max_workers=max_workers) as ex:
results = list(ex.map(transform, arrs))
# validate result from one of the concurrent runs
expected_result = transform(x)
_assert_all_coeffs_equal([expected_result, ], [results[-1], ])
def test_concurrent_cwt():
time, sst = pywt.data.nino()
dt = time[1]-time[0]
transform = partial(pywt.cwt, scales=np.arange(1, 4), wavelet='cmor1.5-1',
sampling_period=dt)
for _ in range(10):
arrs = [sst.copy() for _ in range(50)]
with futures.ThreadPoolExecutor(max_workers=max_workers) as ex:
results = list(ex.map(transform, arrs))
# validate result from one of the concurrent runs
expected_result = transform(sst)
for a1, a2 in zip(expected_result, results[-1]):
assert_array_equal(a1, a2)
def process(self):
LOGGER.warn('process, start ')
start_time = time.time()
with futures.ThreadPoolExecutor(len(self.collectors)) as tp_executor:
results = {
tp_executor.submit(collector.collect): collector
for collector in self.collectors
}
futures.as_completed(results)
duration = time.time() - start_time
self.next_wake_interval = self._wake_interval - duration
if self.next_wake_interval < 1:
LOGGER.warn('process, poll interval took greater than %is',
duration)
self.next_wake_interval = int(self._wake_interval)
LOGGER.warn(
'process, end in %.2fs, next poll will begin at %is from now',
duration, self.next_wake_interval)
def test_concurrent_swt():
# tests error-free concurrent operation (see gh-288)
# swt on 1D data calls the Cython swt
# other cases call swt_axes
with warnings.catch_warnings():
# can remove catch_warnings once the swt2 FutureWarning is removed
warnings.simplefilter('ignore', FutureWarning)
for swt_func, x in zip([pywt.swt, pywt.swt2, pywt.swtn],
[np.ones(8), np.eye(16), np.eye(16)]):
transform = partial(swt_func, wavelet='haar', level=3)
for _ in range(10):
arrs = [x.copy() for _ in range(100)]
with futures.ThreadPoolExecutor(max_workers=max_workers) as ex:
results = list(ex.map(transform, arrs))
# validate result from one of the concurrent runs
expected_result = transform(x)
_assert_all_coeffs_equal(expected_result, results[-1])
def find_all_devices(self, device_cfg):
devices_mp = dict()
discovery_tp_size = self.config.get('discovery_tp_size', 5)
devices = utils.get_device_list(device_cfg)
with futures.ThreadPoolExecutor(discovery_tp_size) as tp_executor:
results = {
tp_executor.submit(self.find_device, device): device
for device in devices
}
devices = [fut.result() for fut in futures.as_completed(results)]
for device in devices:
if device is not None and device.get(
'ip_address') not in devices_mp:
devices_mp[device.get('ip_address')] = device
LOGGER.info('get_all_devices, device_count [%d]',
len(devices_mp.values()))
self.active_devices = devices_mp.values()
return self.active_devices
def __init__(self, server, workers, limit):
log.info('starting benchmark')
self.server, self.port = server.split(':')
self.limit = limit
log.info('using %d workers' % workers)
# sanity checking the server name:
try:
dns.reversename.from_address(self.server)
except dns.exception.SyntaxError:
raise Exception(
'server name must be an ip not DNS name, you gave us: %s' %
self.server)
log.info('benchmarking host: %s port: %s' % (self.server, self.port))
# build threadpool
self.executor = futures.ThreadPoolExecutor(max_workers=workers)
def stop_workers(*ids):
"""stop_workers
Description
-----------
Stops worker engines.
Parameters
----------
ids: int or list of worker descriptions, optional
The id's of the worker engines to stop. If not provided, all
worker engines in the cluster will be stopped.
Returns
-------
list
A list of dicts describing the engines.
"""
if len(ids) == 0:
ids = [worker["id"] for worker in list_workers()]
if len(ids) == 0:
return
return stop_workers(*ids)
else:
ids_to_stop = [get_id(worker_or_id) for worker_or_id in ids]
base_url = get_base_url()
auth = get_auth()
# The stop requests are done concurrently in a thread pool for
# lower latency.
def stop_worker(id):
return requests.put(
base_url + "/" + str(id) + "/stop",
auth=(auth["user"], auth["password"])
)
pool = futures.ThreadPoolExecutor(THREAD_POOL_SIZE)
responses = [pool.submit(stop_worker, id) for id in ids]
return [x.result() for x in futures.wait(responses)[0]]
def start_reading_changes(self):
# Close the input end, because we will read from the pipe here.
self._changes_in.close()
# The queue to store `persist_changes` tasks on.
self._persist_queue = Queue.Queue(maxsize=self.task_queue_length)
self._persist_executor = futures.ThreadPoolExecutor(
max_workers=1
)
# Start the future for tracking the sequence.
self._seq_tracking_future = self._persist_executor.submit(
self._track_seq
)
# When tracker dies, purge the queue to prevent deadlocks.
# TODO(kris): Sort it out properly.
def purge_queue(_):
while True:
try:
self._persist_queue.get_nowait()
self._persist_queue.task_done()
except Queue.Empty:
break
# Purge the queue if nobody is reading from it anymore.
self._seq_tracking_future.add_done_callback(
purge_queue
)
# Now, read changes until you die.
try:
self.read_changes()
except:
logger.exception('Error reading changes! Terminating.')
# Write a termination sequence to the queue, so that the sequence
# tracker can exit.
logger.debug('Writing termination sequence to persist queue...')
self._persist_queue.put((None, None))
def delete_nodes(conf):
names = get_node_names(conf)
todelete = []
conn = get_conn(conf)
for n in conn.list_nodes():
if n.name in names:
todelete.append(n)
pt = PrettyTable(['delete-success', 'uuid', 'name'])
with futures.ThreadPoolExecutor(max_workers=CONCURRENCY) as e:
returns = []
for node in todelete:
returns.append(e.submit(delete_node, conf, node))
for rv in returns:
try:
n = rv.result()
pt.add_row([n[0], n[1].uuid, n[1].name])
except Exception as exc:
traceback.print_exc(file=sys.stdout)
pt.add_row(['EXCEPTION', '', str(exc)])
print pt
def create_nodes(conf):
osimg = os_flavor(conf.loader.image)
if osimg == 'coreos':
print 'etcd discovery url: %s' % (conf.getDiscoveryUrl())
toboot = get_missing_nodes(conf)
pt = PrettyTable(['state', 'id', 'name', 'public_ip', 'private_ip'])
with futures.ThreadPoolExecutor(max_workers=CONCURRENCY) as e:
returns = []
for ni in toboot:
returns.append(e.submit(create_node, conf, ni))
for rv in returns:
try:
n = rv.result()
pt.add_row(
[n.state, n.id, n.name, n.public_ips, n.private_ips])
except Exception as exc:
traceback.print_exc(file=sys.stdout)
pt.add_row(['EXCEPTION', '', str(exc), '', ''])
print pt