def main(args):
nsxchecker = NsxChecker(args.controller, args.password, args.quiet)
nsxchecker.lswitch = nsxchecker.get_lswitch(args.network)
nsxchecker.lports = nsxchecker.get_lports(nsxchecker.lswitch)
nsxchecker.macs = [lport['mac'] for lport in nsxchecker.lports]
if len(nsxchecker.lports) <= 1:
print "No ports to check!"
return 0
if not nsxchecker.quiet:
print "%s ports on this network. " % len(nsxchecker.lports)
if not nsxchecker.lports:
if not nsxchecker.quiet:
print "Nothing found!"
quit()
if args.full:
out_string = "A full check will is %s src/dests."
print out_string % int(math.pow(len(nsxchecker.lports), 2))
pool.map(nsxchecker.check_port_full, nsxchecker.lports)
else:
pool.map(nsxchecker.check_port, nsxchecker.lports)
if not nsxchecker.quiet:
print "-" * 40
fail_percent = percentage(nsxchecker.fail, nsxchecker.total)
success_percent = percentage(nsxchecker.success, nsxchecker.total)
if success_percent > 0:
out_string = "%s percent successful (network ID: %s)"
print out_string % (success_percent, args.network)
if fail_percent > 0:
out_string = "%s percent fail (network ID: %s)"
print out_string % (fail_percent, args.network)
return 0
def main(args):
nsxchecker = NsxChecker(args.controller, args.password, args.quiet)
nsxchecker.lswitch = nsxchecker.get_lswitch(args.network)
nsxchecker.lports = nsxchecker.get_lports(nsxchecker.lswitch)
nsxchecker.macs = [lport['mac'] for lport in nsxchecker.lports]
if len(nsxchecker.lports) <= 1:
print "No ports to check!"
return 0
if not nsxchecker.quiet:
print "%s ports on this network. " % len(nsxchecker.lports)
if not nsxchecker.lports:
if not nsxchecker.quiet:
print "Nothing found!"
quit()
if args.full:
out_string = "A full check will is %s src/dests."
print out_string % int(math.pow(len(nsxchecker.lports), 2))
pool.map(nsxchecker.check_port_full, nsxchecker.lports)
else:
pool.map(nsxchecker.check_port, nsxchecker.lports)
if not nsxchecker.quiet:
print "-" * 40
fail_percent = percentage(nsxchecker.fail, nsxchecker.total)
success_percent = percentage(nsxchecker.success, nsxchecker.total)
if success_percent > 0:
out_string = "%s percent successful (network ID: %s)"
print out_string % (success_percent, args.network)
if fail_percent > 0:
out_string = "%s percent fail (network ID: %s)"
print out_string % (fail_percent, args.network)
return 0
def start(self):
offset = 1
time.sleep(1)
pool = ThreadPool(THREAD_NUM)
while self.run:
offsets = [i + offset for i in range(10)]
pool.map(self.get_a_list, offsets)
offset += 10
def main():
exchanges = [
'bitbankcc',
'bitfinex',
'bitflyer',
'bitmex',
'coincheck',
'kraken',
'quoinex',
'zaif',
]
clients = {} # type: Dict[str, ClientBase]
pool = gevent.pool.Pool()
for ex in exchanges:
clients[ex] = getattr(coinapi, ex).Client() # type: ClientBase
def get_balance(key: str):
balances = {}
for k, v in clients[key].balance().items():
balances[C_MAP.get(k, k)] = v
return key, balances
def get_tick(tick: str):
instrument, ex = TICKER_MAP[tick]
return instrument, clients[ex].tick(instrument)
ticks = dict(pool.map(get_tick, list(TICKER_MAP)))
balances = dict(pool.map(get_balance, exchanges))
fx_rates = get_fxrates()
currency_totals = defaultdict(lambda: dict(qty=.0, jpy=.0))
totals = {}
for ex, balance in sorted(balances.items()):
ex_totals = defaultdict(lambda: dict(qty=.0, jpy=.0))
for k, v in sorted(balance.items()):
q = v['total']
if not q:
continue
if k in TICKER_MAP:
rate = ticks['{}/JPY'.format(k)]['bid']
else:
rate = fx_rates['{}/JPY'.format(k)]
jpy = q * rate
ex_totals[k]['qty'] += q
ex_totals[k]['jpy'] += jpy
ex_totals['_total']['jpy'] += jpy
currency_totals[k]['qty'] += q
currency_totals[k]['jpy'] += jpy
print('# {}'.format(ex))
pprint(dict(ex_totals))
totals[ex] = ex_totals['_total']['jpy']
print('# currency')
pprint(dict(currency_totals))
totals['_total'] = sum(totals.values())
print('# jpy')
pprint(dict(totals))
def process_season(self, season):
self.uploader.set_season_endpoint(season)
pool = gevent.pool.Pool(5)
game_date_url_list = self.fetch_game_list_for_season(season)
pool.map(self._fetch_parse_and_upload, game_date_url_list)
pool.join()
if len(self.uploader.fetch_keys()) != len(
map(lambda (game_date, _): game_date, self.all_games)):
raise Exception('Some games was not uploaded for season %s!',
season)
def main():
authors = getRequestName()
proxies_list = getProxies()
session = getSession(len(proxies_list) + 5)
fout = open('result.txt', 'a+', encoding = 'utf-8')
pool = gevent.pool.Pool(len(proxies_list))
lock = gevent.lock.Semaphore()
func = functools.partial(multiprocess, lock, session, proxies_list, fout)
pool.map(func, enumerate(authors))
def multiprocess_upload(QueueClass, queue_name, tasks, parallel=True):
if parallel is True:
parallel = mp.cpu_count()
elif parallel <= 0:
raise ValueError("Parallel must be a positive number or zero (all cpus). Got: " + str(parallel))
if parallel == 1:
soloprocess_upload(QueueClass, queue_name, tasks)
return
def capturing_soloprocess_upload(*args, **kwargs):
try:
soloprocess_upload(*args, **kwargs)
except Exception as err:
print(err)
error_queue.put(err)
uploadfn = partial(
capturing_soloprocess_upload, QueueClass, queue_name
)
tasks = _scatter(tasks, parallel)
# This is a hack to get dill to pickle dynamically
# generated classes. This is an important use case
# for when we create iterators with generator __iter__
# functions on demand.
# https://github.com/uqfoundation/dill/issues/56
try:
task = next(item for item in tasks if item is not None)
except StopIteration:
return
cls_module = task.__class__.__module__
task.__class__.__module__ = '__main__'
with pathos.pools.ProcessPool(parallel) as pool:
pool.map(uploadfn, tasks)
task.__class__.__module__ = cls_module
if not error_queue.empty():
errors = []
while not error_queue.empty():
err = error_queue.get()
if err is not StopIteration:
errors.append(err)
if len(errors):
raise Exception(errors)
def loop_task(redis_task_list_name):
_wait_run_task_num = count_task(redis_task_list_name)
logging.info("exists task:%s" % _wait_run_task_num)
pool = multiprocessing.Pool(VIDEO_TO_SENTENCE_THREADS)
logging.info("create multiprocessing pool:%s" % VIDEO_TO_SENTENCE_THREADS)
_all_task = [
get_task(redis_task_list_name) for _ in xrange(_wait_run_task_num)
]
logging.info("collect task done, wait to input pool")
pool.map(single_video_translate_task, _all_task)
logging.info("add task to pool done :%s" % _wait_run_task_num)
pool.close()
pool.join()
logging.info("add task done:%s" % datetime.datetime.now())
return _wait_run_task_num
def subpool_map(pool_size, func, iterable):
""" Starts a Gevent pool and run a map. Takes care of setting current_job and cleaning up. """
if not pool_size:
return [func(*args) for args in iterable]
counter = itertools_count()
current_job = get_current_job()
def inner_func(*args):
next(counter)
if current_job:
set_current_job(current_job)
ret = func(*args)
if current_job:
set_current_job(None)
return ret
start_time = time.time()
pool = gevent.pool.Pool(size=pool_size)
ret = pool.map(inner_func, iterable)
pool.join(raise_error=True)
total_time = time.time() - start_time
log.debug("SubPool ran %s greenlets in %0.6fs" % (counter, total_time))
return ret
def subpool_map(pool_size, func, iterable):
""" Starts a Gevent pool and run a map. Takes care of setting current_job and cleaning up. """
if not pool_size:
return [func(*args) for args in iterable]
counter = itertools_count()
current_job = get_current_job()
def inner_func(*args):
next(counter)
if current_job:
set_current_job(current_job)
ret = func(*args)
if current_job:
set_current_job(None)
return ret
start_time = time.time()
pool = gevent.pool.Pool(size=pool_size)
ret = pool.map(inner_func, iterable)
pool.join(raise_error=True)
total_time = time.time() - start_time
log.debug("SubPool ran %s greenlets in %0.6fs" % (counter, total_time))
return ret
def start_listen_gnsq_map(cls, topic, callback):
import gevent.pool
import gnsq
from gnsq.contrib.queue import QueueHandler
consumer = gnsq.Consumer(
# nsqd_tcp_addresses=['192.168.5.134:4150'],
lookupd_http_addresses=['192.168.5.134:4161'],
# lookupd_http_addresses=['127.0.0.1:4161'],
topic=topic,
channel='channel1',
max_in_flight=16,
)
queue = QueueHandler(maxsize=16)
consumer.on_message.connect(queue)
consumer.start(block=False)
pool = gevent.pool.Pool(16)
pool.map(callback, queue)
def fuzz_top_level(self):
print('[*] TOP-LEVEL DOMAIN FUZZING...')
file_handle = open("./dict/top-level.dict")
content_dict = file_handle.read().split('\n')
if content_dict[-1] == '':
del content_dict[-1]
pool = gevent.pool.Pool(self.threads)
data = pool.map(self.get_ip, content_dict)
pass
def run(self):
"""启动函数"""
self.get_city_info()
# print(self.city_info)
for city_name, city_url in self.city_info.items():
# for city_name in ['郑州', '北京', '上海', '广州', '深圳']:
city_url = self.city_info[city_name]
if '-zhaopin' not in city_url:
city_url = city_url.rstrip('/') + '-zhaopin/'
response = self.get_response(url=city_url, method='GET')
html = etree.HTML(response.text)
district_name = html.xpath(
'//div[@data-type="district"]/a[position()>1]/text()')
item = [{
'city_name': city_name,
'district': name,
'keyword': 'python'
} for name in district_name]
print(item)
pool = gevent.pool.Pool(size=1)
pool.map(self.get_job_info, item)
def archive_articles(self):
log.info('downloading news from {name}'.format(**self.__dict__))
pool = gevent.pool.Pool(self.session.max_connections)
pool.map(self.save_article, self._article_number_generator())
r = self.compress_and_sort_index()
if not r:
clean_up(self.name, self.item.identifier, self.date)
return
self.state[self.name] = max(self.articles_archived)
local_state_fname = '{identifier}_state.json'.format(**self.__dict__)
with open(local_state_fname, 'w') as fp:
json.dump(self.state, fp)
## Item is ready to upload, remove lock.
mbox_fname = '{name}.{date}.mbox.gz'.format(**self.__dict__)
mbox_lck_fname = mbox_fname + '.lck'
shutil.move(mbox_lck_fname, mbox_fname)
log.info('archived and indexed {0} '
'articles from {1}'.format(len(self.articles_archived), self.name))
def parallel_call(
function: Callable,
args: Any,
repeatable_args: Union[Tuple[Any, ...], List[Any]] = None,
fold_list: bool = False,
fold_dict: bool = False,
force_starmap: bool = False,
pool_size: int = 32,
) -> Any:
"""
Execute a function in parallel
:param function: Function to execute
:param args: Args to pass to the function
:param repeatable_args: Repeatable args to pass with the original args
:param fold_list: Compress the results into a 1D list
:param fold_dict: Compress the results into a single dictionary
:param force_starmap: Force system to use Starmap over normal selection process
:param pool_size: How large the gevent pool should be
:return: Results from execution, with modifications if desired
"""
pool = gevent.pool.Pool(pool_size)
if repeatable_args:
extra_args_rep = [itertools.repeat(arg) for arg in repeatable_args]
results = pool.map(lambda g_args: function(*g_args),
zip(args, *extra_args_rep))
elif force_starmap:
results = pool.map(lambda g_args: function(*g_args), args)
else:
results = pool.map(function, args)
if fold_list:
return list(itertools.chain.from_iterable(results))
if fold_dict:
return dict(collections.ChainMap(*results))
return results
def subpool_map(pool_size, func, iterable):
""" Starts a Gevent pool and run a map. Takes care of setting current_job and cleaning up. """
if not pool_size:
return [func(*args) for args in iterable]
counter = itertools_count()
current_job = get_current_job()
def inner_func(*args):
""" As each call to 'func' will be done in a random greenlet of the subpool, we need to
register their IDs with set_current_job() to make get_current_job() calls work properly
inside 'func'.
"""
next(counter)
if current_job:
set_current_job(current_job)
try:
ret = func(*args)
except Exception as exc:
trace = traceback.format_exc()
log.error("Error in subpool: %s \n%s" % (exc, trace))
raise
if current_job:
set_current_job(None)
return ret
def inner_iterable():
""" This will be called inside the pool's main greenlet, which ID also needs to be registered """
if current_job:
set_current_job(current_job)
for x in iterable:
yield x
if current_job:
set_current_job(None)
start_time = time.time()
pool = gevent.pool.Pool(size=pool_size)
ret = pool.map(inner_func, inner_iterable())
pool.join(raise_error=True)
total_time = time.time() - start_time
log.debug("SubPool ran %s greenlets in %0.6fs" % (counter, total_time))
return ret
def subpool_map(pool_size, func, iterable):
""" Starts a Gevent pool and run a map. Takes care of setting current_job and cleaning up. """
if not pool_size:
return [func(*args) for args in iterable]
counter = itertools_count()
current_job = get_current_job()
def inner_func(*args):
""" As each call to 'func' will be done in a random greenlet of the subpool, we need to
register their IDs with set_current_job() to make get_current_job() calls work properly
inside 'func'.
"""
next(counter)
if current_job:
set_current_job(current_job)
try:
ret = func(*args)
except Exception as exc:
trace = traceback.format_exc()
log.error("Error in subpool: %s \n%s" % (exc, trace))
raise
if current_job:
set_current_job(None)
return ret
def inner_iterable():
""" This will be called inside the pool's main greenlet, which ID also needs to be registered """
if current_job:
set_current_job(current_job)
for x in iterable:
yield x
if current_job:
set_current_job(None)
start_time = time.time()
pool = gevent.pool.Pool(size=pool_size)
ret = pool.map(inner_func, inner_iterable())
pool.join(raise_error=True)
total_time = time.time() - start_time
log.debug("SubPool ran %s greenlets in %0.6fs" % (counter, total_time))
return ret
def get_keyword(url):
req = requests.get(url, headers=HEADERS)
html = req.text
soup = bs(html, 'html.parser')
keywords = soup.select(
'#PM_ID_ct > div.header > div.section_navbar > div.area_hotkeyword.PM_CL_realtimeKeyword_base > div.ah_list.PM_CL_realtimeKeyword_list_base > ul > li > .ah_a'
)
answer = {'url': [], 'rank': [], 'word': [], 'related': []}
for keyword in keywords:
answer['url'].append(keyword.get('href'))
answer['rank'].append(keyword.select('.ah_r')[0].get_text())
answer['word'].append(keyword.select('.ah_k')[0].get_text())
answer['related'] = list(pool.map(get_relateds, keywords))
return answer
def crawl(url, pool_count=1000):
"""Takes a url and prints to stdout a list of static assets and links
on each linked page in the domain as a JSON object
{
"Link: {
"Links": [],
"Static Assets": [
]
}
}
"""
consider_links = []
seen_links = set()
sitemap = {}
pool = gevent.pool.Pool(pool_count)
consider_links.append(url)
while len(consider_links) > 0:
results = pool.map(makeConn, consider_links)
seen_links.update(consider_links)
del consider_links[:]
for c in results:
if c is None:
continue
assets = c.getAssets()
seen_links.update(assets["Static"])
sitemap[c.parent_link] = {"Static Assets": assets[
"Static"], "Links": assets["Links"]}
for link in assets["Consider_Links"]:
if link not in seen_links:
consider_links.append(link)
return sitemap
def reqNgram(pattern_list):
log("hey")
chunks = [connect(pattern_list[x:x + 10])
for x in xrange(0, len(pattern_list), 10)]
pool = gevent.pool.Pool(5)
results = pool.map(runQuery, chunks)
big_dict = {}
for dic in results:
for d in dic:
log(d)
if "(ALL)" in d:
d = d.replace("(ALL)", "")
if d in big_dict:
big_dict[d] += dic[d]
else:
big_dict[d] = dic[d]
return big_dict
gevent.monkey.patch_all()
import sys
import requests
import gevent
import gevent.pool
import random
import json
def consume(id):
r = requests.get('http://127.0.0.1:5000/stream/' + str(id), stream=True)
for line in r.iter_lines(10):
try:
obj = json.loads(line)
except:
print line
raise
if random.randint(0, 99) <= 1:
print obj
POOL_SIZE = 1000
if __name__ == '__main__':
pool = gevent.pool.Pool(POOL_SIZE)
greenlets = pool.map(consume, xrange(POOL_SIZE))
gevent.joinall(greenlets)
def getBzhanInfoByKeywords(keyword='曾艳芬'):
"""
保存所有封面，返回值是一个json，结构为[{"imgURL": imgURL, "shareURL": flash地址, "title": 标题, 'up': UP主},……]
:type keywords: str
:return: json
"""
k = urlencode({'keyword': keyword})
baseURL = 'http://search.bilibili.com/ajax_api/video?%s&order=totalrank&page=' % k
if not os.path.exists('img'):
os.mkdir('img')
# 预处理
preNum = 1
preR = requests.get(baseURL + str(preNum))
prerResult = preR.json()
numPages = int(prerResult['numPages'])
pool = gevent.pool.Pool(20)
def downloadImg(imgURL):
"""
:type imgURL: str
:return: None
"""
r = requests.get(imgURL)
index = imgURL.index('archive') + 8
saveName = str(imgURL[index:])
with open(os.path.join('img', str(saveName)), 'wb') as f:
f.write(r.content)
def getInfoFromLi(li):
soupTemp = BeautifulSoup(str(li), "html.parser")
avURL = str(soupTemp.a['href'])
_avIndex = avURL.index('av')
avCode = avURL[_avIndex + 2:-1]
shareURL = 'http://static.hdslb.com/miniloader.swf?aid=%s&page=1' % avCode
title = soupTemp.img['title']
imgURL = soupTemp.img['src']
upSpan = soupTemp.find_all('span')[-1]
try:
up = BeautifulSoup(str(upSpan), "html.parser").a.string
except:
up = 'None'
return {
'shareURL': shareURL,
'title': title,
'imgURL': imgURL,
'up': up
}
def getInfoFromHTML(html):
soup = BeautifulSoup(html, "html.parser")
info = soup.find_all('li')
return info
result = []
urls = []
for i in range(1, numPages + 1):
url = baseURL + str(i)
urls.append(url)
print('parsing......')
rs = pool.map(requests.get, urls)
for r in rs:
html = r.json()['html']
lis = getInfoFromHTML(html)
for li in lis:
temp = getInfoFromLi(li)
result.append(temp)
try:
# pool.spawn(downloadImg(temp['imgURL']))
print(temp['imgURL'])
pass
except:
print('error with %s' % temp['imgURL'])
print('ok!!!!!')
pool.join()
return json.dumps(result)
N = 1
PoolNum = 500
for i in range(N):
print('start %d try' % i)
#for status, data in jobs:
# print(status, data[:10])
#tic(urllibT, 'urllib.request')
#jobs = [download_requests(url) for url in urls]
#for status, data in jobs:
# print(status, data[:10])
#tic(requestsT, 'requests')
pool = gevent.pool.Pool(PoolNum)
data = pool.map(download_requests, urls)
#for status, text in data:
# print(status, text[:10])
#tic(requestsT, 'requests with gevent.pool')
print "gevent...."
jobs = [gevent.spawn(download_requests, url) for url in urls]
gevent.joinall(jobs)
#for i in jobs:
# print(i.value[0], i.value[1][:10])
#tic(requestsT, 'requests with gevent.spawn')
'''
import matplotlib.pyplot as plt
x = list(range(1, N+1))
plt.plot(x, urllibL, label='urllib')
plt.plot(x, requestsL, label='requests')
# -*- coding: utf-8 -*-
from gevent import monkey, spawn, joinall, pool
monkey.patch_all() # 遇到阻塞时自动切换协程
def run_task(num):
print("得到数字%s, 平方为%s" % (num, pow(num, 2)))
if __name__ == '__main__':
pool = pool.Pool(2)
nums = [i for i in range(5)]
# 进程池的map()函数将第二个参数中的值一个个传入第一个参数表示函数所需要的参数中
pool.map(run_task, nums)
count += len([x for x in gf if x]) - 1
Q.put(count)
def get_index_urls(item):
for f in item.iter_files():
if f.format == "Comma-Separated Values GZ":
yield "http://archive.org/download/{0}/{1}".format(item.identifier, f.name)
if __name__ == "__main__":
identifier = sys.argv[-1]
item = get_item(identifier)
pool = gevent.pool.Pool(40)
pool.map(get_gzip_file_from_url, get_index_urls(item))
_imagecount = 0
while not Q.empty():
_imagecount += Q.get()
if _imagecount == int(item.metadata.get("imagecount", 0)):
sys.stdout.write("{0} - imagecount is up to date\n".format(item.identifier))
sys.exit(0)
md = dict(imagecount=_imagecount)
r = item.modify_metadata(md)
if r.status_code == 200:
sys.stdout.write("{0} - imagecount is up to date\n".format(item.identifier))
sys.exit(0)
else:
# dst = RedirectDst(dst_host, dst_port)
# dst.send(cmd)
print('redirect: ', cmd)
class RedirectDst(object):
def __init__(self, host, port):
self.host = host
self.port = port
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.connect((host, port))
def send(self, data):
self.s.send(data)
if __name__ == '__main__':
_, host, port, dst_host, dst_port = sys.argv
port = int(port)
cluster = RedisCluster.from_node(host, port)
masters = [r for r in cluster.redis_list if r.is_master()]
fakeslaves = [FakeSlave(m.ip, m.port) for m in masters]
fakeslaves = [
FakeSlave(m.ip, m.port) for m in masters
if m.ip == host and m.port == port
][:1]
print(fakeslaves)
pool = gevent.pool.Pool(len(fakeslaves))
pool.map(lambda s: s.loop(), fakeslaves)
return ret
def inner_iterable():
""" This will be called inside the pool's main greenlet, which ID also needs to be registered """
if current_job:
set_current_job(current_job)
for x in iterable:
yield x
if current_job:
set_current_job(None)
start_time = time.time()
pool = gevent.pool.Pool(size=pool_size)
ret = pool.map(inner_func, inner_iterable())
pool.join(raise_error=True)
total_time = time.time() - start_time
log.debug("SubPool ran %s greenlets in %0.6fs" % (counter, total_time))
return ret
def subpool_imap(pool_size, func, iterable, flatten=False, unordered=False, buffer_size=None):
""" Generator version of subpool_map. Should be used with unordered=True for optimal performance """
if not pool_size:
for args in iterable:
yield func(*args)
'''
resp = requests.get(url2)
global i
i+=1
print i
data = resp.text
print('%d bytes received from %s.' % (len(data), url2))+" "+time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
#print
'''
gevent.joinall([
gevent.spawn(f, 'http://www.wsgjp.com.cn/'),
gevent.spawn(f, 'http://www.wsgjp.com.cn/'),
gevent.spawn(f, 'http://www.wsgjp.com.cn/'),
])
'''
#urls=["http://www.wsgjp.com.cn/"]*300
urls=["http://dba.wsgjp.com.cn/"]*300
print "pool"
pool = gevent.pool.Pool(300)
data = pool.map(f, urls)
print "no pool"
jobs = [gevent.spawn(f, url) for url in urls]
gevent.joinall(jobs)
print i
downloaded = []
if tries > 0:
print '--- Retry attempt {0} of {1} ---'.format(tries, retries)
# Remove existing files
if args.overwrite is None:
new_imdb_profiles = filter(functools.partial(_not(profile_exists), args.outdir),
imdb_profiles)
else:
new_imdb_profiles = imdb_profiles
if len(new_imdb_profiles) == 0:
# Nothing to download
break
results = pool.map(download_imdb_csv, new_imdb_profiles)
pool.join()
for response, profile, error in results:
try:
if error is not None:
raise Exception(error)
# Save
savefile = os.path.join(args.outdir, '{0}.csv'.format(profile['username']))
with open(savefile, 'wb') as saveto:
saveto.write(response)
downloaded.append(profile)
except Exception as e:
failed.append(profile)
tries += 1
(count, tv_type, item_movie_info['title']))
count += 1
if count >= 100000:
return True
def craw(self, tv_type, page):
if page is not None:
self.craw_dianyinggang_tv(tv_type, page)
return
print(movie_type)
obj_spider = SpiderMain()
#放在队列里的数据
all_items = obj_spider.get_all_craw_url_item_url(movie_type)
pool = gevent.pool.Pool(30)
requestsT = startTimer()
"""
for item in all_items:
pool.add(gevent.spawn(obj_spider.craw_dianyinggang_tv, item))
pool.join()
"""
data = pool.map(obj_spider.craw_dianyinggang_tv, all_items)
print("gevent spawn request using time: ", ticT(requestsT))
print("exit all threads\n")
def mvthread(purl, sem, result):
global errthread
with sem:
source = getpagedata(purl)
if type(source) == str:
mytree = lxml.etree.HTML(source)
# 网页存在两种状态
list = mytree.xpath(
'//ul[@class=\"panel\"]//li[@class=\"yk-col4 mr1\"]')
if len(list) <= 5:
list = mytree.xpath(
'//div[@class=\"yk-row\"]//div[@class=\"yk-col4 \"]')
kind = mytree.xpath(
'//div[@class=\"item noborder\"]/ul/li[@class=\"current\"]/span/text()'
)
if len(kind) <= 5:
kind = mytree.xpath(
'//div[@class=\"item border\"]/ul/li[@class=\"current\"]/span/text()'
)
print('---kind.type--->>>--%s' % kind)
# 使用协程池控制协程开启数量
pool = gevent.pool.Pool(15)
urlist = []
for li in list:
cover = li.xpath('.//div[@class=\"p-thumb\"]/img/@src')[0]
num = li.xpath(
'.//ul[@class=\"p-info pos-bottom\"]//span[@class=\"p-time \"]/span/text()'
)
if len(num) == 0:
num = li.xpath(
'.//ul[@class=\"p-info pos-bottom\"]//span[@class=\"p-time hover-hide\"]/span/text()'
)[0]
else:
num = num[0]
title = li.xpath(
'.//ul[@class=\"info-list\"]/li[@class=\"title\"]/a/text()'
)[0]
link = 'http:' + li.xpath(
'.//ul[@class=\"info-list\"]/li[@class=\"title\"]/a/@href'
)[0]
aclist = li.xpath(
'.//ul[@class=\"info-list\"]/li[@class=\"actor\"]/a/text()'
)
if len(aclist) == 0:
aclist = li.xpath(
'.//ul[@class=\"info-list\"]/li[@class=\"actor\"]/text()'
)
actor = li.xpath(
'.//ul[@class=\"info-list\"]/li[@class=\"actor\"]/em/text()'
)
if len(actor) != 0:
actor = actor[0]
else:
actor = 'None'
for i in range(len(aclist)):
if i != len(aclist) - 1:
actor += aclist[i]
actor += '、'
else:
actor += aclist[i]
doc = [kind[0], title, cover, actor, num, link, '']
urlist.append((link, doc, result))
print('------>>>--解析-->>-----', urlist)
# 协程池
pool.map(getdetail, urlist)
else:
errthread.append(purl, sem, result)
# TODO: fix this hack, which is used to skip groups only containig
# a single level in their heirarchy. These groups conflict with items
# from the historical usenet collection!
if '.' not in group:
return
g = NewsGroup(group, session=sesh, logging_level='INFO', ia_sync=True)
# Check for new articles.
count = int(g.last) - int(g.first)
if count <= 0:
log.info('no new articles found for {0}'.format(group))
return
g.archive_articles()
# ________________________________________________________________________________________
if __name__ == '__main__':
global sesh
sesh = GiganewsSession()
news_list = [x.split()[0] for x in open('giganews_listfile.txt')]
random.shuffle(news_list)
pool = gevent.pool.Pool(16)
pool.map(archive_group, news_list)
#for g in news_list:
# print g
# archive_group(g)
return ret
def inner_iterable():
""" This will be called inside the pool's main greenlet, which ID also needs to be registered """
if current_job:
set_current_job(current_job)
for x in iterable:
yield x
if current_job:
set_current_job(None)
start_time = time.time()
pool = gevent.pool.Pool(size=pool_size)
ret = pool.map(inner_func, inner_iterable())
pool.join(raise_error=True)
total_time = time.time() - start_time
log.debug("SubPool ran %s greenlets in %0.6fs" % (counter, total_time))
return ret
def subpool_imap(pool_size,
func,
iterable,
flatten=False,
unordered=False,
buffer_size=None):
""" Generator version of subpool_map. Should be used with unordered=True for optimal performance """
协程池实现
'''
from gevent import monkey
monkey.patch_all()
from gevent import pool
from urllib import request
import time
#协程执行的任务
def run_task(url):
print('Visit --> %s' % url)
try:
re = request.urlopen(url)
data = re.read()
print('%d bytes received from %s .' % (len(data), url))
except BaseException as e:
print(e)
return 'url:%s ---> finisk' % url
if __name__ == '__main__':
#协程池方式
pool = pool.Pool(2)
urls = [
'https://github.com', 'https://www.python.org/',
'https://www.baidu.com'
]
results = pool.map(run_task, urls)
print(results)
gevent.monkey.patch_all()
import sys
import requests
import gevent
import gevent.pool
import random
import json
def consume(id):
r = requests.get('http://127.0.0.1:5000/stream/' + str(id), stream=True)
for line in r.iter_lines(10):
try:
obj = json.loads(line)
except:
print line
raise
if random.randint(0, 99) <= 1:
print obj
POOL_SIZE = 1000
if __name__ == '__main__':
pool = gevent.pool.Pool(POOL_SIZE)
greenlets = pool.map(consume, xrange(POOL_SIZE))
gevent.joinall(greenlets)
def go():
pool = gevent.pool.Pool(20)
pool.map(put_shell,[i for i in range(40)])
def f(x):
r = 0
for k in range(1, K + 2):
r += x**(1 / k**1.5)
return r
if __name__ == '__main__':
#multiprocessing.pool测试
print('multiprocessing.pool.Pool:\n')
start = time.time()
l = []
pool = Pool(3)
for num, result in zip(NUMBERS, pool.map(f, NUMBERS)):
l.append(result)
print(len(l))
print('COST: {}'.format(time.time() - start))
#multiprocessing.pool.Pool.apply_async测试
print('multiprocessing.pool.Pool.async:\n')
start = time.time()
l = []
po = Pool(3)
tt = [po.apply_async(f, i) for i in NUMBERS]
print(len(tt), type(tt[0]))
print('COST: {}'.format(time.time() - start))
#concurrent.future.ProcessPoolExecutor without chunksize测试
print('ProcessPoolExecutor without chunksize:\n')
def gevent_vmap(func, iterable, *iterables, max_workers=10):
pool = gevent.pool.Pool(max_workers)
return pool.map(vcall(func), *concatv((iterable, ), *iterables))
