def poolworker():
index = 1
pool = gevent.pool.Pool(5)
while 1:
if signal_stop: break
pool.spawn(_poolworker, index)
index += 1
def upload(self, passphrase=None, *filename):
encrypt = passphrase is not None
if not passphrase:
pass
elif isinstance(passphrase, str):
self.passphrase = passphrase
else:
with open(passphrase) as f:
self.passphrase = f.read()
count = 0
pool = gevent.pool.Pool(10)
for file in filename:
count += 1
if self.shelve.has_upload(file):
pool.spawn(continue_greenlet, vault=self.vault, file=file, object_id=self.shelve.get_upload(
file), shelf=self.shelve, count=count, total=len(filename), encrypt=encrypt, passphrase=passphrase)
log.info(
'Spawning greenlet to finish filename {0}', format(file))
elif not self.shelve.has_archive(file):
pool.spawn(
create_greenlet, vault=self.vault, file=file, callback=self.get_upload_id, shelf=self.shelve, count=count,
total=len(filename), encrypt=encrypt, passphrase=passphrase)
log.info('Spawning greenlet for filename {0}'.format(file))
else:
log.info(
'File {0} is already at glacier, this is file {1} of {2}',
file, count, len(filename))
gevent.sleep(0)
def customer_sendtime():
"""
自定义时间发送报告
:return:
"""
try:
t3 = time.time()
pool = gevent.pool.Pool(_MAXTHREAD)
for s in CustomerSetting.objects.filter(
is_spamrpt_sendtime=True,
spamrpt_sendtime__contains=time.strftime("%H:%M:")).exclude(
customer__gateway_status='disabled'):
# for s in CustomerSetting.objects.filter(is_spamrpt_sendtime=True).exclude(customer__gateway_status='disabled'):
if s.spamrpt:
mails = get_mails_from_sendtime(s.customer_id,
s.spamrpt_sendtime)
for mail_to in set([m.mail_to for m in mails]):
pool.spawn(work_send, s.customer_id, mail_to, 'customer',
s.spamrpt_sendtime)
if s.m_spamrpt:
pool.spawn(work_send, s.customer_id, '', 'manager',
s.spamrpt_sendtime)
log.info('waiting stop...')
pool.join()
t4 = time.time()
log.info('spam_rpt send total time={}'.format(t4 - t3))
return
except (DatabaseError, InterfaceError) as e:
log.error(u'DatabaseError', exc_info=1)
connection.close()
except BaseException as e:
log.error(u'spam_rpt: exception', exc_info=1)
def cleanup_containers(self, auth_kwargs, container_base, concurrency,
policy):
storage_urls, token = self._authenticate(auth_kwargs)
_, container_list = client.get_account(random.choice(storage_urls),
token)
our_container_re = re.compile(self.DELETER_RE %
(container_base, policy))
start_time = time.time()
obj_count = 0
container_count = 0
pool = gevent.pool.Pool(concurrency)
for container_info in container_list:
# e.g. {'count': 41, 'bytes': 496485, 'name': 'doc'}
if our_container_re.match(container_info['name']):
pool.spawn(_container_deleter, concurrency, storage_urls,
token, container_info)
container_count += 1
obj_count += container_info['count']
else:
logging.debug('Ignoring non-ssbench container %r',
container_info['name'])
pool.join()
delta_t = time.time() - start_time
logging.info('Deleted %.1f containers/s, %.1f objs/s',
container_count / delta_t, obj_count / delta_t)
def scan():
sql = "SELECT email, browser, os, country, simple_country, area, ip_first, ip_last, open_total, open_first, open_last FROM active_emails;"
res = DB.query(REMOTE_PG, sql)
pool = gevent.pool.Pool(50)
for data in res:
pool.spawn(worker, data)
pool.join()
def manager_main():
"""
跟客户管理员发送报告
:return:
"""
try:
t3 = time.time()
mail_model = get_mail_model(get_mail_date())
customer_list = mail_model.objects.exclude(customer__gateway_status='disabled') \
.filter(state='reject', mail_to__isnull=False, review_result='reject',
customer__customersetting__m_spamrpt=True, customer__customersetting__is_spamrpt_sendtime=False) \
.distinct('customer_id') \
.values_list('customer_id', flat=True)
pool = gevent.pool.Pool(_MAXTHREAD)
for customer_id in customer_list:
pool.spawn(work_send, customer_id, '', 'manager')
log.info('waiting stop...')
pool.join()
t4 = time.time()
log.info('m_spam_rpt send total time={}'.format(t4 - t3))
return
except (DatabaseError, InterfaceError) as e:
log.error(u'DatabaseError', exc_info=1)
connection.close()
except BaseException as e:
log.error(u'spam_rpt: exception', exc_info=1)
gevent.sleep(10)
def cleanup_containers(self, auth_kwargs, container_base, concurrency):
storage_urls, token = self._authenticate(auth_kwargs)
_, container_list = client.get_account(
random.choice(storage_urls), token)
our_container_re = re.compile('%s_\d+$' % container_base)
start_time = time.time()
obj_count = 0
container_count = 0
pool = gevent.pool.Pool(concurrency)
for container_info in container_list:
# e.g. {'count': 41, 'bytes': 496485, 'name': 'doc'}
if our_container_re.match(container_info['name']):
pool.spawn(_container_deleter, concurrency, storage_urls,
token, container_info)
container_count += 1
obj_count += container_info['count']
else:
logging.debug('Ignoring non-ssbench container %r',
container_info['name'])
pool.join()
delta_t = time.time() - start_time
logging.info('Deleted %.1f containers/s, %.1f objs/s',
container_count / delta_t, obj_count / delta_t)
def schedule_green_jobs(fns,
concurrency=DEFAULT_THREADS,
progress=None,
total=None):
import gevent.pool
if total is None:
try:
total = len(fns)
except TypeError: # generators don't have len
pass
pbar = tqdm(total=total, desc=progress, disable=(not progress))
results = []
def updatefn(fn):
def realupdatefn():
res = fn()
pbar.update(1)
results.append(res)
return realupdatefn
pool = gevent.pool.Pool(concurrency)
for fn in fns:
pool.spawn(updatefn(fn))
pool.join()
pool.kill()
pbar.close()
return results
def worker_task(user_list):
pool = gevent.pool.Pool(5)
for user_id in user_list:
pool.spawn(do_worker_task, user_id)
gevent.sleep(0.01)
pool.join()
return
def print_friend_username(http, friend_id):
friend_url = URL('/' + str(friend_id))
friend_url['access_token'] = Token
# 在链接之前，线程会处于阻塞状态
response = http.get(friend_url.request_uri)
assert response.status_code == 200
friend = json.load(response)
# 判断是否存在username
if friend.has_key('username'):
print(f"{friend_url['username']} :{friend['name']}")
else:
print(f"{friend['name']}")
# 设置一次运行20个线程
pool = gevent.pool.Pool(20)
# 循环读取
for item in data:
friend_id = item['id']
pool.spawn(print_friend_username, http, friend_id)
pool.json()
# 关闭
http.close()
def scanurl():
hosts = load_target()
def find_plugins_by_name(search):
for name, plugins in COMPONENT_PLUGIN_INFO.iteritems():
if name in search:
return plugins
return False
for i in hosts:
url, server, title = i['url'], i['server'], i['title'].lower()
parse = urlparse.urlparse(url)
port = 80
l = parse.netloc.split(':')
if len(l) == 2:
host, port = l
port = int(port)
else:
host = l[0]
plugins = find_plugins_by_name(title)
if plugins:
for plugin in plugins:
pool.spawn(run_task, plugin, host, port)
pool.join()
def main(argv):
if len(argv) < 2 or not os.path.exists(argv[1]):
print("Usage: ping.py [config]")
return 1
# Initialize global settings
init_settings(argv)
# Initialize logger
loglevel = logging.INFO
if SETTINGS['debug']:
loglevel = logging.DEBUG
logformat = ("%(asctime)s,%(msecs)05.1f %(levelname)s (%(funcName)s) "
"%(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=SETTINGS['logfile'],
filemode='w')
print("Writing output to {}, press CTRL+C to terminate..".format(
SETTINGS['logfile']))
logging.info("Removing all keys")
REDIS_CONN.delete('reachable')
REDIS_CONN.delete('open')
REDIS_CONN.delete('opendata')
# Initialize a pool of workers (greenlets)
pool = gevent.pool.Pool(SETTINGS['workers'])
pool.spawn(cron, pool)
pool.join()
return 0
def workon(iap):
pool = gevent.pool.Pool(self.poolsize)
for i in range(self.poolsize):
pool.spawn(iap.work)
pool.join()
def interval_sendtime():
"""
:return:
"""
try:
t3 = time.time()
pool = gevent.pool.Pool(_MAXTHREAD)
for s in CustomerSetting.objects.filter(
interval_spamrpt__gt=0).exclude(
customer__gateway_status='disabled'):
interval = int(s.interval_spamrpt)
customer_id = s.customer_id
if check_interval(customer_id, interval):
log.info(
'check interval fail(customer_id:{}, interval:{})'.format(
customer_id, interval))
continue
if s.spamrpt:
mails = get_mails_from_interval(customer_id, interval)
for mail_to in set([m.mail_to for m in mails]):
pool.spawn(work_send, customer_id, mail_to, 'customer',
interval)
if s.m_spamrpt:
pool.spawn(work_send, customer_id, '', 'manager', interval)
log.info('waiting stop...')
pool.join()
t4 = time.time()
log.info('spam_rpt send total time={}'.format(t4 - t3))
return
except (DatabaseError, InterfaceError) as e:
log.error(u'DatabaseError', exc_info=1)
connection.close()
except BaseException as e:
log.error(u'spam_rpt: exception', exc_info=1)
def process_events():
r = redis.from_url(State.config['redis'])
queues = ['event:p%s' % i for i in xrange(State.config['max_level'] + 1)]
while True:
q, raw = r.blpop(queues)
pool.spawn(process_single_event, json.loads(raw))
def process_events():
r = redis.from_url(State.config['redis'])
queues = ['satori-events:%s' % i for i in range(15)]
while True:
o, raw = r.blpop(queues)
pool.spawn(process_single_event, json.loads(raw))
def propogate():
common = {'foo': 'bar', 'bar': 'foo'}
pool = gevent.pool.Pool(10)
pool.spawn(change_foo, common)
pool.spawn(change_bar, common)
pool.join()
print common
def main():
init()
# log.info('start worker_other.....')
# worker_other()
# log.info('finish worker_other.....')
# gevent.sleep(0.1)
log.info('start worker_day.....')
pool = gevent.pool.Pool(10)
for tablename in glb_maillog_tables:
pool.spawn(worker_day, tablename)
gevent.sleep(0.02)
pool.join()
gevent.sleep(0.1)
log.info('finish worker_day.....')
log.info('start worker_success.....')
worker_success()
log.info('finish worker_success.....')
log.info('start worker_success.....')
worker_success()
log.info('finish worker_success.....')
log.info('start worker_redis.....')
worker_redis()
log.info('finish worker_redis.....')
return
def async ():
index = 200000
pool = gevent.pool.Pool(100)
while index:
index -= 1
pool.spawn(_async)
pool.join()
def test_fox_cub(games_to_test, dataset, client):
pool = gevent.pool.Pool(1024)
# unique testing id
session_id = str(uuid.uuid4())
for game in games_to_test:
_, features = dataset.prepare_observation(game)
home_team_season = dataset.get_team_stats(game.HomeTeam,
game.Season)['season']
away_team_season = dataset.get_team_stats(game.AwayTeam,
game.Season)['season']
home_team_res = home_team_season.get_team_scores(game.HomeTeam)
away_team_res = away_team_season.get_team_scores(game.AwayTeam)
season_avg = {
"avgScoredHome": features.avg_goals_home_team / 2,
"avgScoredAway": features.avg_goals_away_team / 2
}
pool.spawn(client.get_stats, home_team_res, away_team_res, season_avg,
game.HomeTeam, game.AwayTeam, session_id)
pool.join()
return session_id
def propogate():
common = {'foo': 'bar', 'bar': 'foo'}
pool = gevent.pool.Pool(10)
pool.spawn(change_foo, common)
pool.spawn(change_bar, common)
pool.join()
print common
def _regular_run(self):
setproctitle.setproctitle('TDDC-ProxyAliveCheck-{}'.format(default_config.FEATURE))
log.info('Proxy Alive Check Is Running.')
pool = gevent.pool.Pool(self.REPEATED_CONCURRENT)
gevent.sleep(5)
while True:
keys = RedisEx().keys('tddc:worker:config:common:proxy_check_list:*')
items = [RedisEx().hgetall(key) for key in keys]
platforms = {item.get('s_platform') for item in items
if item and item.get('s_platform') and item.get('s_feature')
and item.get('s_url') and item.get('b_valid')}
pre_timestamp = RedisEx().hgetall('tddc:proxy:repeated')
ts = int(time.time())
for platform in platforms:
if ts - int(pre_timestamp.get(platform, 0)) < self.REPEATED_INTERVAL:
continue
coroutines = [
pool.spawn(self._check_handle, self.proxy_conf.pool, platform, proxy)
for proxy in RedisEx().smembers('{}:{}'.format(self.proxy_conf.pool, platform))
]
gevent.joinall(coroutines)
RedisEx().hset('tddc:proxy:repeated', platform, ts)
for http_type in ['http', 'https']:
if ts - int(pre_timestamp.get(http_type, 0)) < self.REPEATED_INTERVAL:
continue
coroutines = [
pool.spawn(self._check_handle, self.proxy_conf.source, http_type, proxy)
for proxy in RedisEx().smembers('{}:{}'.format(self.proxy_conf.source, http_type))
]
gevent.joinall(coroutines)
RedisEx().hset('tddc:proxy:repeated', http_type, ts)
gevent.sleep(10)
def main_loop():
# pool_size = config.getint('gevent', 'worker.pool.size')
# (crimi) - Setting pool_size to 1 to avoid deadlocks. This is until we are able to demonstrate that
# the deadlocks are able to be avoided.
# An improvement would be to do the DB updates on single worker, allowing everything else to
# happen concurrently. But expected load for 1.0 isn't great .. more than manageable with 1 worker.
#
pool_size = 1
pool = gevent.pool.Pool(pool_size)
logger.info('Started gevent pool with size %d', pool_size)
consumer = kafkareader.create_consumer(config)
while True:
try:
raw_event = kafkareader.read_message(consumer)
logger.debug('READ MESSAGE %s', raw_event)
event = MessageItem(json.loads(raw_event))
if event.get_command() in known_commands:
pool.spawn(topology_event_handler, event)
else:
logger.debug('Received unknown type or command %s', raw_event)
except Exception as e:
logger.exception(e.message)
def go(self):
logging.debug('Worker %s starting...', self.worker_id)
gevent.spawn(self._result_writer)
pool = gevent.pool.Pool(self.concurrency)
jobs = self.work_pull.recv()
if self.profile_count:
import cProfile
prof = cProfile.Profile()
prof.enable()
gotten = 1
self.spawned = 0
while jobs:
job_data = msgpack.loads(jobs)
for job_datum in job_data:
if 'container' in job_datum:
logging.debug('WORK: %13s %s/%-17s',
job_datum['type'], job_datum['container'],
job_datum['name'])
else:
logging.debug('CMD: %13s', job_datum['type'])
if job_datum['type'] == 'SUICIDE':
logging.info('Got SUICIDE; closing sockets and exiting.')
self.work_pull.close()
self.results_push.close()
os._exit(88)
pool.spawn(self.handle_job, job_datum)
self.spawned += 1
if self.profile_count and gotten >= self.profile_count:
prof.disable()
prof_output_path = '/tmp/worker_go.%d.prof' % os.getpid()
prof.dump_stats(prof_output_path)
logging.info('PROFILED worker go() to %s', prof_output_path)
self.profile_count = None
gotten += 1
jobs = self.work_pull.recv()
def workon(iap):
pool = gevent.pool.Pool(self.poolsize)
for i in range(self.poolsize):
pool.spawn(iap.work)
pool.join()
def main(argv):
if len(argv) < 2 or not os.path.exists(argv[1]):
print("Usage: ping.py [config]")
return 1
# Initialize global settings
init_settings(argv)
# Initialize logger
loglevel = logging.INFO
if SETTINGS['debug']:
loglevel = logging.DEBUG
logformat = ("%(asctime)s,%(msecs)05.1f %(levelname)s (%(funcName)s) "
"%(message)s")
logging.basicConfig(level=loglevel,
format=logformat,
filename=SETTINGS['logfile'],
filemode='w')
print("Writing output to {}, press CTRL+C to terminate..".format(
SETTINGS['logfile']))
logging.info("Removing all keys")
REDIS_CONN.delete('reachable')
REDIS_CONN.delete('open')
REDIS_CONN.delete('opendata')
# Initialize a pool of workers (greenlets)
pool = gevent.pool.Pool(SETTINGS['workers'])
pool.spawn(cron, pool)
pool.join()
return 0
def scan():
index = 0
pool = gevent.pool.Pool(10)
while True:
index += 1
pool.spawn(woker1, index)
gevent.sleep(1)
pool.join()
def woker_imap():
smtp_dict = getSmtpData()
pool = gevent.pool.Pool(10)
for smtp_account_id in smtp_dict:
smtp_list = smtp_dict[smtp_account_id]
pool.spawn(do_woker_imap, smtp_account_id, smtp_list)
pool.join()
return
def server(port, pool):
s = socket.socket()
s.bind(('0.0.0.0', port))
s.listen()
while True:
cli, addr = s.accept()
print('Wellcom %s to SockerServer' % str(addr[0]))
pool.spawn(handle_request, cli)
def worker_redis(cr):
pool = gevent.pool.Pool(10)
for start in range(0, 100, 10):
for domain in GLB_DOMAINS:
end = start + 10
pool.spawn(do_worker_redis, cr, domain, start, end)
pool.join()
return
def scanner():
pool = gevent.pool.Pool(_MAXTHREAD)
for t in _TABLES:
if signal_stop: break
pool.spawn(worker, t)
gevent.sleep(0.01)
pool.join()
return
def pooledDownloadContent(self, inner_paths, pool_size=100):
self.log.debug("New downloadContent pool: len: %s" % len(inner_paths))
self.worker_manager.started_task_num += len(inner_paths)
pool = gevent.pool.Pool(pool_size)
for inner_path in inner_paths:
pool.spawn(self.downloadContent, inner_path)
self.worker_manager.started_task_num -= 1
self.log.debug("Ended downloadContent pool len: %s" % len(inner_paths))
def worker_redis():
T = ['163.com', 'qq.com', '*']
pool = gevent.pool.Pool(10)
for index in xrange(1, 11):
for domain in T:
pool.spawn(do_worker_redis, domain, index)
pool.join()
return
def server(port, pool):
s = socket.socket()
s.bind(('0.0.0.0', port))
s.listen()
while True:
cli, addr = s.accept()
#print("Welcome %s to SocketServer" % str(addr[0]))
pool.spawn(handle_request, cli) #通过pool.spawn()运行协程
def worker_3():
log.info('start worker_3...')
pool = gevent.pool.Pool(5)
while True:
pool.spawn(do_worker_3, random.randint(1, 100000))
pool.join()
log.info('finish worker_3...')
return
def pooledDownloadContent(self, inner_paths, pool_size=100):
self.log.debug("New downloadContent pool: len: %s" % len(inner_paths))
self.worker_manager.started_task_num += len(inner_paths)
pool = gevent.pool.Pool(pool_size)
for inner_path in inner_paths:
pool.spawn(self.downloadContent, inner_path)
self.worker_manager.started_task_num -= 1
self.log.debug("Ended downloadContent pool len: %s" % len(inner_paths))
def consumer():
pool = gevent.pool.Pool(5)
while 1:
if not GLB_THREADS:
gevent.sleep(0.2)
continue
ip, email = GLB_THREADS.pop()
pool.spawn(__consumer, ip, email)
def pooledDownloadFile(self, inner_paths, pool_size=100, only_if_bad=False):
self.log.debug("New downloadFile pool: len: %s" % len(inner_paths))
self.worker_manager.started_task_num += len(inner_paths)
pool = gevent.pool.Pool(pool_size)
for inner_path in inner_paths:
if not only_if_bad or inner_path in self.bad_files:
pool.spawn(self.needFile, inner_path, update=True)
self.worker_manager.started_task_num -= 1
self.log.debug("Ended downloadFile pool len: %s" % len(inner_paths))
def dosearch(self):
for offset in [i*200 for i in range(40)]:
pool.spawn(self.getTracks,self.querystr,offset)
pool.join()
result = sorted(self.results, key=lambda x:x.tosort, reverse=True)[:20]
for t in result:
t.widget = makeWidget(t.id)
print "(%s) %s - %s"%(t.tosort,t.username.encode('ascii', 'ignore'),t.title.encode('ascii', 'ignore'))
return result
def run_with_gevent():
from qs.misc import call_in_loop
import gevent.pool
pool = gevent.pool.Pool()
for i in range(numgreenlets):
pool.spawn(call_in_loop(1.0, start_worker))
pool.join()
def process_events():
r = redis.from_url(State.config['redis'])
queues = [
'satori-events:%s' % i for i in range(15)
]
while True:
o, raw = r.blpop(queues)
pool.spawn(process_single_event, json.loads(raw))
def test_proxy_list(http_proxies, pool_size, server_port):
pool = gevent.pool.Pool(pool_size)
my_ip = whats_my_ip()
globals()['LOCAL_SERVER'] = 'http://{}:{}/'.format(my_ip, server_port)
for proxy in http_proxies:
ip, port = proxy.rsplit(':')
pool.spawn(test_single_proxy, my_ip, ip, port)
pool.join()
queue.put(StopIteration)
def resolve_hostname(self):
"""
Concurrently resolves hostname for the unresolved addresses.
"""
pool = gevent.pool.Pool(len(self.resolved['hostname']))
with gevent.Timeout(15, False):
for address in self.resolved['hostname']:
pool.spawn(self.set_hostname, address)
pool.join()
def upgrade_article(row): pool.spawn(_upgrade_article, row) global count count += 1 if count > 50: count = 0 for greenlet in list(pool): if greenlet.dead: pool.discard(greenlet)
def spawn_clients(pool, network, n):
clients = []
last, kad_client = spawn_client(pool, network)
clients.append(kad_client)
for i in xrange(0,n-1):
last, kad_client = spawn_client(pool, network, last)
clients.append(kad_client)
print "test: spawned %s nodes" % n
pool.spawn(client_summary, clients)
def process_events():
r = redis.from_url(State.config['redis'])
queues = [
'event:p%s' % i for i in
xrange(State.config['max_level'] + 1)
]
while True:
q, raw = r.blpop(queues)
pool.spawn(process_single_event, json.loads(raw))
def start(self):
pool = gevent.pool.Pool(size=self.concurrency)
try:
for i in xrange(1, self.num_connectors + 1):
pool.spawn(self.connector)
time.sleep(self.spawn_interval)
pool.join()
except KeyboardInterrupt:
pass
def sync_table(table, fields):
f1 = ", ".join(fields)
pieces = {
"ataobao2.item": 100,
"ataobao2.item_by_date": 1000,
"ataobao2.brand_by_date": 10,
"ataobao2.shop_by_date": 10,
}.get(table, 1)
start = -2 ** 63
step = 2 ** 64 / pieces
print "migrating {} {}".format(table, f1)
for i in range(pieces):
start = -2 ** 63 + step * i
end = min(2 ** 63 - 1, -2 ** 63 + step * (i + 1))
with db1.connection() as cur:
print "piece", i + 1
# print 'select {} from {} where token({})>=:v1 and token({})<:v2'.format(f1, table, fields[0], fields[0]), dict(v1=start, v2=end)
if table.endswith("_by_date") and "datestr" in fields:
d0 = (datetime.utcnow() + timedelta(hours=8) - timedelta(days=2)).strftime("%Y-%m-%d")
cur.execute(
"select {} from {} where token({})>=:v1 and token({})<:v2 and datestr>=:d0 allow filtering".format(
f1, table, fields[0], fields[0]
),
dict(v1=start, v2=end, d0=d0),
consistency_level="ONE",
)
else:
cur.execute(
"select {} from {} where token({})>=:v1 and token({})<:v2".format(f1, table, fields[0], fields[0]),
dict(v1=start, v2=end),
consistency_level="ONE",
)
for j, row in enumerate(cur):
if j % 1000 == 0:
print "syncd {}".format(j)
params = {}
fs = list(fields)
for k, v in zip(fields, row):
if k == "date":
if v is not None and len(v) == 8:
v = struct.unpack("!q", v)[0]
else:
continue
if v is not None:
params[k] = v
fs = params.keys()
fs1 = ", ".join(fs)
fs2 = ", ".join([":" + f for f in fs])
if "id" in params or "datestr" in params or "name" in params:
if table == "ataobao2.item_by_date" and "date" not in params:
continue
# print 'INSERT INTO {} ({}) VALUES ({})'.format(table, fs1, fs2), params
pool.spawn(db2.execute, "insert into {} ({}) values ({})".format(table, fs1, fs2), params)
def download_images(posts):
"""Downloads images for the given posts"""
pool = gevent.pool.Pool(size=96)
for post in progress.bar(posts, width=60, every=100):
if not post.static:
continue
pool.spawn(download_image, post)
# wait for all jobs to finish
pool.join()
def crawler(u):
global crawled
response = requests.get(u)
print response.status_code, u
for link in re.findall('<a href="(http.*?)"', response.content):
if crawled < 10 and not pool.full():
crawled += 1
pool.spawn(crawler, link)
def make_nuwiki(fsdir, metabook, options, podclient=None, status=None):
id2wiki = {}
for x in metabook.wikis:
id2wiki[x.ident] = (x, [])
for x in metabook.articles():
assert x.wikiident in id2wiki, "no wikiconf for %r (%s)" % (x.wikiident, x)
id2wiki[x.wikiident][1].append(x)
is_multiwiki = len(id2wiki) > 1
if is_multiwiki:
progress = fetch.shared_progress(status=status)
else:
progress = None
fetchers = []
for id, (wikiconf, articles) in id2wiki.items():
if id is None:
id = ""
assert not is_multiwiki, "id must be set in multiwiki"
if not is_multiwiki:
id = ""
assert "/" not in id, "bad id: %r" % (id,)
my_fsdir = os.path.join(fsdir, id)
if is_multiwiki:
my_mb = collection()
my_mb.items = articles
else:
my_mb = metabook
wikitrust(wikiconf.baseurl, my_mb)
fetchers.append(start_fetcher(fsdir=my_fsdir, progress=progress, base_url=wikiconf.baseurl,
metabook=my_mb, options=options, podclient=podclient, status=status))
if is_multiwiki:
if not os.path.exists(fsdir):
os.makedirs(fsdir)
open(os.path.join(fsdir, "metabook.json"), "wb").write(metabook.dumps())
myjson.dump(dict(format="multi-nuwiki"), open(os.path.join(fsdir, "nfo.json"), "wb"))
pool = gevent.pool.Pool()
for x in fetchers:
pool.spawn(x.run)
pool.join(raise_error=True)
import signal
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
def runworker(args):
gevent.monkey.patch_all()
initlog(optdict.get('-l', 'INFO'))
app = apps.Application(args[0])
size = int(optdict.get('-s', '100'))
pool = gevent.pool.Pool(size)
for n in xrange(size):
pool.spawn(worker.BeanstalkWorker(
app, optdict['-q'], optdict.get('-H', 'localhost'),
optdict.get('-p', '11300'), int(optdict.get('-t', '10'))).run)
pool.join()
def run(self):
def update_latest_ids(cid):
data = get_json(cid, page=1, sort='_oldstart')
nids = get_ids(data)
if nids:
print 'found {} ids in category {}'.format(len(nids), cid)
ai2.put(*list(nids))
pool = gevent.pool.Pool(10)
for cid in fecids:
pool.spawn(update_latest_ids, cid)
pool.join()
def spawn_client(pool, network, initial_node=None):
if initial_node is None:
nodes = []
else:
nodes = [initial_node]
rpc_chan = queue.Queue()
rpc_client = Rpc_Client(network, rpc_chan)
node = rpc_client.return_node()
kad_client = Kad_Client(pool, rpc_client, rpc_chan)
pool.spawn(client_actions, kad_client, nodes)
return node, kad_client
def process_group(group):
group_results = []
pool = gevent.pool.Pool(urls_group_size)
for url in group:
if not is_valid_url(url, allow_no_protocol=True):
completed_urls[url] = (False, "Invalid URL")
if len(completed_urls) == len(urls): #all done, trigger callback
return completed_callback(completed_urls)
else:
continue
assert url.startswith('http://') or url.startswith('https://')
pool.spawn(make_stream_request, url)
pool.join()
def run(self):
"""This method puts the daemon into a poll/action loop.
This method should not be extended or overwritten. Instead,
implementations of this daemon should implement the 'get_message()'
and 'handle_message()' methods.
This loop makes use of an gevent Pool to manage maximum greenthread
concurrency. The behaviour of the pool, and greenthreads in general, is
such that there must be a cooperative yield in order for gevent to
switch context into another greenthread.
This loop, by default, will only yield on an empty message and when the
Pool has allocated the maximum allowed greenthreads. To yield the
loop after each message, set the aggressive_yield bit to True.
"""
pool = gevent.pool.Pool(size=self.pool_size)
while True:
message = self.get_message()
LOG.debug(
"Daemon (%r) got message (%r).",
self.pidfile,
message
)
if message is None:
LOG.debug(
"Daemon (%r) received no message. Going idle for (%r).",
self.pidfile,
self.idle_time
)
self.sleep(self.idle_time)
continue
LOG.debug(
"Daemon (%r) attempting to start new greenthread with (%r) "
"active and (%r) free",
self.pidfile,
pool.size - pool.free_count(),
pool.free_count()
)
pool.spawn(self.handle_message, message)
if self.aggressive_yield is True:
gevent.sleep(0)
def retrieve(self, topic=None, *filename):
count = 0
pool = gevent.pool.Pool(10)
for file in filename:
count += 1
if self.shelve.has_archive(file):
object_id = self.shelve.get_archive(file)
arn = get_arn(self.key, self.secret, topic) if topic else None
pool.spawn(
fetch_greenlet, vault=self.vault, object_id=object_id,
count=count, total=len(filename), arn=arn, shelf=self.shelve, file=file)
log.info(
"File {0} is being fetched from glacier, will call back to {1}")
gevent.sleep(0)
def delete(self, *filename):
count = 0
pool = gevent.pool.Pool(10)
for file in filename:
count += 1
print file, self.shelve.has_archive(file)
if self.shelve.has_archive(file):
object_id = self.shelve.get_archive(file)
pool.spawn(
delete_greenlet, vault=self.vault, object_id=object_id, count=count,
total=len(filename), shelf=self.shelve, file=file)
log.info(
"File {0} is being fetched from glacier, will call back to {1}")
gevent.sleep(0)
def cron(pool):
"""
Assigned to a worker to perform the following tasks periodically to
maintain a continuous network-wide connections:
[Master]
1) Checks for a new snapshot
2) Loads new reachable nodes into the reachable set in Redis
3) Signals listener to get reachable nodes from opendata set
4) Sets bestblockhash in Redis
[Master/Slave]
1) Spawns workers to establish and maintain connection with reachable nodes
"""
publish_key = 'snapshot:{}'.format(hexlify(CONF['magic_number']))
snapshot = None
while True:
if CONF['master']:
new_snapshot = get_snapshot()
if new_snapshot != snapshot:
nodes = get_nodes(new_snapshot)
if len(nodes) == 0:
continue
logging.info("New snapshot: %s", new_snapshot)
snapshot = new_snapshot
logging.info("Nodes: %d", len(nodes))
reachable_nodes = set_reachable(nodes)
logging.info("New reachable nodes: %d", reachable_nodes)
# Allow connections to stabilize before publishing snapshot
gevent.sleep(CONF['socket_timeout'])
REDIS_CONN.publish(publish_key, int(time.time()))
connections = REDIS_CONN.scard('open')
logging.info("Connections: %d", connections)
set_bestblockhash()
for _ in xrange(min(REDIS_CONN.scard('reachable'), pool.free_count())):
pool.spawn(task)
workers = CONF['workers'] - pool.free_count()
logging.info("Workers: %d", workers)
gevent.sleep(CONF['cron_delay'])
def test_contest(self):
ns = NamespaceSemaphore()
ns.acquire('/ex', 3, 0)
self.q = gevent.queue.Queue()
pool = gevent.pool.Pool(size=5)
for i in xrange(5):
pool.spawn(self.acquire, ns)
pool.join()
self.assertEquals(self.q.qsize(), 1)
self.q.get()
for i in xrange(5):
pool.spawn(self.acquire, ns)
pool.join()
self.assertEquals(self.q.qsize(), 1)
