def application(environ, start_response): """ application handler """ gev = None path = environ.get("PATH_INFO", "").lstrip("/") for regex, callback in ROUTES: match = re.search(regex, path) if match is not None: environ["web.url_args"] = match.groups() qs = parse_qs(environ["QUERY_STRING"]) environ["web.qs"] = {k: [escape(v) for v in qs[k]] for k in qs} gev = gevent.spawn(callback, environ) gevent.joinall([gev]) break if gev is None: start_response("404 NOT FOUND", HEADERS) yield ("%s\n" % json.dumps({"error": "Not Found"})).encode("utf-8") return if gev.value[0]: start_response(STATUS, HEADERS) yield ("%s\n" % json.dumps(gev.value[1])).encode("utf-8") return else: start_response(500, HEADERS) yield ("%s\n" % json.dumps({"error": gev.value[1]})).encode("utf-8") return
def main(): gevent.joinall([ gevent.spawn(setter), gevent.spawn(waiter), gevent.spawn(waiter), gevent.spawn(waiter) ])
def run_schema_transformer(args): global _vnc_lib def connection_state_update(status, message=None): ConnectionState.update( conn_type=ConnType.APISERVER, name='ApiServer', status=status, message=message or '', server_addrs=['%s:%s' % (args.api_server_ip, args.api_server_port)]) # end connection_state_update # Retry till API server is up connected = False connection_state_update(ConnectionStatus.INIT) while not connected: try: _vnc_lib = VncApi( args.admin_user, args.admin_password, args.admin_tenant_name, args.api_server_ip, args.api_server_port, api_server_use_ssl=args.api_server_use_ssl) connected = True connection_state_update(ConnectionStatus.UP) except requests.exceptions.ConnectionError as e: # Update connection info connection_state_update(ConnectionStatus.DOWN, str(e)) time.sleep(3) except (RuntimeError, ResourceExhaustionError): # auth failure or haproxy throws 503 time.sleep(3) global transformer transformer = SchemaTransformer(args) gevent.joinall(transformer._vnc_kombu.greenlets())
def needFile(self, inner_path, *args, **kwargs): if inner_path.endswith("|all"): @util.Pooled(20) def pooledNeedBigfile(*args, **kwargs): return self.needFile(*args, **kwargs) inner_path = inner_path.replace("|all", "") file_info = self.needFileInfo(inner_path) file_size = file_info["size"] piece_size = file_info["piece_size"] piece_num = int(math.ceil(float(file_size) / piece_size)) file_threads = [] piecefield = self.storage.piecefields.get(file_info["sha512"]) for piece_i in range(piece_num): piece_from = piece_i * piece_size piece_to = min(file_size, piece_from + piece_size) if not piecefield or not piecefield[piece_i]: res = pooledNeedBigfile("%s|%s-%s" % (inner_path, piece_from, piece_to), blocking=False) if res is not True and res is not False: file_threads.append(res) gevent.joinall(file_threads) else: return super(SitePlugin, self).needFile(inner_path, *args, **kwargs)
def run_svc_monitor(args=None): monitor = SvcMonitor(args) monitor._zookeeper_client = _zookeeper_client # Retry till API server is up connected = False monitor.logger.api_conn_status_update(ConnectionStatus.INIT) while not connected: try: vnc_api = VncApi( args.admin_user, args.admin_password, args.admin_tenant_name, args.api_server_ip, args.api_server_port, api_server_use_ssl=args.api_server_use_ssl) connected = True monitor.logger.api_conn_status_update(ConnectionStatus.UP) except requests.exceptions.ConnectionError as e: monitor.logger.api_conn_status_update( ConnectionStatus.DOWN, str(e)) time.sleep(3) except ResourceExhaustionError: # haproxy throws 503 time.sleep(3) monitor.post_init(vnc_api, args) timer_task = gevent.spawn(launch_timer, monitor) gevent.joinall([timer_task])
def check_domain(self, domain, timeout): """Check specified domain.""" resolver = dns.resolver.Resolver() try: answers = resolver.query(domain.name, "MX") except dns.resolver.NoAnswer: return ip_list = [] for answer in answers: address = None try: ipaddress.ip_address(str(answer.exchange)) except ValueError: try: address = socket.gethostbyname(str(answer.exchange)) except socket.gaierror: pass else: address = str(answer.exchange) finally: if address is not None: ip_list.append(address) if len(ip_list) == 0: return jobs = [ gevent.spawn(self.query, ip_list, provider) for provider in self.providers] gevent.joinall(jobs, timeout) for job in jobs: if not job.successful(): continue provider, results = job.value self.store_domain_result(domain, provider, results)
def map(requests, stream = False, size = 5, exception_handler = None): """Concurrently converts a list of Requests to Responses. :param requests: a collection of Request objects. :param stream: If True, the content will not be downloaded immediately. :param size: Specifies the number of requests to make at a time. If None, no throttling occurs. :param exception_handler: Callback function, called when exception occured. Params: Request, Exception """ requests = list(requests) pool = Pool(size) if size else None jobs = [send(r, pool, stream=stream) for r in requests] gevent.joinall(jobs) ret = [] #up until this point, identical to grequests library for request in requests:#error handling is different. if request.response:#if 200 code if(len(request.response.text)!= 9420):#sometimes a 200 code is #associated with a bad page #that has no content but no errors ret.append(request.response) else: ret.append("fail:bad Page")#return a string logging the error else: ret.append(request.exception) del request.exception return ret
def start_service(self): """ 启动服务 :return: """ args = ArgumentParser().args try: Jobs().add_interval_job(UPDATE_INTERVAL, self.update) if not self.is_sm: port = {"tcp": args.tcp_port} port.update({"https": args.http_port} if args.is_https else {"http": args.http_port}) self.adv = ServiceAdvertiser(self.service_type, port, self.get_jid(), self.service_version) self.adv.advertise() checker_ls = [] self.add_port_checker(args, checker_ls) PortChecker(checker_ls).start() self.services(args, self.thread_ls) self.__sys_services(args, self.thread_ls) logger.warn("start services for %s, args:%s" % (self.service_type, args)) gevent.joinall([thread.start() for thread in self.thread_ls]) except: logger.error(traceback.format_exc()) sys.exit(0)
def exploration(exp): global started if not started: return False evt.wait() do_alignment(robot.alignment()) global sensors cur = exp.getRealTimeMap(sensors, robot.explored_map) if cur[1]: done_exploration() return False if cur[0]: # if robot.try_left: # evt.wait() # do_alignment(robot.alignment()) send_cmd(cur[0]) robot.step(cur[0]) #Tues:add evt.wait() to see where it can slow down the simulator # evt.wait() print("[Tornado] exploration > %s" %(robot.current)) msg1 = robot.msg_for_android() btq.append(msg1) gevent.joinall([ gevent.spawn(delay_call, exploration, exp) ])
def main(): if len(sys.argv) > 1: code = sys.argv[1] print 'stock code %s' % (code) global TID TID = code with open('robot.yaml') as f: cfg = yaml.load(f) rip = cfg['robot_listen']['rip'] rport = cfg['robot_listen']['rport'] addr = (rip, rport) print addr tester = Tester(TID, addr) pq_ip = cfg['pre_quo']['pqip'] pq_port = cfg['pre_quo']['port'] print pq_ip, pq_port pre_quo = pre_quotation.PreQuotation(tester, (pq_ip, pq_port)) tester.pre_quo = pre_quo q_ip = cfg['quo_server']['qip'] q_port = cfg['quo_server']['port'] print q_ip, q_port quota = quotation.Quotation(tester, (q_ip, q_port)) tester.quo = quota jobs = [] jobs.append(gevent.spawn(pre_quo.recv_data)) jobs.append(gevent.spawn(quota.recv_data)) jobs.append(gevent.spawn(tester.recv_data)) gevent.joinall(jobs) return 0
def client(cid): s = socket.create_connection(('127.0.0.1', 7890)) w = s.makefile('w') r = s.makefile('r') def send(): times = 0 while times < 100: times += 1 gevent.sleep(1) data = str(random.randint(10, 10000)) w.write('%s, from client %d\n' % (data, cid)) w.flush() print 'client', cid, 'send:', data def recv(): while True: line = r.readline() print 'client', cid, 'recive:', line, if not line: break send_job = gevent.spawn_later(1, send) recv_job = gevent.spawn(recv) def clear(*args): gevent.killall([send_job, recv_job]) s.close() send_job.link(clear) recv_job.link(clear) gevent.joinall([send_job, recv_job]) print 'client', cid, 'finish'
def main(): gevent.monkey.patch_all() signal.signal(signal.SIGTERM, lambda signum, fame: teardown()) signal.signal(signal.SIGINT, lambda signum, fame: teardown()) atexit.register(teardown) setup() gevent.joinall([gevent.spawn(process.communicate) for process in processes])
def _get_media(self, api, mid): media = spawn(api.media, mid) likes = spawn(api.media_likes, media_id=mid) gevent.joinall([media, likes]) media, likes = media.get(), likes.get() errors = get_errors(media, likes) if errors: if any([e.error_type == 'APINotAllowedError' for e in errors]): return render('profile-noauth.html', ukey=request.ukey) app.logger.error([str(e) for e in errors]) return notfound(u'服务器暂时出问题了') ukey = media.user.id isfollow = False if request.ukey: try: isfollow = is_follow(ukey, api) except InstagramAPIError: return notfound(u'服务器暂时出问题了') isstar = False for i in likes: if request.ukey and request.ukey == i.id: isstar = True isme = False if request.ukey and ukey == request.ukey: isme = True return dict(media=media, isme=isme, isfollow=isfollow, likes=likes[:5], isstar=isstar)
def fetch_remote(base_config, name="origin"): jobs = [] seen = set() fetched = set() for path, config in base_config.span_configs(): if path in seen: continue seen.add(path) git = get_git(path) print(" [{cwd}] fetching {name}".format(cwd=path, name=name)) jobs.append(gevent.spawn(git.fetch, name)) for branch in (b for b in config.branches if ":" in b): remote, branch = branch.split(":", 1) if remote not in git.remote().split(): url = remote_url(git, remote) print(" [{path}] adding remote: {remote} -> {url}" .format(**locals())) git.remote("add", remote, url) print(" [{path}] fetching {remote} {branch}".format(**locals())) jobs.append(gevent.spawn(git.fetch, remote, branch)) fetched.add(remote) for pr in config.pull_requests: print(" [{path}] fetching pull request {pr}".format(**locals())) pr = 'pull/{pr}/head:enterprise-{pr}'.format(pr=pr) jobs.append(gevent.spawn(git.fetch, 'origin', pr)) gevent.joinall(jobs) print("fetched {}".format(", ".join(['origin'] + sorted(fetched))))
def main(args): S = Scraper(args) input_check(args) if args.targets: targets = get_targets_from_args(args.targets) elif args.shodansearch: targets = shodan_search(args.shodansearch, args.apikey, int(args.numpages), args.ipfile) elif args.ipfile: targets = get_ips_from_file(args.ipfile) if targets == [] or targets == None: exit('[!] No valid targets') # Mechanize doesn't respect timeouts when it comes to reading/waiting for SSL info so this is necessary setdefaulttimeout(int(args.timeout)) con = int(args.concurrent) # By default run 1000 concurrently at a time target_groups = [targets[x:x+con] for x in xrange(0, len(targets), con)] for chunk_targets in target_groups: jobs = [gevent.spawn(S.run, target) for target in chunk_targets] gevent.joinall(jobs)
def createThread(nums): strnums = [str(x) for x in nums] threads = [gevent.spawn(getInfo, i) for i in strnums] print "start! -- %d" % len(threads) gevent.joinall(threads) # once one coroutine is done, all others will be killed print "It's done! "
def main(): """ Main Avogadro entrypoint. Runs all metric collection agents. """ parser = OptionParser(version=__version__.__version__) AvogadroAgent.addParserOptions(parser) try: (options, args) = parser.parse_args(sys.argv) gevent.joinall([AvogadroCPUTimesAgent.spawn(options=options), AvogadroMemoryAgent.spawn(options=options), AvogadroDiskReadBytesAgent.spawn(options=options), AvogadroDiskWriteBytesAgent.spawn(options=options), AvogadroDiskReadTimeAgent.spawn(options=options), AvogadroDiskWriteTimeAgent.spawn(options=options), AvogadroNetworkBytesSentAgent.spawn(options=options), AvogadroNetworkBytesReceivedAgent.spawn(options=options), AvogadroKeyCountAgent.spawn(options=options), AvogadroKeyDownDownAgent.spawn(options=options), AvogadroKeyUpDownAgent.spawn(options=options), AvogadroKeyHoldAgent.spawn(options=options)]) except IndexError: parser.print_help(sys.stderr) sys.exit()
def main(): global qtimes,qerr reps = int(sys.argv[1]) for j in range(reps): jobs = [gevent.spawn(get_url, url) for url in urls] #print("Size of jobs is {n}".format(n=len(jobs))) gevent.joinall(jobs, timeout=30) if not qerr.empty(): qerr.put(StopIteration) for err in qerr: print(err) print("jobs size {s}".format(s=len(jobs))) print("qstart size {n}".format(n=qstart.qsize())) print("qtimes size {n}".format(n=qtimes.qsize())) qtimes.put(StopIteration) times = [] for item in qtimes: times.append(item) print("Min {t}".format(t=min(times))) print("Max {t}".format(t=max(times))) print("Mean {t}".format(t=stats.mean(times))) print("StdDev {t}".format(t=stats.stdev(times)))
def fetch_info(self): """ Fetch info for each entity in collection. """ jobs = [gevent.spawn(entity.fetch_info) for entity in self._entities] gevent.joinall(jobs, timeout=5)
def do_net(self, dst, block, method, params): node = self.node uid = node.uid res = {} inc = False try: if uid in dst: inc = True except: pass if dst is None or inc: try: h = getattr(self, "net_" + method) res[uid] = h(*params) except GreenletExit: return except: pass if dst is None or not inc: if block: job = node.call(dst, self.uid, method, *params) else: node.notify(dst, self.uid, method, *params) job = None if block: if job is not None: joinall(job) for v in job.itervalues(): res.update(v.get()) return res
def run_all(): gevent.signal(signal.SIGQUIT, gevent.kill) run_emitters() run_listeners() eventloop = gevent.spawn(run_eventloop) __jobs.append(eventloop) gevent.joinall(__jobs, raise_error=True)
def start(self,oneshot=False): redis_kwargs = {} for x in self.conf: if x.startswith('redis_'): redis_kwargs[x[6:]] = self.conf[x] self.redis = redis.StrictRedis(**redis_kwargs) try: self.init_handlers() # starte or not start observer if not oneshot: self.start_observer() else: self._oneshot = True # set our signal handler and save reference to default self.default_signal_handler = signal.signal( signal.SIGINT, self.loop_interrupt_handler) # now block until finnished loops = [ gevent.spawn(self.process_loop), gevent.spawn(self.send_loop), ] gevent.joinall(loops) # save the state, signal.signal(signal.SIGINT, self.save_interrupt_handler) self.save_state() except redis.ConnectionError: raise click.ClickException( 'connection to redis with:\n\n {0}\n\n failed!'.format( pprint.pformat(self.redis.connection_pool.connection_kwargs)))
def main(self): parser = argparse.ArgumentParser() parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file') parser.add_argument('-p', type=int, metavar='PORT', default=5500, help="WebSockets server port") args = parser.parse_args() configure_logging('/var/log/containerd.log', 'DEBUG') setproctitle.setproctitle('containerd') gevent.signal(signal.SIGTERM, self.die) gevent.signal(signal.SIGQUIT, self.die) self.config = args.c self.init_datastore() self.init_dispatcher() self.init_mgmt() self.init_nat() self.init_ec2() self.logger.info('Started') # WebSockets server kwargs = {} s4 = WebSocketServer(('', args.p), ServerResource({ '/console': ConsoleConnection, }, context=self), **kwargs) s6 = WebSocketServer(('::', args.p), ServerResource({ '/console': ConsoleConnection, }, context=self), **kwargs) serv_threads = [gevent.spawn(s4.serve_forever), gevent.spawn(s6.serve_forever)] gevent.joinall(serv_threads)
def _stop_pending_timers(self): """ Safely stops all pending and active timers. For all timers still waiting to run, calls kill on them. For active timers, let them exit naturally and prevent the reschedule by setting the _no_reschedule flag. """ # prevent reschedules self._no_reschedule = True gls = [] for timer_id in self.schedule_entries: spawns = self.__get_spawns(timer_id) for spawn in spawns: gls.append(spawn) # only kill spawns that haven't started yet if spawn._start_event is not None: spawn.kill() log.debug("_stop_pending_timers: timer %s deleted", timer_id) self.schedule_entries.clear() # wait for running gls to finish up gevent.joinall(gls, timeout=10) # allow reschedules from here on out self._no_reschedule = False
def issues(page=0): def gh_request(issueUrl): req = urllib2.Request( "%s?access_token=%s" % (issueUrl, access_token) ) response = urllib2.urlopen(req) respData = json.loads(response.read()) return respData # we'll be getting issues from here: # def getPostedIssues(page = 0, recent = True, pageSize = 25): issues = getPostedIssues(page, True, int(settings.webappPageSize)) issueUrls = issues['todoList'] jobs = [gevent.spawn(gh_request, url) for url in issueUrls] gevent.joinall(jobs, timeout=8) data = [job.value for job in jobs] return jsonify( data = data, pageNumber = issues['pageNumber'], pageCount = issues['pageCount'], )
def _split_commits_and_tags(obj_store, lst, ignore_unknown=False, pool=None): """Split object id list into two list with commit SHA1s and tag SHA1s. Same implementation as object_store._split_commits_and_tags except we use gevent to parallelize object retrieval. """ commits = set() tags = set() def find_commit_type(sha): try: o = obj_store[sha] except KeyError: if not ignore_unknown: raise else: if isinstance(o, Commit): commits.add(sha) elif isinstance(o, Tag): tags.add(sha) commits.add(o.object[1]) else: raise KeyError('Not a commit or a tag: %s' % sha) jobs = [pool.spawn(find_commit_type, s) for s in lst] gevent.joinall(jobs) return (commits, tags)
def __init__(self, object_store, haves, wants, progress=None, get_tagged=None, concurrency=1, get_parents=None): def collect_tree_sha(sha): self.sha_done.add(sha) cmt = object_store[sha] _collect_filetree_revs(object_store, cmt.tree, self.sha_done) self.object_store = object_store p = pool.Pool(size=concurrency) have_commits, have_tags = \ _split_commits_and_tags(object_store, haves, True, p) want_commits, want_tags = \ _split_commits_and_tags(object_store, wants, False, p) all_ancestors = object_store._collect_ancestors(have_commits)[0] missing_commits, common_commits = \ object_store._collect_ancestors(want_commits, all_ancestors) self.sha_done = set() jobs = [p.spawn(collect_tree_sha, c) for c in common_commits] gevent.joinall(jobs) for t in have_tags: self.sha_done.add(t) missing_tags = want_tags.difference(have_tags) wants = missing_commits.union(missing_tags) self.objects_to_send = set([(w, None, False) for w in wants]) if progress is None: self.progress = lambda x: None else: self.progress = progress self._tagged = get_tagged and get_tagged() or {}
def parallelPost(self, payload): queue = gevent.queue.Queue() gList = [] if type(self.resourceList) is list: for i in self.resourceList: queue.put(i) for t in range(len(self.resourceList)): gl = gevent.spawn(postRequest, queue, payload) gList.append(gl) print str(gList) gevent.joinall(gList) return GreenletRequests.NodeResponsesPost if type(self.resourceList) is dict: for k, v in self.resourceList.iteritems(): print k queue.put(k) for k, v in self.resourceList.iteritems(): gl = gevent.spawn(postRequest, queue, v) gList.append(gl) print str(gList) gevent.joinall(gList) return GreenletRequests.NodeResponsesPost
def search_weekends(): if not (request.args.get("departureCity") and request.args.get("arrivalCity")): raise Exception("missing args") greenlets = [] for delta in [0, 1]: departure_date = date.today() + timedelta(days=7 * delta) friday = next_weekday(departure_date, 4) sunday = next_weekday(departure_date, 6) greenlets.append(gevent.spawn( ScrapyRTClient.get_round_trip, request.args["departureCity"], request.args["arrivalCity"], friday, sunday )) gevent.joinall(greenlets) round_trips = [g.value for g in greenlets] return jsonify( results=[rt.to_json() for rt in round_trips], success=True )
def main(args): users = User.objects.filter(username__in=args.users.split(",")) try: provider = Provider.objects.get(id=args.provider_id) except Provider.DoesNotExist: print("A provider for id=%s could not be found" % args.provider_id) sys.exit(1) async_request_list = [] for user in users: identity = Identity.objects.get(created_by=user, provider=provider) user_tokens = Token.objects.filter(user=user).order_by('-issuedTime') if user_tokens.count() == 0: print("No tokens for user: "******". No instances will launch on their account.") continue latest_token = user_tokens[0] headers = { 'Authorization': 'Token ' + latest_token.key } instances = Instance.objects.filter(only_current(), created_by=user, name=args.name) for instance in instances: launch_url = settings.SERVER_URL + "/api/v1/provider/" + provider.uuid + "/identity/" + identity.uuid + "/instance/" + instance.provider_alias job = gevent.spawn(delete_instance, launch_url, headers, provider, user) async_request_list.append(job) print "Sending requests to Atmosphere..." gevent.joinall(async_request_list) print "Script finished running successfully!"
def process_start(url_list): # with Loc: tasks = [] for data in url_list: tasks.append(gevent.spawn(get_data, data)) gevent.joinall(tasks) #使用协程来执行
def asynchronous(): threads = [] for i in range(1, 10): threads.append(gevent.spawn(fetch, i)) gevent.joinall(threads)
import gevent from gevent import socket hosts = ['www.crappytaxidermy.com', 'www.walterpottertaxidermy.com', # 'www.antique-taxidermy.com'] 'www.yahoo.com'] jobs = [gevent.spawn(gevent.socket.gethostbyname, host) for host in hosts] gevent.joinall(jobs, timeout=5) for job in jobs: print(job.value)
def close_spider(self, spider): self.file.close() # 堵塞协程池 print(len(self.gevent_pools)) gevent.joinall(self.gevent_pools) print('close_spider----', spider.name)
def start(self, *entry_points): """Run entry point.""" gevent.joinall([gevent.spawn(ep) for ep in entry_points])
Lock.release() return try: print "\n\n\nGoGo Tester Py - Ver %s\n\nDevloped by NKUCodingcat <*****@*****.**>\n\n" % Ver log = open(root + "log.log", "w") jobs = [ gevent.spawn(LimitCheck), ] + [ gevent.spawn(Socket_TestNext, ippool) for i in range(int(sock_thread_num) / 2) ] + [ gevent.spawn(Socket_TestNext, ippool, False) for i in range(int(sock_thread_num) - int(sock_thread_num) / 2) ] + [gevent.spawn(SSL_TestNext) for i in range(int(ssl_thread_num))] gevent.joinall(jobs) finally: print "\n\nSearch Complete. The result will be saved at %s/res" % root if not os.path.exists(root + "res"): os.mkdir(root + "res") HTMLGEN.HTMLGEN( json.dumps([i for i in Succ if IPy.IP(i[0][1]).version() == 4]), open(root + "res/ip_4.txt", "w")).close() HTMLGEN.HTMLGEN( json.dumps([i for i in Succ if IPy.IP(i[0][1]).version() == 6]), open(root + "res/ip_6.txt", "w")).close() log.close() print "\n\n"
def query_using_greenlets(module): g1 = gevent.spawn(query, module, 10, 100) g2 = gevent.spawn(query, module, 100, 1000) g3 = gevent.spawn(query, module, 1000, 5000) gevent.joinall([g1, g2, g3])
def announce(self, force=False, mode="start", pex=True): if time.time() < self.time_announce + 30 and not force: return # No reannouncing within 30 secs self.time_announce = time.time() trackers = config.trackers # Filter trackers based on supported networks if config.disable_udp: trackers = [ tracker for tracker in trackers if not tracker.startswith("udp://") ] if not self.connection_server.tor_manager.enabled: trackers = [ tracker for tracker in trackers if ".onion" not in tracker ] if mode == "update" or mode == "more": # Only announce on one tracker, increment the queried tracker id self.last_tracker_id += 1 self.last_tracker_id = self.last_tracker_id % len(trackers) trackers = [trackers[self.last_tracker_id] ] # We only going to use this one errors = [] slow = [] add_types = [] if self.connection_server: my_peer_id = self.connection_server.peer_id # Type of addresses they can reach me if self.connection_server.port_opened: add_types.append("ip4") if self.connection_server.tor_manager.enabled and self.connection_server.tor_manager.start_onions: add_types.append("onion") else: my_peer_id = "" s = time.time() announced = 0 threads = [] fileserver_port = config.fileserver_port for tracker in trackers: # Start announce threads tracker_protocol, tracker_address = tracker.split("://") thread = gevent.spawn(self.announceTracker, tracker_protocol, tracker_address, fileserver_port, add_types, my_peer_id, mode) threads.append(thread) thread.tracker_address = tracker_address thread.tracker_protocol = tracker_protocol gevent.joinall(threads, timeout=10) # Wait for announce finish for thread in threads: if thread.value: if thread.value > 1: slow.append("%.2fs %s://%s" % (thread.value, thread.tracker_protocol, thread.tracker_address)) announced += 1 else: if thread.ready(): errors.append( "%s://%s" % (thread.tracker_protocol, thread.tracker_address)) else: # Still running slow.append( "10s+ %s://%s" % (thread.tracker_protocol, thread.tracker_address)) # Save peers num self.settings["peers"] = len(self.peers) self.saveSettings() if len(errors) < len(threads): # Less errors than total tracker nums self.log.debug( "Announced types %s in mode %s to %s trackers in %.3fs, errors: %s, slow: %s" % (add_types, mode, announced, time.time() - s, errors, slow)) else: if mode != "update": self.log.error("Announce to %s trackers in %.3fs, failed" % (announced, time.time() - s)) if pex: if not [ peer for peer in self.peers.values() if peer.connection and peer.connection.connected ]: # If no connected peer yet then wait for connections gevent.spawn_later(3, self.announcePex, need_num=10) # Spawn 3 secs later else: # Else announce immediately if mode == "more": # Need more peers self.announcePex(need_num=10) else: self.announcePex()
def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False): s = time.time() self.log.debug("Downloading %s..." % inner_path) found = self.needFile(inner_path, update=self.bad_files.get(inner_path)) content_inner_dir = helper.getDirname(inner_path) if not found: self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications)) if check_modifications: # Download failed, but check modifications if its succed later self.onFileDone.once( lambda file_name: self.checkModifications(0), "check_modifications") return False # Could not download content.json self.log.debug("Got %s" % inner_path) changed, deleted = self.content_manager.loadContent( inner_path, load_includes=False) # Start download files file_threads = [] if download_files: for file_relative_path in self.content_manager.contents[ inner_path].get("files", {}).keys(): file_inner_path = content_inner_dir + file_relative_path # Start download and dont wait for finish, return the event res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) if res is not True and res is not False: # Need downloading and file is allowed file_threads.append(res) # Append evt # Optionals files if inner_path == "content.json": gevent.spawn(self.updateHashfield) if self.settings.get("autodownloadoptional"): for file_relative_path in self.content_manager.contents[ inner_path].get("files_optional", {}).keys(): file_inner_path = content_inner_dir + file_relative_path # Start download and dont wait for finish, return the event res = self.needFile( file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) if res is not True and res is not False: # Need downloading and file is allowed file_threads.append(res) # Append evt # Wait for includes download include_threads = [] for file_relative_path in self.content_manager.contents[ inner_path].get("includes", {}).keys(): file_inner_path = content_inner_dir + file_relative_path include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer) include_threads.append(include_thread) self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads))) gevent.joinall(include_threads) self.log.debug("%s: Includes download ended" % inner_path) if check_modifications: # Check if every file is up-to-date self.checkModifications(0) self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed))) gevent.joinall(file_threads) self.log.debug("%s: DownloadContent ended in %.2fs" % (inner_path, time.time() - s)) if not self.worker_manager.tasks: self.onComplete() # No more task trigger site complete return True
protocols=['http-only', 'chat']) ws.daemon = False ws2.daemon = False ws.name = "Reader/Writer" ws2.name = "Reader" ws.connect() ws2.connect() ''' what we're doing here is that we're sending new entities and getting them back on the websocket ''' greenlets = [ gevent.spawn(ws.incoming), gevent.spawn(ws.outgoing), ] gws2 = gevent.spawn(ws2.incoming) gevent.joinall(greenlets) ws2.close() gws2.join(timeout=1) # here's our final test print("Counts: %s %s" % (ws.count, ws2.count)) assert ws.count == calls, ("Expected Responses were given! %d %d" % (ws.count, calls)) assert ws2.count >= (9 * calls / 10), ( "2nd Client got less than 9/10 of the results! %s" % ws2.count) print("Looks like the tests passed!") finally: #except KeyboardInterrupt: ws.close() ws2.close() gevent.sleep(1) os.system("kill -9 $(lsof -t -i:8000)")
def process_start(tasks, myip, queue2, cntl): spawns = [] for task in tasks: spawns.append(gevent.spawn(detect_proxy, myip, task, queue2)) gevent.joinall(spawns) cntl.put(os.getpid()) # 子进程退出是加入控制队列
def publish(self, limit="default", inner_path="content.json"): published = [] # Successfully published (Peer) publishers = [] # Publisher threads if not self.peers: self.announce() threads = 5 if limit == "default": if len(self.peers) > 50: limit = 3 threads = 3 else: limit = 5 connected_peers = self.getConnectedPeers() if len(connected_peers ) > limit * 2: # Publish to already connected peers if possible peers = connected_peers else: peers = self.peers.values() self.log.info( "Publishing %s to %s/%s peers (connected: %s)..." % (inner_path, limit, len(self.peers), len(connected_peers))) if not peers: return 0 # No peers found random.shuffle(peers) event_done = gevent.event.AsyncResult() for i in range(min(len(self.peers), limit, threads)): publisher = gevent.spawn(self.publisher, inner_path, peers, published, limit, event_done) publishers.append(publisher) event_done.get() # Wait for done if len(published) < min(len(self.peers), limit): time.sleep(0.2) # If less than we need sleep a bit if len(published) == 0: gevent.joinall( publishers) # No successful publish, wait for all publisher # Make sure the connected passive peers got the update passive_peers = [ peer for peer in peers if peer.connection and not peer.connection.closed and peer.key.endswith(":0") and peer not in published ] # Every connected passive peer that we not published to self.log.info( "Successfuly %s published to %s peers, publishing to %s more passive peers" % (inner_path, len(published), len(passive_peers))) for peer in passive_peers[0:3]: gevent.spawn(self.publisher, inner_path, passive_peers, published, limit=limit + 3) # Send my hashfield to every connected peer if changed gevent.spawn(self.sendMyHashfield, 100) return len(published)
def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("config") args = parser.parse_args() with open(args.config, "r") as handler: config = json.load(handler) # validate the endpoints node_to_endpoint = dict() node_to_address = dict() for node_name, node_info in config["nodes"].items(): if urlsplit(node_info["endpoint"]).scheme == "": raise ValueError("'endpoint' must have the protocol defined") url_deposit = f"{node_info['endpoint']}/api/{API_VERSION}/address" result = requests.get(url_deposit).json() if result["our_address"] != node_info["address"]: raise ValueError( f"Address mismatch, configuration {node_info['address']}, " f"API response {result['our_address']}") node_to_endpoint[node_name] = node_info["endpoint"] node_to_address[node_name] = node_info["address"] nodeaddress_to_channelopenqueue: OpenQueue = defaultdict(list) nodeaddress_to_channeldepositqueue: DepositQueue = defaultdict(list) # Schedule the requests to evenly distribute the load. This is important # because as of 0.100.5 channel cannot be opened concurrently, by dividing # the load evenly we make sure the channels are opened as fast as possible. for token_address, channels_to_open in config["networks"].items(): for channel in channels_to_open: node1 = channel["node1"] node2 = channel["node2"] participant1 = node_to_address[node1] participant2 = node_to_address[node2] current_channel1 = channel_details(node_to_endpoint[node1], token_address, participant2) current_channel2 = channel_details(node_to_endpoint[node2], token_address, participant1) nodes_are_synchronized = bool(current_channel1) == bool( current_channel2) msg = (f"The channel must exist in both or neither of the nodes.\n" f"{current_channel1}\n" f"{current_channel2}") assert nodes_are_synchronized, msg if current_channel1 is None: queue_channel_open( nodeaddress_to_channelopenqueue, nodeaddress_to_channeldepositqueue, channel, token_address, node_to_address, node_to_endpoint, ) else: assert current_channel1 and current_channel2 queue_channel_deposit( nodeaddress_to_channeldepositqueue, channel, current_channel1, current_channel2, token_address, node_to_address, node_to_endpoint, ) open_greenlets = set( gevent.spawn(channel_open, open_queue) for open_queue in nodeaddress_to_channelopenqueue.values()) deposit_greenlets = set( gevent.spawn(channel_deposit_with_the_same_token_network, deposit_queue) for deposit_queue in nodeaddress_to_channeldepositqueue.values()) all_greenlets = set() all_greenlets.update(open_greenlets) all_greenlets.update(deposit_greenlets) gevent.joinall(all_greenlets, raise_error=True)
import gevent, time def foo1(): print('a') gevent.sleep(2) print(3) def foo2(): print(2) gevent.sleep(2) print(4) a = gevent.spawn(foo1) b = gevent.spawn(foo2) gevent.joinall([a, b])
def main(): """Entry point for RWS. return (int): exit code (0 on success, 1 on error) """ parser = argparse.ArgumentParser( description="Ranking for CMS.") parser.add_argument("--config", type=argparse.FileType("rt"), help="override config file") parser.add_argument("-d", "--drop", action="store_true", help="drop the data already stored") args = parser.parse_args() config.load(args.config) if args.drop: print("Are you sure you want to delete directory %s? [y/N]" % config.lib_dir, end='') ans = raw_input().lower() if ans in ['y', 'yes']: print("Removing directory %s." % config.lib_dir) shutil.rmtree(config.lib_dir) else: print("Not removing directory %s." % config.lib_dir) return 1 Contest.store.load_from_disk() Task.store.load_from_disk() Team.store.load_from_disk() User.store.load_from_disk() Submission.store.load_from_disk() Subchange.store.load_from_disk() Scoring.store.init_store() toplevel_handler = RoutingHandler(DataWatcher(), ImageHandler( os.path.join(config.lib_dir, '%(name)s'), os.path.join(config.web_dir, 'img', 'logo.png'))) wsgi_app = SharedDataMiddleware(DispatcherMiddleware( toplevel_handler, { '/contests': StoreHandler(Contest.store), '/tasks': StoreHandler(Task.store), '/teams': StoreHandler(Team.store), '/users': StoreHandler(User.store), '/submissions': StoreHandler(Submission.store), '/subchanges': StoreHandler(Subchange.store), '/faces': ImageHandler( os.path.join(config.lib_dir, 'faces', '%(name)s'), os.path.join(config.web_dir, 'img', 'face.png')), '/flags': ImageHandler( os.path.join(config.lib_dir, 'flags', '%(name)s'), os.path.join(config.web_dir, 'img', 'flag.png')), }), {'/': config.web_dir}) servers = list() if config.http_port is not None: http_server = WSGIServer( (config.bind_address, config.http_port), wsgi_app) servers.append(http_server) if config.https_port is not None: https_server = WSGIServer( (config.bind_address, config.https_port), wsgi_app, certfile=config.https_certfile, keyfile=config.https_keyfile) servers.append(https_server) try: gevent.joinall(list(gevent.spawn(s.serve_forever) for s in servers)) except KeyboardInterrupt: pass finally: gevent.joinall(list(gevent.spawn(s.stop) for s in servers)) return 0
import gevent from gevent import Greenlet def foo(message, n): gevent.sleep(0) print message thread1 = Greenlet.spawn(foo, "Hello", 1) thread2 = Greenlet.spawn(foo, "I live!", 2) thread3 = gevent.spawn(lambda x: (x + 1), 2) threads = [thread1, thread2, thread3] gevent.joinall(threads)
def run(self): generator_run_tasks = [gen.run_generator() for gen in self._generators] generator_tasks = [gen_task for gen_task_sublist in \ generator_run_tasks for gen_task in gen_task_sublist ] gevent.joinall(generator_tasks)
def run( privatekey, registry_contract_address, secret_registry_contract_address, discovery_contract_address, listen_address, structlog, logfile, scenario, stage_prefix, ): # pylint: disable=unused-argument # TODO: only enabled structlog on "initiators" structlog.configure(structlog, log_file=logfile) (listen_host, listen_port) = split_endpoint(listen_address) config = App.DEFAULT_CONFIG.copy() config['host'] = listen_host config['port'] = listen_port config['privatekey_hex'] = privatekey privatekey_bin = decode_hex(privatekey) rpc_client = JSONRPCClient( '127.0.0.1', 8545, privatekey_bin, ) blockchain_service = BlockChainService(privatekey_bin, rpc_client) discovery = ContractDiscovery( blockchain_service, decode_hex(discovery_contract_address), ) registry = blockchain_service.token_network_registry( registry_contract_address, ) secret_registry = blockchain_service.secret_registry( secret_registry_contract_address, ) throttle_policy = TokenBucket( config['protocol']['throttle_capacity'], config['protocol']['throttle_fill_rate'], ) transport = UDPTransport( discovery=discovery, udpsocket=gevent.server._udp_socket((listen_host, listen_port)), throttle_policy=throttle_policy, config=config['protocol'], ) app = App( config=config, chain=blockchain_service, query_start_block=0, default_registry=registry, default_secret_registry=secret_registry, transport=transport, discovery=discovery, ) app.start() app.discovery.register( app.raiden.address, listen_host, listen_port, ) from_block = 0 app.raiden.install_all_blockchain_filters( app.raiden.default_registry, app.raiden.default_secret_registry, from_block, ) if scenario: script = json.load(scenario) tools = ConsoleTools( app.raiden, app.discovery, app.config['settle_timeout'], app.config['reveal_timeout'], ) transfers_by_peer = {} tokens = script['tokens'] token_address = None peer = None our_node = hexlify(app.raiden.address) log.warning('our address is {}'.format(our_node)) for token in tokens: # skip tokens that we're not part of nodes = token['channels'] if our_node not in nodes: continue partner_nodes = [node for node in nodes if node != our_node] # allow for prefunded tokens if 'token_address' in token: token_address = token['token_address'] else: token_address = tools.create_token(registry_contract_address) transfers_with_amount = token['transfers_with_amount'] # FIXME: in order to do bidirectional channels, only one side # (i.e. only token['channels'][0]) should # open; others should join by calling # raiden.api.deposit, AFTER the channel came alive! # NOTE: leaving unidirectional for now because it most # probably will get to higher throughput log.warning('Waiting for all nodes to come online') api = RaidenAPI(app.raiden) for node in partner_nodes: api.start_health_check_for(node) while True: all_reachable = all( api.get_node_network_state(node) == NODE_NETWORK_REACHABLE for node in partner_nodes) if all_reachable: break gevent.sleep(5) log.warning('All nodes are online') if our_node != nodes[-1]: our_index = nodes.index(our_node) peer = nodes[our_index + 1] tools.token_network_register( app.raiden.default_registry.address, token_address) amount = transfers_with_amount[nodes[-1]] while True: try: app.discovery.get(peer.decode('hex')) break except KeyError: log.warning( 'Error: peer {} not found in discovery'.format( peer)) time.sleep(random.randrange(30)) while True: try: log.warning('Opening channel with {} for {}'.format( peer, token_address)) api.channel_open(app.raiden.default_registry.address, token_address, peer) break except KeyError: log.warning( 'Error: could not open channel with {}'.format( peer)) time.sleep(random.randrange(30)) while True: try: log.warning('Funding channel with {} for {}'.format( peer, token_address)) api.channel_deposit( app.raiden.default_registry.address, token_address, peer, amount, ) break except Exception: log.warning( 'Error: could not deposit {} for {}'.format( amount, peer)) time.sleep(random.randrange(30)) if our_index == 0: last_node = nodes[-1] transfers_by_peer[last_node] = int(amount) if stage_prefix is not None: open('{}.stage1'.format(stage_prefix), 'a').close() log.warning('Done with initialization, waiting to continue...') event = gevent.event.Event() gevent.signal(signal.SIGUSR2, event.set) event.wait() transfer_results = {'total_time': 0, 'timestamps': []} def transfer(token_address, amount_per_transfer, total_transfers, peer, is_async): def transfer_(): log.warning('Making {} transfers to {}'.format( total_transfers, peer)) initial_time = time.time() times = [0] * total_transfers for index in range(total_transfers): RaidenAPI(app.raiden).transfer( app.raiden.default_registry.address, token_address.decode('hex'), amount_per_transfer, peer, ) times[index] = time.time() transfer_results['total_time'] = time.time() - initial_time transfer_results['timestamps'] = times log.warning('Making {} transfers took {}'.format( total_transfers, transfer_results['total_time'])) log.warning('Times: {}'.format(times)) if is_async: return gevent.spawn(transfer_) else: transfer_() # If sending to multiple targets, do it asynchronously, otherwise # keep it simple and just send to the single target on my thread. if len(transfers_by_peer) > 1: greenlets = [] for peer_, amount in transfers_by_peer.items(): greenlet = transfer(token_address, 1, amount, peer_, True) if greenlet is not None: greenlets.append(greenlet) gevent.joinall(greenlets) elif len(transfers_by_peer) == 1: for peer_, amount in transfers_by_peer.items(): transfer(token_address, 1, amount, peer_, False) log.warning('Waiting for termination') open('{}.stage2'.format(stage_prefix), 'a').close() log.warning('Waiting for transfers to finish, will write results...') event = gevent.event.Event() gevent.signal(signal.SIGUSR2, event.set) event.wait() open('{}.stage3'.format(stage_prefix), 'a').close() event = gevent.event.Event() gevent.signal(signal.SIGQUIT, event.set) gevent.signal(signal.SIGTERM, event.set) gevent.signal(signal.SIGINT, event.set) event.wait() else: log.warning('No scenario file supplied, doing nothing!') open('{}.stage2'.format(stage_prefix), 'a').close() event = gevent.event.Event() gevent.signal(signal.SIGQUIT, event.set) gevent.signal(signal.SIGTERM, event.set) gevent.signal(signal.SIGINT, event.set) event.wait() app.stop()
def startFindOptional(self, reset_task=False, find_more=False, high_priority=False): # Wait for more file requests if len(self.tasks) < 20 or high_priority: time.sleep(0.01) if len(self.tasks) > 90: time.sleep(5) else: time.sleep(0.5) optional_tasks = [ task for task in self.tasks if task["optional_hash_id"] ] if not optional_tasks: return False optional_hash_ids = set( [task["optional_hash_id"] for task in optional_tasks]) time_tasks = self.time_task_added self.log.debug( "Finding peers for optional files: %s (reset_task: %s, find_more: %s)" % (optional_hash_ids, reset_task, find_more)) found = self.findOptionalTasks(optional_tasks, reset_task=reset_task) if found: found_peers = set( [peer for peers in found.values() for peer in peers]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids) or find_more or ( high_priority and any(len(peers) < 10 for peers in found.itervalues())): self.log.debug("No local result for optional files: %s" % (optional_hash_ids - set(found))) # Query hashfield from connected peers threads = [] peers = self.site.getConnectedPeers() if not peers: peers = self.site.getConnectablePeers() for peer in peers: if not peer.time_hashfield: threads.append(gevent.spawn(peer.updateHashfield)) gevent.joinall(threads, timeout=5) if time_tasks != self.time_task_added: # New task added since start optional_tasks = [ task for task in self.tasks if task["optional_hash_id"] ] optional_hash_ids = set( [task["optional_hash_id"] for task in optional_tasks]) found = self.findOptionalTasks(optional_tasks) self.log.debug( "Found optional files after query hashtable connected peers: %s/%s" % (len(found), len(optional_hash_ids))) if found: found_peers = set([ peer for hash_id_peers in found.values() for peer in hash_id_peers ]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids) or find_more: self.log.debug( "No connected hashtable result for optional files: %s" % (optional_hash_ids - set(found))) # Try to query connected peers threads = [] peers = [ peer for peer in self.site.getConnectedPeers() if peer not in self.asked_peers ] if not peers: peers = self.site.getConnectablePeers() for peer in peers: threads.append( gevent.spawn(peer.findHashIds, list(optional_hash_ids))) self.asked_peers.append(peer) for i in range(5): time.sleep(1) thread_values = [ thread.value for thread in threads if thread.value ] if not thread_values: continue found_ips = helper.mergeDicts(thread_values) found = self.addOptionalPeers(found_ips) self.log.debug( "Found optional files after findhash connected peers: %s/%s (asked: %s)" % (len(found), len(optional_hash_ids), len(threads))) if found: found_peers = set([ peer for hash_id_peers in found.values() for peer in hash_id_peers ]) self.startWorkers(found_peers) if len(thread_values) == len(threads): # Got result from all started thread break if len(found) < len(optional_hash_ids): self.log.debug("No findHash result, try random peers: %s" % (optional_hash_ids - set(found))) # Try to query random peers if time_tasks != self.time_task_added: # New task added since start optional_tasks = [ task for task in self.tasks if task["optional_hash_id"] ] optional_hash_ids = set( [task["optional_hash_id"] for task in optional_tasks]) threads = [] peers = self.site.getConnectablePeers(ignore=self.asked_peers) for peer in peers: threads.append( gevent.spawn(peer.findHashIds, list(optional_hash_ids))) self.asked_peers.append(peer) gevent.joinall(threads, timeout=15) found_ips = helper.mergeDicts( [thread.value for thread in threads if thread.value]) found = self.addOptionalPeers(found_ips) self.log.debug( "Found optional files after findhash random peers: %s/%s" % (len(found), len(optional_hash_ids))) if found: found_peers = set([ peer for hash_id_peers in found.values() for peer in hash_id_peers ]) self.startWorkers(found_peers) if len(found) < len(optional_hash_ids): self.log.debug("No findhash result for optional files: %s" % (optional_hash_ids - set(found)))
def mesos_cpu_metrics_provider( marathon_service_config, marathon_tasks, mesos_tasks, log_utilization_data={}, noop=False, **kwargs, ): """ Gets the mean cpu utilization of a service across all of its tasks. :param marathon_service_config: the MarathonServiceConfig to get data from :param marathon_tasks: Marathon tasks to get data from :param mesos_tasks: Mesos tasks to get data from :param log_utilization_data: A dict used to transfer utilization data to autoscale_marathon_instance() :returns: the service's mean utilization, from 0 to 1 """ autoscaling_root = compose_autoscaling_zookeeper_root( service=marathon_service_config.service, instance=marathon_service_config.instance, ) zk_last_time_path = '%s/cpu_last_time' % autoscaling_root zk_last_cpu_data = '%s/cpu_data' % autoscaling_root with ZookeeperPool() as zk: try: last_time = zk.get(zk_last_time_path)[0].decode('utf8') last_cpu_data = zk.get(zk_last_cpu_data)[0].decode('utf8') log_utilization_data[last_time] = last_cpu_data last_time = float(last_time) last_cpu_data = (datum for datum in last_cpu_data.split(',') if datum) except NoNodeError: last_time = 0.0 last_cpu_data = [] monkey.patch_socket() jobs = [gevent.spawn(task.stats_callable) for task in mesos_tasks] gevent.joinall(jobs, timeout=60) mesos_tasks_first_run = dict( zip([task['id'] for task in mesos_tasks], [job.value for job in jobs])) current_time = int(datetime.now().strftime('%s')) time_delta = current_time - last_time # Mesos slave statistics endpoint returns a crazy CPU value when a container is # in the process of being killed. So we fetch twice, and use the lower of the two. # First, we clear the cache. for task in mesos_tasks: del task.slave._cache['stats'] jobs = [gevent.spawn(task.stats_callable) for task in mesos_tasks] gevent.joinall(jobs, timeout=60) mesos_tasks_second_run = dict( zip([task['id'] for task in mesos_tasks], [job.value for job in jobs])) mesos_cpu_data = {} for task_id, stats in mesos_tasks_first_run.items(): stats2 = mesos_tasks_second_run.get(task_id) if stats is not None and stats2 is not None: try: utime = min(float(stats['cpus_user_time_secs']), float(stats2['cpus_user_time_secs'])) stime = min(float(stats['cpus_system_time_secs']), float(stats2['cpus_system_time_secs'])) limit = float(stats['cpus_limit']) - .1 mesos_cpu_data[task_id] = (stime + utime) / limit except KeyError: pass if not mesos_cpu_data: raise MetricsProviderNoDataError( "Couldn't get any cpu data from Mesos") cpu_data_csv = ','.join('%s:%s' % (cpu_seconds, task_id) for task_id, cpu_seconds in mesos_cpu_data.items()) log_utilization_data[str(current_time)] = cpu_data_csv if not noop: with ZookeeperPool() as zk: zk.ensure_path(zk_last_cpu_data) zk.ensure_path(zk_last_time_path) zk.set(zk_last_cpu_data, str(cpu_data_csv).encode('utf8')) zk.set(zk_last_time_path, str(current_time).encode('utf8')) utilization = {} for datum in last_cpu_data: last_cpu_seconds, task_id = datum.split(':') if task_id in mesos_cpu_data: utilization[task_id] = (mesos_cpu_data[task_id] - float(last_cpu_seconds)) / time_delta if not utilization: raise MetricsProviderNoDataError( """The mesos_cpu metrics provider doesn't have Zookeeper data for this service. This is expected for its first run.""" ) task_utilization = utilization.values() mean_utilization = mean(task_utilization) return mean_utilization
import time start = time.time() tic = lambda: 'at %1.1f seconds' % (time.time() - start) def gr1(): # Busy waits for a second, but we don't want to stick around... print('Started Polling: %s' % tic()) select.select([], [], [], 2) print('Ended Polling: %s' % tic()) def gr2(): # Busy waits for a second, but we don't want to stick around... print('Started Polling: %s' % tic()) select.select([], [], [], 2) print('Ended Polling: %s' % tic()) def gr3(): print("Hey lets do some stuff while the greenlets poll, %s" % tic()) gevent.sleep(1) gevent.joinall([ gevent.spawn(gr1), gevent.spawn(gr2), gevent.spawn(gr3), ])
def run(self): logger.info('Starting databridge') for n, w in self.workers.items(): logger.info('starting {}'.format(n)) w.start() gevent.joinall([w.g for w in self.workers.values()])
def by_urllib2(): jobs = [gevent.spawn(worker, url, True) for url in urls] gevent.joinall(jobs)
def tearDown(self): self.kamstrup_management_server.stop() gevent.joinall([self.server_greenlet]) # tidy up (again)... conpot_core.get_sessionManager().purge_sessions()
import gevent def test(n): for x in range(n): print(gevent.getcurrent(), x) #写一个耗时操作,协程在运行的时候碰见了耗时操作就会切换另一个协程执行 #不写效果不一样 #协程中休眠使用如下这种 gevent.sleep(0.5) gevent.joinall( [gevent.spawn(test, 5), gevent.spawn(test, 5), gevent.spawn(test, 5)])
#!/usr/bin/env python # -*- coding:utf-8 -*- # import time import gevent from gevent import monkey, time monkey.patch_all() def func1(): print(123) time.sleep(3) print(456) def func2(): print('---------') time.sleep(1) print('=========') g1 = gevent.spawn(func1) g2 = gevent.spawn(func2) gevent.joinall([g1, g2]) print('main')
#!/usr/bin/env python # -*-coding:utf-8-*- # date:2018/8/7 from urllib.request import urlopen import gevent def f(url): print("GET: %s" % url) resp = urlopen(url) data = resp.read() with open("xx.html", "wb") as f: f.write(data) print("%d bytes received from %s" % (len(data), url)) f('http://www.ifeng.com/?_zbs_firefox_gg') f('http://www.ifeng.com/?_zbs_firefox_gg') f('http://www.ifeng.com/?_zbs_firefox_gg') # 这种方式 gevent.joinall 快很多 gevent.joinall([ gevent.spawn(f, 'http://www.ifeng.com/?_zbs_firefox_gg'), gevent.spawn(f, 'http://www.ifeng.com/?_zbs_firefox_gg'), gevent.spawn(f, 'http://www.ifeng.com/?_zbs_firefox_gg') ])
def by_requests(): jobs = [gevent.spawn(worker, url) for url in urls] gevent.joinall(jobs)
def func1(): print("func1 start") time.sleep(2) print("func1 end") def func2(): print("func2 start") time.sleep(2) print("func2 end") f1 = gevent.spawn(func1) f2 = gevent.spawn(func2) gevent.joinall([f1, f2]) """ func1 start func2 start func1 end func2 end """ """ gevent.spawn(func.argv) 功能:生成协程对象 参数: func: 协程函数 argv: 协程函数的参数(不定参)
def test_regression_multiple_revealsecret(raiden_network, token_addresses): """ Multiple RevealSecret messages arriving at the same time must be handled properly. Secret handling followed these steps: The Secret message arrives The secret is registered The channel is updated and the correspoding lock is removed * A balance proof for the new channel state is created and sent to the payer The channel is unregistered for the given hashlock The step marked with an asterisk above introduced a context-switch, this allowed a second Reveal Secret message to be handled before the channel was unregistered, because the channel was already updated an exception was raised for an unknown secret. """ app0, app1 = raiden_network token = token_addresses[0] identifier = 1 secret = sha3('test_regression_multiple_revealsecret') hashlock = sha3(secret) expiration = app0.raiden.get_block_number() + 100 amount = 10 mediated_transfer = channel(app0, app1, token).create_mediatedtransfer( transfer_initiator=app0.raiden.address, transfer_target=app1.raiden.address, fee=0, amount=amount, identifier=identifier, expiration=expiration, hashlock=hashlock, ) app0.raiden.sign(mediated_transfer) message_data = mediated_transfer.encode() app1.raiden.protocol.receive(message_data) reveal_secret = RevealSecret(secret) app0.raiden.sign(reveal_secret) reveal_secret_data = reveal_secret.encode() secret = Secret( identifier=identifier, nonce=mediated_transfer.nonce + 1, channel=channel(app0, app1, token).channel_address, transferred_amount=amount, locksroot=EMPTY_MERKLE_ROOT, secret=secret, ) app0.raiden.sign(secret) secret_data = secret.encode() messages = [ secret_data, reveal_secret_data, ] wait = [ gevent.spawn_later( .1, app1.raiden.protocol.receive, data, ) for data in messages ] gevent.joinall(wait)