def api_delete_feed():
feed_id = request.json['id']
inbox = Queue()
mail = Mail(inbox, {'type': 'delete-feed', 'feed_id': feed_id})
deadlineManager.inbox.put(mail)
resp = inbox.get()
return jsonify({"msg": resp['success']})
class FlowTests(TestCase):
def create(self, conf={}, events=[]):
self.input = Queue()
self.output = Queue()
context = DummyContext()
with context:
self.i = self.create_stage(**conf)
self.input = self.i.setup(self.output)
self.assertEquals(1, len(context.stages))
self.i.start()
for ev in events:
self.input.put(ev)
return self.i
def wait(self, timeout=1.0, events=1):
with gevent.Timeout(timeout):
# wait for input to be consumed and output to be produced
while self.input.qsize():
gevent.sleep(0.0)
while self.output.qsize() < events:
gevent.sleep(0.0)
self.i.stop()
if events:
return [self.output.get() for n in xrange(events)]
def test_kill_run(self, datetime, kill_pid_tree_mock):
"""
Test :func:`.kill_run`.
"""
event_queue = Mock()
kill_request = Mock()
kill_request.id = 1234
kill_request.run.pid = 5678
dts = datetime.now.return_value.isoformat.return_value
kill_queue = Queue()
kill_queue.put(kill_request)
exit_queue = Mock()
exit_queue_return = [Empty, None]
def exit_queue_side_effect(*args, **kwargs):
value = exit_queue_return.pop(0)
if callable(value):
raise value()
exit_queue.get.side_effect = exit_queue_side_effect
kill_run(kill_queue, event_queue, exit_queue)
kill_pid_tree_mock.assert_called_with(5678)
kill_request.patch.assert_called_with({
'execute_dts': dts,
})
event_queue.put.assert_called_with((
'{"kill_request_id": 1234, "kind": "kill_request", '
'"event": "executed"}'
))
def __init__(self, config):
super(ContractingDataBridge, self).__init__()
self.config = config
self.tenders_sync_client = TendersClientSync('',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.client = TendersClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.contracting_client = ContractingClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version')
)
self.initial_sync_point = {}
self.tenders_queue = Queue(maxsize=500)
self.handicap_contracts_queue = Queue(maxsize=500)
self.contracts_put_queue = Queue(maxsize=500)
self.contracts_retry_put_queue = Queue(maxsize=500)
class CHubCallbackQueueBase(CHubCallbackBasicBase):
def __init__(self,sHubId):
CHubCallbackBasicBase.__init__(self,sHubId)
self.__queue4Return = Queue() #当前应答队列
def PutCmdStrToReturnQueue(self, lsCmdStr):
self.__queue4Return.put(lsCmdStr)
def GetCmdStrFmReturnQueue(self):
return self.__queue4Return.get() # (sClientIPPort,dwCmdId,CmdOStr)
def HandleRequestCmd(self, sClientIPPort, dwCmdId, CmdIStr):
# 处理客户端请求命令
bDone = CHubCallbackBasicBase.HandleRequestCmd(self, sClientIPPort, dwCmdId, CmdIStr)
if not bDone and CmdIStr[0].startswith(CMD0_ECHO_CMD):
CmdOStr = ['OK','CHubCallbackQueueBase']
CmdOStr.extend(CmdIStr)
dwCmdId = GetCmdReplyFmRequest(dwCmdId)
self.PutCmdStrToReturnQueue([sClientIPPort,dwCmdId,CmdOStr])
bDone = True
return bDone
def DoHandleCheckAllLinkReply(self):
# 处理检查所有链接的应答返还消息（包括通知消息）等
# 该函数在该类中实现后，一般情况下子类无需再继承。
while not self.bQuitLoopFlag:
return self.GetCmdStrFmReturnQueue()
def joinall(greenlets, timeout=None, raise_error=False, count=None):
from gevent.queue import Queue
queue = Queue()
put = queue.put
if count is None:
count = len(greenlets)
timeout = Timeout.start_new(timeout)
try:
try:
for greenlet in greenlets:
greenlet.rawlink(put)
if raise_error:
for _ in xrange(count):
greenlet = queue.get()
if not greenlet.successful():
raise greenlet.exception
else:
for _ in xrange(count):
queue.get()
except:
if sys.exc_info()[1] is not timeout:
raise
finally:
for greenlet in greenlets:
greenlet.unlink(put)
finally:
timeout.cancel()
def test_last_update_cache(self):
handle = self.start_worker()
queue = Queue()
o_process = handle.process
def new_process(msg):
o_process(msg)
queue.put(True)
handle.process = new_process
definition = SBE37_CDM_stream_definition()
publisher = Publisher()
stream_def_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id)
time = float(0.0)
for granule in self.make_points(definition=definition, stream_id=stream_id, N=10):
publisher.publish(granule, to_name=(self.XP, stream_id+'.data'))
# Determinism sucks
try:
queue.get(timeout=5)
except Empty:
self.assertTrue(False, 'Process never received the message.')
doc = self.db.read(stream_id)
ntime = doc.variables['time'].value
self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.')
time = ntime
class __AudioNode(gevent.Greenlet):
RATE = 44100
CHUNK = 512
PORT = 20000
def __init__(self, is_log=True):
gevent.Greenlet.__init__(self)
self.is_log = is_log
self.command = Queue()
def is_quit(self):
try:
cmd = self.command.get_nowait()
return cmd == "q"
except Empty:
return False
def stop(self, msg=""):
self.command.put("q")
if self.is_log:
print "%s - stopping %s" % (self.__class__.__name__, msg)
def _run(self):
if self.is_log:
print "starting: %s" % self.__class__.__name__
try:
self.engine()
finally:
self.pa.close()
self.sock.close()
def get_subscriber(self, last_event_id=None):
"""Obtain a new subscriber.
The returned subscriber will receive all messages after the one
with the given index (if they are still in the cache).
last_event_id (unicode): the ID of the last message the client
did receive, to request the one generated since then to be
sent again. If not given no past message will be sent.
return (Subscriber): a new subscriber instance.
"""
queue = Queue()
# If a valid last_event_id is provided see if cache can supply
# missed events.
if last_event_id is not None and \
re.match("^[0-9A-Fa-f]+$", last_event_id):
last_event_key = int(last_event_id, 16)
if len(self._cache) > 0 and last_event_key >= self._cache[0][0]:
# All missed events are in cache.
for key, msg in self._cache:
if key > last_event_key:
queue.put(msg)
else:
# Some events may be missing. Ask to reinit.
queue.put(b"event:reinit\n\n")
# Store the queue and return a subscriber bound to it.
self._sub_queues.add(queue)
return Subscriber(queue)
class Qoorate(BrooklynCodeBrubeck):
"""Custom application class for Qoorate."""
def __init__(self, settings_file=None, project_dir=None,
*args, **kwargs):
""" Most of the parameters are dealt with by Brubeck,
Additional functionality follow call to super
"""
super(Qoorate, self).__init__(settings_file, project_dir, **kwargs)
pool_size = 10
if self.db_conn == None:
# create our db_conn if we have the settings to
if settings_file != None:
mysql_settings = self.get_settings('mysql')
if mysql_settings != None:
logging.debug("creating application db_conn pool")
self.db_conn = Queue()
for i in range(pool_size):
self.db_conn.put_nowait(create_db_conn(mysql_settings))
self.template_env.filters['sc'] = unsanitize_safe_htmlentities
def determine_relevency(self, item):
"""schedule an indexing using concurrency"""
logging.info("qoorate_determine_relevency, start: %s" % item)
logging.info("qoorate_generate_relevency, star greenlet: %s" % item)
qoorate_determine_relevency(item)
logging.info("qoorate_generate_relevency, end: %s" % item)
class Spider(object): """docstring for Spider""" def __init__(self, route, RDB): self.route = route self.visited = RDB #must be a redis client self.todolst = Queue(100) def put(self, item): self.todolst.put(item) def _fetch(self, timeout): todo = self.todolst route = self.route visited = self.visited try: while True: url = todo.get(timeout=timeout) handler = route.match(url) if not handler: continue hdl = handler(url) next_urls = hdl.get() visited.set(url,url) gevent.sleep(0.1) [todo.put(ul,timeout=timeout+10) for ul in next_urls if not (visited.exists(ul) or todo.full())] except Empty,Full: #except : #fix me traceback.print_exc() return
def parallel_loop(func=None, elem=None, *args, **kwargs):
import gevent
from gevent.queue import Queue
ret = []
api.bui.cli.logger.debug('Using gevent')
if not callable(func):
api.abort(500, 'The provided \'func\' is not callable!')
if not elem:
return []
output = Queue()
processes = [
gevent.spawn(
func,
e,
output,
*args,
**kwargs
) for e in elem
]
# wait for process termination
gevent.joinall(processes)
for p in processes:
tmp = output.get()
if isinstance(tmp, basestring):
api.abort(500, tmp)
elif tmp:
ret.append(tmp)
return ret
class WebSocketHandler(ThreadedHandler):
"""WebSocket API for handlers
This provides a socket-like interface similar to the browser
WebSocket API for managing a WebSocket connection.
"""
def __init__(self, sock, protocols, extensions, environ):
ThreadedHandler.__init__(self, sock, protocols, extensions)
self.environ = environ
self._messages = Queue()
self._lock = Lock()
self._th = gevent.spawn(self._receive)
def closed(self, code, reason=None):
self._messages.put(StreamClosed(code, reason))
def received_message(self, m):
self._messages.put(copy.deepcopy(m))
def receive(self, msg_obj=False):
msg = self._messages.get()
if isinstance(msg, StreamClosed):
# Maybe we'll do something better
return None
if msg_obj:
return msg
else:
return msg.data
class Actor(gevent.Greenlet):
def __init__(self):
self.running = False
self.inbox = Queue()
gevent.Greenlet.__init__(self)
def received(self, message):
"""
Define your subclass to handle incoming messages here...
"""
raise NotImplementedError()
def cast(self, message):
""" Send a message to the actor.
If the actor is busy, the message will be enqueued for later
consumption. There is no return value.
>>> a = Actor()
>>> a.received = lambda msg: msg
>>> a.cast("hello")
"""
self.inbox.put_nowait(message)
def _run(self):
self.running = True
while self.running:
message = self.inbox.get()
self.received(message)
class WorkerPool(object):
def __init__(self):
self.pool_size = options.pool_size
self.job_pool = Pool(size=self.pool_size)
self.result = Queue()
self.target_queue = Queue()
def add_job(self, job_func, *args, **kwargs):
job = self.job_pool.apply_async(
job_func,
args=args,
kwds=kwargs,
callback=self._call_func)
self.job_pool.add(job)
def run(self, timeout=None):
self.job_pool.join(timeout=timeout, raise_error=False)
def _call_func(self, job_ret):
if job_ret:
self.result.put(job_ret)
def shutdown(self):
self.job_pool.kill()
def test_async_multi_publish_consume():
with conn.channel() as channel:
# first message
message_body = 'test_async_multi_publish_consume message 1'
channel.basic_publish(
exchange='unit_test_room',
routing_key='user1',
body=message_body
)
recv_queue = Queue()
rchannel = conn.allocate_channel()
rchannel.basic_consume(queue='listener1', callback=recv_queue.put)
resp = recv_queue.get()
eq_(resp.body, message_body)
resp.ack()
assert recv_queue.empty()
with conn.channel() as channel:
# second message
message_body = 'test_async_multi_publish_consume message 2'
channel.basic_publish(
exchange='unit_test_room',
routing_key='user1',
body=message_body
)
resp = recv_queue.get()
eq_(resp.body, message_body)
resp.ack()
def __init__(self, addr, game):
self.hub = gevent.get_hub()
self.addr = addr
self.sock = None
self.game = game
self.time_diff = 0
self.write_queue = Queue()
self.write_tr = None
self.read_queue = Queue()
self.read_tr = None
self.dispatch_tr = None
entype = Config.get("encrypt", None)
if entype == "rc4":
self.c2s_encrypt = RC4(Config["c2s_key"]).crypt
self.s2c_encrypt = RC4(Config["s2c_key"]).crypt
elif entype == None:
self.c2s_encrypt = None
self.s2c_encrypt = None
else:
raise ValueError("not support %s encrypt"%entype)
self._sessions = {}
class LocalControlProgramDescriptor(object):
def __init__(self, hc, agentUuid, program_id):
self.log = logging.getLogger('LocalControlProgramDescriptor_{}'.format(program_id))
self.agentUuid = agentUuid
self.id = program_id
self.hc = hc
self.queue = Queue()
def _receive_msg(self, msg):
self.queue.put(msg)
def recv(self, block=True, timeout=None):
try:
self.log.debug("Waiting for msg in blocking call")
msg = self.queue.get(block=block, timeout=timeout)
return msg
except gevent.timeout.Timeout as e:
return None
except gevent.queue.Empty as e:
return None
def send(self, msg):
return self.hc.send_msg_to_local_control_program(self.id, msg, self.agentUuid)
def close(self):
return self.hc.stop_local_control_program(self.id, self.agentUuid)
class IMapUnordered(Greenlet):
def __init__(self, spawn, func, iterable):
from gevent.queue import Queue
Greenlet.__init__(self)
self.spawn = spawn
self.func = func
self.iterable = iterable
self.queue = Queue()
self.count = 0
def __iter__(self):
return self.queue
def _run(self):
try:
func = self.func
for item in self.iterable:
self.count += 1
self.spawn(func, item).rawlink(self._on_result)
finally:
self.__dict__.pop('spawn', None)
self.__dict__.pop('func', None)
self.__dict__.pop('iterable', None)
def _on_result(self, greenlet):
self.count -= 1
if greenlet.successful():
self.queue.put(greenlet.value)
if self.ready() and self.count <= 0:
self.queue.put(StopIteration)
def _run(self):
utils.log("[%s] parsing site %s" % (self, self.base))
queue = Queue()
pool = Pool(64)
seed = 'http://community.seattletimes.nwsource.com/entertainment/i_results.php?search=venue&type=Restaurant&page=1'
pool.spawn(self._parseResultsPage, pool, queue, seed, '1', True)
while True:
items = []
while not queue.empty():
item = queue.get_nowait()
items.append(item)
if 0 == len(items) and 0 == len(pool):
break
for item in items:
pool.spawn(item[0], pool, queue, item[1], item[2], item[3])
time.sleep(0.01)
pool.join()
self._output.put(StopIteration)
class SystemClient(object):
def __init__(self, socket, address, gateway, server=False):
self.socket = socket
self.address = address
self.gateway = gateway
self.server = server
self.queue = Queue(maxsize=100)
self.receiver = Receiver(self)
self.sender = Sender(self)
def start(self):
self.receiver.start()
self.sender.start()
def kill(self):
if gevent.getcurrent() in [self.receiver, self.sender]:
logger.error("SystemClient.kill() may not be called by the client's greenlets!")
self.gateway.unregister(self)
return
self.receiver.kill()
self.sender.kill()
self.socket.close()
def send(self, message, sender):
self.queue.put(message)
def close(self, message=""):
try:
logger.warning(u"Closing connection to {0} due to {1}".format(self.address, message))
self.gateway.unregister(self)
except Exception, e:
traceback.print_exc()
logger.error(u"Got {0} while trying to close and unregister a client!".format(e))
class C2DMService(object):
def __init__(self, source, email, password):
self.source = source
self.email = email
self.password = password
self._send_queue = Queue()
self._send_queue_cleared = Event()
self.log = logging.getLogger('pulsus.service.c2dm')
def _send_loop(self):
self._send_greenlet = gevent.getcurrent()
try:
self.log.info("C2DM service started")
while True:
notification = self._send_queue.get()
try:
self._do_push(notification)
except Exception, e:
self.log.exception("Error while pushing")
self._send_queue.put(notification)
gevent.sleep(5.0)
finally:
if self._send_queue.qsize() < 1 and \
not self._send_queue_cleared.is_set():
self._send_queue_cleared.set()
class WebSocketClient(ThreadedClient):
def __init__(self, url, protocols=None, version='8'):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
ThreadedClient.__init__(self, url, protocols=protocols, version=version, sock=sock)
self._lock = Semaphore()
self._th = Greenlet(self._receive)
self._messages = Queue()
self.extensions = []
def opened(self, protocols, extensions):
self.protocols = protocols
self.extensions = extensions
def received_message(self, m):
self._messages.put(copy.deepcopy(m))
def write_to_connection(self, bytes):
if not self.client_terminated:
return self.sock.sendall(bytes)
def closed(self, code, reason=None):
self._messages.put(StreamClosed(code, reason))
def receive(self, msg_obj=False):
msg = self._messages.get()
if isinstance(msg, StreamClosed):
return None
if msg_obj:
return msg
else:
return msg.data
class Worker():
def __init__(self,inputdict, timeout, outputmode, validation_func):
self.threads = []
self.queue = Queue()
self.inputdict = inputdict
self.timeout = timeout
self.outputmode = outputmode
self.validation_func = validation_func
def infi(self, th, thm):
k = 0
while k<10000:
print 'I am in INFI ', th, thm
time.sleep(.1)
print "out while infi", thm
self.queue.put_nowait(thm)
def test(self, th, thm):
print "inside test", thm
self.queue.put_nowait(thm)
def start(self, thm):
print "Hii"
self.threads.append(gevent.spawn(self.infi, 1, thm))
self.threads.append(gevent.spawn(self.test, 2, thm))
while self.queue.empty():
print "queue is empty %s" % thm
gevent.sleep(0)
raise TaskComplete
def stop(self):
gevent.killall(self.threads)
class NotifyingQueue(Event):
""" A queue that follows the wait protocol. """
def __init__(self):
super(NotifyingQueue, self).__init__()
self._queue = Queue()
def put(self, item):
""" Add new item to the queue. """
self._queue.put(item)
self.set()
def empty(self):
return self._queue.empty()
def get(self, block=True, timeout=None):
""" Removes and returns an item from the queue. """
value = self._queue.get(block, timeout)
if self._queue.empty():
self.clear()
return value
def stop(self):
""" Request a stop event. """
self.set()
class ConnectionPool:
def __init__(self, db_config, time_to_sleep=30, test_run=False):
self.username = db_config.get('user')
self.password = db_config.get('password')
self.host = db_config.get('host')
self.port = int(db_config.get('port'))
self.max_pool_size = 20
self.test_run = test_run
self.pool = None
self.time_to_sleep = time_to_sleep
self._initialize_pool()
def get_initialized_connection_pool(self):
return self.pool
def _initialize_pool(self):
self.pool = Queue(maxsize=self.max_pool_size)
current_pool_size = self.pool.qsize()
if current_pool_size < self.max_pool_size: # this is a redundant check, can be removed
for _ in xrange(0, self.max_pool_size - current_pool_size):
try:
conn = db.connect(host=self.host,
user=self.username,
passwd=self.password,
port=self.port)
self.pool.put_nowait(conn)
except db.OperationalError, e:
LOGGER.error("Cannot initialize connection pool - retrying in {} seconds".format(self.time_to_sleep))
LOGGER.exception(e)
break
self._check_for_connection_loss()
class RemoteIterator(object):
def __init__(self):
self.queue = Queue()
def __iter__(self):
return self
def send(self, value):
if self.queue is None:
raise StopIteration
self.queue.put((True, value))
def throw(self, exc):
if self.queue is None:
raise StopIteration
self.queue.put((False, exc))
def close(self):
self.throw(StopIteration)
def next(self):
if self.queue is None:
raise StopIteration
yields, value = self.queue.get()
if yields:
return value
else:
self.queue = None
raise value
class BlackBerryPushService(object):
def __init__(self, app_id, password, push_url):
self.app_id = app_id
self.password = password
self.push_url = push_url
self._send_queue = Queue()
self._send_queue_cleared = Event()
self.log = logging.getLogger('pulsus.service.bbp')
def _send_loop(self):
self._send_greenlet = gevent.getcurrent()
try:
self.log.info("BlackBerry Push service started")
while True:
notification = self._send_queue.get()
try:
self._do_push(notification)
except Exception, e:
print e
self._send_queue.put(notification)
gevent.sleep(5.0)
finally:
if self._send_queue.qsize() < 1 and \
not self._send_queue_cleared.is_set():
self._send_queue_cleared.set()
class Actor(Greenlet): __metaclass__ = MetaActor def __init__(self): Greenlet.__init__(self) self.inbox = Queue() Actor.actors.append(self) def send(self, actor, message): actor.inbox.put(message) def receive(self, message): raise NotImplemented() @staticmethod def wait_actors(): gevent.joinall(Actor.actors) def loop(self): if not self.inbox.empty(): self.receive(self.inbox.get()) gevent.sleep() def _run(self): while self.started: self.loop()
def _run(self):
utils.log("[%s] parsing site %s" % (self, self.base))
queue = Queue()
pool = Pool(64)
seed = 'http://www.nytimes.com/best-sellers-books/'
pool.spawn(self._parseResultsPage, pool, queue, seed, 'current', True)
while True:
items = []
while not queue.empty():
item = queue.get_nowait()
items.append(item)
if 0 == len(items) and 0 == len(pool):
break
for item in items:
pool.spawn(item[0], pool, queue, item[1], item[2], item[3])
time.sleep(0.01)
pool.join()
self._output.put(StopIteration)
def transfer_queue():
"""Transfer event queue used by the confirmation sender."""
return Queue()
class CoroutineEngine(object):
"""协程加速控件"""
def __init__(self, core, power: int = None, **kwargs) -> None:
"""
:param core: 驱动核心
:param power: 协程功率
:param progress_name: 进度条
"""
# 初始化参数
self.max_queue_size = 0
self.power = power
# self.work_Q = Queue()
# 驱动器
self.core = core
# 额外参数
self.config_path = kwargs.get("config_path") # 配置文件
self.docker = kwargs.get("docker") # 业务列表
self.progress_name = kwargs.get('progress_name') # 进度条注释
self.silence = kwargs.get('silence') # selenium 静默启动
self.work_Q = Queue()
def load_tasks(self, tasks: list) -> None:
if isinstance(tasks, list):
for task in tasks:
self.work_Q.put_nowait(task)
def flexible_power(self):
"""
@todo 优化弹性协程算法
@return:
"""
import psutil
# 若未指定self.power 则使用弹性协程方案调度资源
if not self.power:
self.max_queue_size = self.work_Q.qsize()
# limit = round((psutil.cpu_count() / CRAWLER_SEQUENCE.__len__()))
limit = psutil.cpu_count()
self.power = limit if self.max_queue_size >= limit else self.max_queue_size
def launch(self, ) -> None:
while not self.work_Q.empty():
task = self.work_Q.get_nowait()
self.control_driver(task)
def control_driver(self, task) -> None:
"""
重写此模块
:param task:
:return:
"""
def progress_manager(self, total, desc='Example', leave=True, ncols=100, unit='B', unit_scale=True) -> None:
"""
进度监测
:return:
"""
from tqdm import tqdm
import time
# iterable: 可迭代的对象, 在手动更新时不需要进行设置
# desc: 字符串, 左边进度条描述文字
# page_size: 总的项目数
# leave: bool值, 迭代完成后是否保留进度条
# file: 输出指向位置, 默认是终端, 一般不需要设置
# ncols: 调整进度条宽度, 默认是根据环境自动调节长度, 如果设置为0, 就没有进度条, 只有输出的信息
# unit: 描述处理项目的文字
with tqdm(total=total, desc=desc, leave=leave, ncols=ncols,
unit=unit, unit_scale=unit_scale) as progress_bar:
progress_bar.update(self.power)
# while self.max_queue_size != Middleware.hera.qsize():
# progress_bar.update(Middleware.hera.qsize())
# time.sleep(1)
while not self.work_Q.empty():
now_1 = self.work_Q.qsize()
time.sleep(0.1)
now_2 = self.work_Q.qsize() - now_1
progress_bar.update(abs(now_2))
def run(self, speed_up=True, use_bar=False) -> None:
"""
协程任务接口
:return:
"""
task_list = []
if isinstance(self.docker, list):
# 刷新任务队列
self.load_tasks(tasks=self.docker)
else:
# 业务交接
self.work_Q = self.docker
# 弹性协程
if not speed_up:
self.power = 1
else:
self.flexible_power()
logger.info('Flexible Power:{}'.format(self.power))
logger.info('Queue Capacity:{}'.format(self.max_queue_size))
# 启动进度条
if use_bar:
import threading
threading.Thread(target=self.progress_manager,
args=(self.max_queue_size, self.progress_name + '[{}]'.format(self.power))).start()
for x in range(self.power):
task = gevent.spawn(self.launch)
task_list.append(task)
gevent.joinall(task_list)
def _queue():
# 队列
q = Queue()
q.put(1)
while not q.empty():
q.get()
def __init__(self):
self.queue = Queue()
def __init__(self, download_urls):
super(DownloadByGevent, self).__init__()
self.workQueue = Queue(1000)
self.download_urls = download_urls
def _init():
TaskResponseQueueManager._queue = Queue(0)
return True
from PIL import Image, ImageFile
from gevent.event import AsyncResult
from gevent.queue import Empty, Queue
from gevent.timeout import Timeout
import flask
from flask import send_from_directory
from app import app
DATA_DIR = 'app/src/data'
KEEP_ALIVE_DELAY = 25
MAX_IMAGE_SIZE = 800, 600
MAX_IMAGES = 10
MAX_DURATION = 300
BROADCAST_QUEUE = Queue()
try: # Reset saved files on each start
# rmtree(DATA_DIR, True)
os.mkdir(DATA_DIR)
except OSError:
pass
def broadcast(message):
"""Notify all waiting waiting gthreads of message."""
waiting = []
try:
while True:
waiting.append(BROADCAST_QUEUE.get(block=False))
except Empty:
class Worker():
def __init__(self, master_address, worker_address, debug):
self.id = None
self.master_address = master_address
if (worker_address is None):
self.worker_address = self.getMyAddress()
self.is_remote = False
else:
self.worker_address = worker_address
self.is_remote = True
self.all_task_list = {}
self.task_queue = Queue()
self.debug = debug
self.streaming_data = {}
self.streaming_meta_data = {}
self.event_queue = Queue()
self.task_node_table = {}
self.worker_list = None
def getMyAddress(self):
try:
csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
csock.connect(('8.8.8.8', 80))
(addr, port) = csock.getsockname()
csock.close()
return addr + ":" + port
except socket.error:
return "127.0.0.1"
def get_partition_infor(self, partition_infor, job_id, worker_list):
self.streaming_meta_data = partition_infor
self.worker_list = worker_list
if job_id not in self.streaming_data.keys():
self.streaming_data[job_id] = {}
for partition in partition_infor[self.id]:
if partition not in self.streaming_data[job_id].keys():
self.streaming_data[job_id][partition] = []
# debug_print_by_name('wentao', str(self.streaming_data))
def find_worker_in_metadata(self, partition_id, metadata):
worker = []
debug_print_by_name('wentao', str(metadata))
for worker_id, partition_list in metadata.items():
if partition_id in partition_list and worker_id != self.id:
worker.append(worker_id)
return worker
def replicate(self, job_id, partition, value, timestamp):
if job_id in self.streaming_data.keys():
self.streaming_data[job_id][partition].append({
'value':
value,
'timestamp':
timestamp
})
#print self.streaming_data
def get_streaming_message(self, message):
"""
Function to get and store streaming message.
:param value: spark streaming message
spark streaming message is "job_id,value"
"""
job_id, value = message.split(",")
job_id = int(job_id)
#print message
try:
if job_id in self.streaming_data.keys():
p_id = None
length = 9999999
partitions = self.streaming_data[job_id]
for partition_id, data in partitions.items():
if len(data) < length:
length = len(data)
p_id = partition_id
timestamp = int(time.time())
partitions[p_id].append({
'value': value,
'timestamp': timestamp
})
#todo replica
worker_list = self.find_worker_in_metadata(
p_id, self.streaming_meta_data)
debug_print_by_name('wentao', str(worker_list))
for worker_id in worker_list:
client = get_client(self.worker_list[worker_id]['address'],
1)
execute_command(client, client.replicate, job_id, p_id,
value, timestamp)
#print self.streaming_data
except Exception:
pass
#self.streaming_data = {}
def startRPCServer(self):
master = zerorpc.Server(self)
if self.is_remote:
addr = self.worker_address
else:
addr = "0.0.0.0:" + self.worker_address.split(":")[1]
# print "worker address is: %s at %s " % (addr, time.asctime(time.localtime(time.time())))
# addr = "tcp://0.0.0.0:"+port
master.bind('tcp://' + addr)
master.run()
def runPartition(self, task):
job_id = task.job_id
task_id = task.task_id
# create job if not exist
if not self.all_task_list.has_key(job_id):
task_list = {}
task_list[task_id] = {"status": Status.START, "data": None}
self.all_task_list[job_id] = task_list
else:
self.all_task_list[job_id][task_id] = {
"status": Status.START,
"data": None
}
debug_print(
"[Worker]Start task with job : %s task: %s at %s" %
(job_id, task_id, time.asctime(time.localtime(time.time()))),
self.debug)
# try:
debug_print_by_name('wentao', str(task.input_source))
result = task.last_rdd.get(task.input_source)
# debug_print("[Worker] Result of Task {0} is generated:{1}".format(task.task_id, result),self.debug)
self.all_task_list[job_id][task_id] = {
"status": Status.FINISH,
"data": result
}
debug_print(
"[Worker]Finish task with job : %s task: %s at %s" %
(job_id, task_id, time.asctime(time.localtime(time.time()))),
self.debug)
# except Exception:
# print Exception.
# debug_print("[Worker] Result of Task {0} is failed".format(task.task_id), self.debug)
# self.all_task_list[job_id][task_id] = {"status": Status.FAIL, "data": None}
def get_rdd_result(self, job_id, task_id, partition_id):
data = None
if self.all_task_list.has_key(
job_id) and self.all_task_list[job_id].has_key(task_id):
data = self.all_task_list[job_id][task_id]['data']
# if data is not None:
# debug_print(
# "[Worker]Get RDD result val {0} with job : {1} task: {2} partition: {3} at {4}".format(data, job_id, task_id, partition_id, time.asctime(time.localtime(time.time()))),
# self.debug)
#print "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%data={0}, partition_id={1} isDict={2}".format(data, partition_id, isinstance(data, dict))
if isinstance(data, dict):
if data.has_key(int(partition_id)):
return data[int(partition_id)]
else:
return []
return data
def register(self):
while self.id is None:
client = get_client(self.master_address)
self.id = execute_command(client, client.registerWorker,
self.worker_address)
# self.id = client.registerWorker(self.worker_address)
if self.id is not None:
debug_print(
"[Worker] worker %d %s registered at %s " %
(self.id, self.worker_address,
time.asctime(time.localtime(time.time()))), self.debug)
break
else:
gevent.sleep(2)
def filter_data(self, job_id, interval):
cur_time = time.time()
data = self.streaming_data[job_id]
for partition_id, partition_data in data.items():
#debug_print_by_name('kaijie', str(partition_data))
data[partition_id] = filter(
lambda a: cur_time - a['timestamp'] < interval, partition_data)
def check_if_streaming(self, task):
rdd_iter = task.last_rdd
while True:
if not isinstance(rdd_iter, NarrowRDD):
return isinstance(rdd_iter, Streaming)
rdd_iter = rdd_iter.parent
def start_task(self, serialized_task, task_node_table):
task = unpickle_object(serialized_task)
for source in task.input_source:
source['task_node_table'] = self.task_node_table
if self.check_if_streaming(task):
# source['streaming_data'] = self.streaming_data
# debug_print_by_name('wentao', str(self.streaming_data))
# debug_print_by_name('wentao', str(task.input_source[0]))
self.filter_data(task.input_source[0]['job_id'],
task.input_source[0]['interval'])
s_data = []
job_id = task.input_source[0]['job_id']
parition_id = task.input_source[0]['partition_id']
debug_print_by_name('wentao', str(self.streaming_data))
debug_print_by_name('wentao', str(job_id))
debug_print_by_name('wentao', str(parition_id))
for data in self.streaming_data[job_id][parition_id]:
s_data.append(data['value'])
source['streaming_data'] = s_data
debug_print("[Worker] Received Task {0}".format(task.task_id),
self.debug)
# event = {
# 'type' : 'Update',
# 'data' : task_node_table
# }
self.event_queue.put(task_node_table)
self.task_queue.put(task)
return 0
def task_manager(self):
while True:
while not self.task_queue.empty():
task = self.task_queue.get()
# debug_print("Create thread: %s at %s" % (0, time.asctime(time.localtime(time.time()))))
thread = gevent.spawn(self.runPartition, task)
debug_print(
"[Worker] Task created: Key: {0} at {1}".format(
task.task_id,
time.asctime(time.localtime(time.time()))), self.debug)
gevent.sleep(0)
def update_task_node_table(self, task_node_table):
try:
self.event_queue.put(task_node_table)
except:
return 1
return 0
def event_handler(self):
while True:
while not self.event_queue.empty():
task_node_table = self.event_queue.get()
#update task_node_table
self.task_node_table.update(task_node_table)
# for job_task_id, worker_info in task_node_table :
# self.task_node_table[job_task_id] = worker_info
gevent.sleep(0)
def heartbeat(self):
while True:
if self.id is not None:
# traverse task list and report processing tasks
task_status_list = {}
for job_id, task_list in self.all_task_list.items():
task_status_list[job_id] = {}
for task_id, value in task_list.items():
if value['status'] != Status.FINISH_REPORTED:
task_status_list[job_id][task_id] = value['status']
client = get_client(self.master_address)
debug_print(
"[Worker] Worker update task status: worker_id: %s at %s" %
(self.id, time.asctime(time.localtime(time.time()))),
self.debug)
print("task status list: ", str(task_status_list))
ret = execute_command(client, client.updateWorkerStatus,
self.id, task_status_list)
# ret = client.updateWorkerStatus(self.id, task_status_list)
if ret is not None:
# client.close()
if ret == 0:
# if already reported finish task, don't need to report finish again
for job_id, task_list in self.all_task_list.items():
for task_id, value in task_list.items():
if value['status'] == Status.FINISH:
value['status'] = Status.FINISH_REPORTED
debug_print_by_name('kaijie', str(self.all_task_list))
gevent.sleep(2)
def run(self):
self.register()
# self.startRPCServer()
thread1 = gevent.spawn(self.heartbeat)
thread2 = gevent.spawn(self.event_handler)
thread3 = gevent.spawn(self.task_manager)
thread4 = gevent.spawn(self.startRPCServer)
# self.startRPCServer()
gevent.joinall([thread1, thread2, thread3, thread4])
class SequentialGeventHandler(object):
"""Gevent handler for sequentially executing callbacks.
This handler executes callbacks in a sequential manner. A queue is
created for each of the callback events, so that each type of event
has its callback type run sequentially.
Each queue type has a greenlet worker that pulls the callback event
off the queue and runs it in the order the client sees it.
This split helps ensure that watch callbacks won't block session
re-establishment should the connection be lost during a Zookeeper
client call.
Watch callbacks should avoid blocking behavior as the next callback
of that type won't be run until it completes. If you need to block,
spawn a new greenlet and return immediately so callbacks can
proceed.
"""
name = "sequential_gevent_handler"
sleep_func = staticmethod(gevent.sleep)
def __init__(self):
"""Create a :class:`SequentialGeventHandler` instance"""
self.callback_queue = Queue()
self._running = False
self._async = None
self._state_change = Semaphore()
self._workers = []
class timeout_exception(gevent.event.Timeout):
def __init__(self, msg):
gevent.event.Timeout.__init__(self, exception=msg)
def _create_greenlet_worker(self, queue):
def greenlet_worker():
while True:
try:
func = queue.get()
if func is _STOP:
break
func()
except Empty:
continue
except Exception as exc:
log.warning("Exception in worker greenlet")
log.exception(exc)
return gevent.spawn(greenlet_worker)
def start(self):
"""Start the greenlet workers."""
with self._state_change:
if self._running:
return
self._running = True
# Spawn our worker greenlets, we have
# - A callback worker for watch events to be called
for queue in (self.callback_queue,):
w = self._create_greenlet_worker(queue)
self._workers.append(w)
atexit.register(self.stop)
def stop(self):
"""Stop the greenlet workers and empty all queues."""
with self._state_change:
if not self._running:
return
self._running = False
for queue in (self.callback_queue,):
queue.put(_STOP)
while self._workers:
worker = self._workers.pop()
worker.join()
# Clear the queues
self.callback_queue = Queue() # pragma: nocover
if hasattr(atexit, "unregister"):
atexit.unregister(self.stop)
def select(self, *args, **kwargs):
return gevent.select.select(*args, **kwargs)
def socket(self, *args, **kwargs):
return create_tcp_socket(socket)
def create_connection(self, *args, **kwargs):
return create_tcp_connection(socket, *args, **kwargs)
def event_object(self):
"""Create an appropriate Event object"""
return gevent.event.Event()
def lock_object(self):
"""Create an appropriate Lock object"""
return gevent.thread.allocate_lock()
def rlock_object(self):
"""Create an appropriate RLock object"""
return RLock()
def async_result(self):
"""Create a :class:`AsyncResult` instance
The :class:`AsyncResult` instance will have its completion
callbacks executed in the thread the
:class:`SequentialGeventHandler` is created in (which should be
the gevent/main thread).
"""
return AsyncResult()
def spawn(self, func, *args, **kwargs):
"""Spawn a function to run asynchronously"""
return gevent.spawn(func, *args, **kwargs)
def dispatch_callback(self, callback):
"""Dispatch to the callback object
The callback is put on separate queues to run depending on the
type as documented for the :class:`SequentialGeventHandler`.
"""
self.callback_queue.put(lambda: callback.func(*callback.args))
#导入gevent、time、requests
from gevent.queue import Queue
#从gevent库里导入queue模块
start = time.time()
url_list = ['https://www.baidu.com/',
'https://www.sina.com.cn/',
'http://www.sohu.com/',
'https://www.qq.com/',
'https://www.163.com/',
'http://www.iqiyi.com/',
'https://www.tmall.com/',
'http://www.ifeng.com/']
work = Queue()
#创建队列对象，并赋值给work。
for url in url_list:
#遍历url_list
work.put_nowait(url)
#用put_nowait()函数可以把网址都放进队列里。
def crawler():
while not work.empty():
#当队列不是空的时候，就执行下面的程序。
url = work.get_nowait()
#用get_nowait()函数可以把队列里的网址都取出。
r = requests.get(url)
#用requests.get()函数抓取网址。
print(url,work.qsize(),r.status_code)
#打印网址、队列长度、抓取请求的状态码。
'''
Created on 2015-10-23
@author: Shawn
'''
from gevent.monkey import patch_all
patch_all()
from socket import socket
from gevent.queue import Queue
import gevent.socket
import conf_server
queue = Queue()
queue.put('123')
queue.put('456')
print queue.peek()
print queue.peek()
import json
print json.loads('123')
# address = (conf_server.SERVER_IP, conf_server.SERVER_PORT)
# s = socket()
# s.connect(address)
#
def __init__(self, pool_size=5000):
self.job_pool = Pool(size=pool_size)
self.result = Queue()
self.target_queue = Queue()
def get(self, *args, **kw):
self.event.set()
return Queue.get(self, *args, **kw)
def test_imap_unordered_no_stop(self):
q = Queue()
q.put(1234)
gevent.spawn_later(0.1, q.put, StopIteration)
result = list(self.pool.imap_unordered(lambda _: _, q))
self.assertEqual(result, [1234])
def __init__(self, sock, addr):
Endpoint.__init__(self, sock, addr)
Greenlet.__init__(self)
self.ctlcmds = Queue(0)
self.userid = 0
self.init_gamedata_mixin()
# TODO:垂直挖掘STAFF机场
from gevent import monkey
monkey.patch_all()
import gevent
from gevent.queue import Queue
from spiderNest.preIntro import *
target_Q = Queue()
response_Q = Queue()
dataList = []
class sAirportSpider(object):
def __init__(self):
# 白嫖首页
self.airHome = 'https://52bp.org'
def get_vip_link(self):
return self.slaver(self.airHome + '/vip-airport.html', 'url')
def get_free_link(self):
return self.slaver(self.airHome + '/free-airport.html', 'url')
def get_all_link(self):
return self.slaver(self.airHome + '/airport.html', 'url')
@staticmethod
def get_sAirHome(mode):
def __init__(self):
super(NotifyingQueue, self).__init__()
self._queue = Queue()
def getUrl(oUrl,
encoding='utf-8',
headers=None,
data=None,
method=None,
allowCache=True,
usePool=True,
pool=pool_getUrl):
def _getUrl(result_queue, url_json, oUrl, encoding, headers, data, method,
allowCache, callmethod):
try:
if requests and session:
req = requests.Request(
method=method if method else "GET",
url=oUrl,
headers=headers if headers else fake_headers,
data=data)
prepped = req.prepare()
resp = session.send(prepped)
if encoding == "raw":
html_text = resp.content
else:
resp.encoding = encoding
html_text = resp.text
else:
# url 包含中文时 parse.quote_from_bytes(oUrl.encode('utf-8'), ':/&%?=+')
req = urllib.request.Request(
oUrl,
headers=headers if headers else {},
data=data,
method=method)
with urllib.request.urlopen(req) as response:
headers = response.info()
cType = headers.get('Content-Type', '')
match = re.search('charset\s*=\s*(\w+)', cType)
if match:
encoding = match.group(1)
blob = response.read()
if headers.get('Content-Encoding', '') == 'gzip':
data = gzip.decompress(blob)
else:
data = blob
if encoding == "raw":
html_text = data
else:
html_text = data.decode(encoding, 'ignore')
if allowCache:
urlcache[url_json] = html_text
result_queue.put(html_text)
return
except socket.timeout:
logging.warning(callmethod +
'request attempt %s timeout' % str(i + 1))
except urllib.error.URLError:
logging.warning(callmethod +
'request attempt %s URLError' % str(i + 1))
except http.client.RemoteDisconnected:
logging.warning(callmethod +
'request attempt %s RemoteDisconnected' %
str(i + 1))
except http.client.IncompleteRead:
logging.warning(callmethod +
'request attempt %s IncompleteRead' % str(i + 1))
except:
logging.exception(callmethod + "get url " + url_json + "fail")
result_queue.put(None)
return
callmethod = get_caller_info()
url_json = {
"oUrl": oUrl,
"encoding": encoding,
"headers": headers,
"data": data,
"method": method
}
url_json = json.dumps(url_json, sort_keys=True, ensure_ascii=False)
if allowCache:
if url_json in urlcache:
html_text = urlcache[url_json]
logging.debug(callmethod + "cache get:" + url_json)
return html_text
logging.debug(callmethod + "normal get:" + url_json)
else:
logging.debug(callmethod + "nocache get:" + url_json)
usePool = False
if requests and session:
retry_num = 1
else:
retry_num = 10
for i in range(retry_num):
queue = Queue(1)
if usePool:
pool.spawn(_getUrl, queue, url_json, oUrl, encoding, headers, data,
method, allowCache, callmethod)
else:
_getUrl(queue, url_json, oUrl, encoding, headers, data, method,
allowCache, callmethod)
result = queue.get()
if result is not None:
return result
return None
from gevent import monkey
monkey.patch_all()
import gevent,requests, bs4, csv
from gevent.queue import Queue
work = Queue()
food_list=[]
#前3个常见食物分类的前3页的食物记录的网址：
url_1 = 'http://www.boohee.com/food/group/{type}?page={page}'
for x in range(1, 4):
for y in range(1, 4):
real_url = url_1.format(type=x, page=y)
work.put_nowait(real_url)
#第11个常见食物分类的前3页的食物记录的网址：
url_2 = 'http://www.boohee.com/food/view_menu?page={page}'
for x in range(1,4):
real_url = url_2.format(page=x)
work.put_nowait(real_url)
#请写出crawler函数和启动协程的代码：
def crawler():
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
}
while not work.empty():
url = work.get_nowait()
res = requests.get(url, headers=headers)
bs_res = bs4.BeautifulSoup(res.text, 'html.parser')
foods = bs_res.find_all('li', class_='item clearfix')
temp_food = []
class VncIfmapClient(object):
# * Not all properties in an object needs to be published
# to IfMap.
# * In some properties, not all fields are relevant
# to be publised to IfMap.
# If the property is not relevant at all, define the property
# with None. If it is partially relevant, then define the fn.
# which would handcraft the generated xml for the object.
IFMAP_PUBLISH_SKIP_LIST = {
# Format - <prop_field> : None | <Handler_fn>
u"perms2": None,
u"id_perms": build_idperms_ifmap_obj
}
def handler(self, signum, frame):
file = open("/tmp/api-server-ifmap-cache.txt", "w")
file.write(pformat(self._id_to_metas))
file.close()
def __init__(self, db_client_mgr, ifmap_srv_ip, ifmap_srv_port, uname,
passwd, ssl_options):
self._ifmap_srv_ip = ifmap_srv_ip
self._ifmap_srv_port = ifmap_srv_port
self._username = uname
self._password = passwd
self._ssl_options = ssl_options
self._dequeue_greenlet = None
self._CONTRAIL_XSD = "http://www.contrailsystems.com/vnc_cfg.xsd"
self._IPERMS_NAME = "id-perms"
self._NAMESPACES = {
'env': "http://www.w3.org/2003/05/soap-envelope",
'ifmap': "http://www.trustedcomputinggroup.org/2010/IFMAP/2",
'meta':
"http://www.trustedcomputinggroup.org/2010/IFMAP-METADATA/2",
'contrail': self._CONTRAIL_XSD
}
self._db_client_mgr = db_client_mgr
self._sandesh = db_client_mgr._sandesh
ConnectionState.update(
conn_type=ConnType.IFMAP,
name='IfMap',
status=ConnectionStatus.INIT,
message='',
server_addrs=["%s:%s" % (ifmap_srv_ip, ifmap_srv_port)])
self._conn_state = ConnectionStatus.INIT
self._is_ifmap_up = False
self._queue = Queue(self._get_api_server()._args.ifmap_queue_size)
self.reset()
# Set the signal handler
signal.signal(signal.SIGUSR2, self.handler)
self._init_conn()
self._publish_config_root()
self._health_checker_greenlet =\
vnc_greenlets.VncGreenlet('VNC IfMap Health Checker',
self._health_checker)
# end __init__
@classmethod
def object_alloc(cls, obj_class, parent_res_type, fq_name):
res_type = obj_class.resource_type
my_fqn = ':'.join(fq_name)
parent_fqn = ':'.join(fq_name[:-1])
my_imid = 'contrail:%s:%s' % (res_type, my_fqn)
if parent_fqn:
if parent_res_type is None:
err_msg = "Parent: %s type is none for: %s" % (parent_fqn,
my_fqn)
return False, (409, err_msg)
parent_imid = 'contrail:' + parent_res_type + ':' + parent_fqn
else: # parent is config-root
parent_imid = 'contrail:config-root:root'
# Normalize/escape special chars
my_imid = escape(my_imid)
parent_imid = escape(parent_imid)
return True, (my_imid, parent_imid)
# end object_alloc
def object_set(self, obj_class, my_imid, existing_metas, obj_dict):
update = {}
# Properties Meta
for prop_field in obj_class.prop_fields:
field = obj_dict.get(prop_field)
if field is None:
continue
# construct object of xsd-type and get its xml repr
# e.g. virtual_network_properties
prop_field_types = obj_class.prop_field_types[prop_field]
is_simple = not prop_field_types['is_complex']
prop_type = prop_field_types['xsd_type']
# e.g. virtual-network-properties
prop_meta = obj_class.prop_field_metas[prop_field]
if prop_field in VncIfmapClient.IFMAP_PUBLISH_SKIP_LIST:
# Field not relevant, skip publishing to IfMap
if not VncIfmapClient.IFMAP_PUBLISH_SKIP_LIST[prop_field]:
continue
# Call the handler fn to generate the relevant fields.
if callable(
VncIfmapClient.IFMAP_PUBLISH_SKIP_LIST[prop_field]):
prop_xml = VncIfmapClient.IFMAP_PUBLISH_SKIP_LIST[
prop_field](prop_field, field)
meta = Metadata(prop_meta,
'', {'ifmap-cardinality': 'singleValue'},
ns_prefix='contrail',
elements=prop_xml)
else:
log_str = '%s is marked for partial publish\
to Ifmap but handler not defined' % (prop_field)
self.config_log(log_str, level=SandeshLevel.SYS_DEBUG)
continue
elif is_simple:
norm_str = escape(str(field))
meta = Metadata(prop_meta,
norm_str, {'ifmap-cardinality': 'singleValue'},
ns_prefix='contrail')
else: # complex type
prop_cls = str_to_class(prop_type, __name__)
buf = cStringIO.StringIO()
# perms might be inserted at server as obj.
# obj construction diff from dict construction.
if isinstance(field, dict):
prop_cls(**field).exportChildren(buf,
level=1,
name_=prop_meta,
pretty_print=False)
elif isinstance(field, list):
for elem in field:
if isinstance(elem, dict):
prop_cls(**elem).exportChildren(buf,
level=1,
name_=prop_meta,
pretty_print=False)
else:
elem.exportChildren(buf,
level=1,
name_=prop_meta,
pretty_print=False)
else: # object
field.exportChildren(buf,
level=1,
name_=prop_meta,
pretty_print=False)
prop_xml = buf.getvalue()
buf.close()
meta = Metadata(prop_meta,
'', {'ifmap-cardinality': 'singleValue'},
ns_prefix='contrail',
elements=prop_xml)
# If obj is new (existing metas is none) or
# if obj does not have this prop_meta (or)
# or if the prop_meta is different from what we have currently,
# then update
if (not existing_metas or not prop_meta in existing_metas
or ('' in existing_metas[prop_meta]
and str(meta) != str(existing_metas[prop_meta]['']))):
self._update_id_self_meta(update, meta)
# end for all property types
# References Meta
for ref_field in obj_class.ref_fields:
refs = obj_dict.get(ref_field)
if not refs:
continue
for ref in refs:
ref_fq_name = ref['to']
ref_fld_types_list = list(obj_class.ref_field_types[ref_field])
ref_res_type = ref_fld_types_list[0]
ref_link_type = ref_fld_types_list[1]
ref_meta = obj_class.ref_field_metas[ref_field]
ref_imid = get_ifmap_id_from_fq_name(ref_res_type, ref_fq_name)
ref_data = ref.get('attr')
if ref_data:
buf = cStringIO.StringIO()
attr_cls = str_to_class(ref_link_type, __name__)
attr_cls(**ref_data).exportChildren(buf,
level=1,
name_=ref_meta,
pretty_print=False)
ref_link_xml = buf.getvalue()
buf.close()
else:
ref_link_xml = ''
meta = Metadata(ref_meta,
'', {'ifmap-cardinality': 'singleValue'},
ns_prefix='contrail',
elements=ref_link_xml)
self._update_id_pair_meta(update, ref_imid, meta)
# end for all ref types
self._publish_update(my_imid, update)
return (True, '')
# end object_set
def object_create(self, obj_ids, obj_dict):
obj_type = obj_ids['type']
obj_class = self._db_client_mgr.get_resource_class(obj_type)
if not 'parent_type' in obj_dict:
# parent is config-root
parent_type = 'config-root'
parent_imid = 'contrail:config-root:root'
else:
parent_type = obj_dict['parent_type']
parent_imid = obj_ids.get('parent_imid', None)
# Parent Link Meta
update = {}
parent_cls = self._db_client_mgr.get_resource_class(parent_type)
parent_link_meta = parent_cls.children_field_metas.get('%ss' %
(obj_type))
if parent_link_meta:
meta = Metadata(parent_link_meta,
'', {'ifmap-cardinality': 'singleValue'},
ns_prefix='contrail')
self._update_id_pair_meta(update, obj_ids['imid'], meta)
self._publish_update(parent_imid, update)
(ok, result) = self.object_set(obj_class, obj_ids['imid'], None,
obj_dict)
return (ok, result)
# end object_create
def _object_read_to_meta_index(self, ifmap_id):
# metas is a dict where key is meta-name and val is list of dict of
# form [{'meta':meta}, {'id':id1, 'meta':meta}, {'id':id2, 'meta':meta}]
metas = {}
if ifmap_id in self._id_to_metas:
metas = self._id_to_metas[ifmap_id].copy()
return metas
# end _object_read_to_meta_index
def object_update(self, obj_cls, new_obj_dict):
ifmap_id = get_ifmap_id_from_fq_name(obj_cls.resource_type,
new_obj_dict['fq_name'])
# read in refs from ifmap to determine which ones become inactive after update
existing_metas = self._object_read_to_meta_index(ifmap_id)
if not existing_metas:
# UPDATE notify queued before CREATE notify, Skip publish to IFMAP.
return (True, '')
# remove properties that are no longer active
props = obj_cls.prop_field_metas
for prop, meta in props.items():
if meta in existing_metas and new_obj_dict.get(prop) is None:
self._delete_id_self_meta(ifmap_id, meta)
# remove refs that are no longer active
delete_list = []
refs = dict(
(obj_cls.ref_field_metas[rf], obj_cls.ref_field_types[rf][0])
for rf in obj_cls.ref_fields)
#refs = {'virtual-network-qos-forwarding-class': 'qos-forwarding-class',
# 'virtual-network-network-ipam': 'network-ipam',
# 'virtual-network-network-policy': 'network-policy',
# 'virtual-network-route-table': 'route-table'}
for meta, ref_res_type in refs.items():
old_set = set(existing_metas.get(meta, {}).keys())
new_set = set()
ref_obj_type = self._db_client_mgr.get_resource_class(
ref_res_type).object_type
for ref in new_obj_dict.get(ref_obj_type + '_refs', []):
to_imid = get_ifmap_id_from_fq_name(ref_res_type, ref['to'])
new_set.add(to_imid)
for inact_ref in old_set - new_set:
delete_list.append((inact_ref, meta))
if delete_list:
self._delete_id_pair_meta_list(ifmap_id, delete_list)
(ok, result) = self.object_set(obj_cls, ifmap_id, existing_metas,
new_obj_dict)
return (ok, result)
# end object_update
def object_delete(self, obj_ids):
ifmap_id = obj_ids['imid']
parent_imid = obj_ids.get('parent_imid')
existing_metas = self._object_read_to_meta_index(ifmap_id)
meta_list = []
for meta_name, meta_infos in existing_metas.items():
# Delete all refs/links in the object.
# Refs are identified when the key is a non-empty string.
meta_list.extend([(k, meta_name) for k in meta_infos if k != ''])
if parent_imid:
# Remove link from parent
meta_list.append((parent_imid, None))
if meta_list:
self._delete_id_pair_meta_list(ifmap_id, meta_list)
# Remove all property metadata associated with this ident
self._delete_id_self_meta(ifmap_id, None)
return (True, '')
# end object_delete
def _init_conn(self):
self._mapclient = client(
("%s" % (self._ifmap_srv_ip), "%s" % (self._ifmap_srv_port)),
self._username, self._password, self._NAMESPACES,
self._ssl_options)
connected = False
while not connected:
try:
resp_xml = self._mapclient.call('newSession',
NewSessionRequest())
except socket.error as e:
msg = 'Failed to establish IF-MAP connection: %s' % str(e)
self.config_log(msg, level=SandeshLevel.SYS_WARN)
time.sleep(3)
continue
resp_doc = etree.parse(StringIO.StringIO(resp_xml))
err_codes = resp_doc.xpath(
'/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode',
namespaces=self._NAMESPACES)
if not err_codes:
connected = True
else:
msg = "Failed to establish IF-MAP connection: %s" % err_codes
self.config_log(msg, level=SandeshLevel.SYS_WARN)
session_id = self._mapclient.get_session_id()
try:
self._mapclient.call('endSession',
EndSessionRequest(session_id))
except socket.error as e:
msg = "Failed to end the IF-MAP session %s: %s" %\
(session_id, str(e))
self.config_log(msg, level=SandeshLevel.SYS_WARN)
time.sleep(3)
ConnectionState.update(conn_type=ConnType.IFMAP,
name='IfMap',
status=ConnectionStatus.UP,
message='',
server_addrs=[
"%s:%s" %
(self._ifmap_srv_ip, self._ifmap_srv_port)
])
self._conn_state = ConnectionStatus.UP
msg = 'IFMAP connection ESTABLISHED'
self.config_log(msg, level=SandeshLevel.SYS_NOTICE)
self._mapclient.set_session_id(
newSessionResult(resp_xml).get_session_id())
self._mapclient.set_publisher_id(
newSessionResult(resp_xml).get_publisher_id())
# end _init_conn
def _get_api_server(self):
return self._db_client_mgr._api_svr_mgr
# end _get_api_server
def reset(self):
self._id_to_metas = {}
while not self._queue.empty():
self._queue.get_nowait()
if (self._dequeue_greenlet is not None
and gevent.getcurrent() != self._dequeue_greenlet):
self._dequeue_greenlet.kill()
self._dequeue_greenlet =\
vnc_greenlets.VncGreenlet("VNC IfMap Dequeue",
self._ifmap_dequeue_task)
# end reset
def _publish_config_root(self):
# Remove all resident data
result = ifmap_wipe(self._mapclient)
if result is None:
msg = "Cannot purge the IF-MAP server before publishing root graph"
self.config_log(msg, level=SandeshLevel.SYS_WARN)
# Build default config-root
buf = cStringIO.StringIO()
perms = Provision.defaults.perms
perms.exportChildren(buf, level=1, pretty_print=False)
id_perms_xml = buf.getvalue()
buf.close()
update = {}
meta = Metadata(self._IPERMS_NAME,
'', {'ifmap-cardinality': 'singleValue'},
ns_prefix='contrail',
elements=id_perms_xml)
self._update_id_self_meta(update, meta)
self._publish_update("contrail:config-root:root", update)
# end _publish_config_root
def config_log(self, msg, level):
self._db_client_mgr.config_log(msg, level)
# end config_log
@ignore_exceptions
def _generate_ifmap_trace(self, oper, body):
req_id = get_trace_id()
ifmap_trace = IfmapTrace(request_id=req_id)
ifmap_trace.operation = oper
ifmap_trace.body = body
return ifmap_trace
# end _generate_ifmap_trace
def _publish_to_ifmap_enqueue(self, oper, oper_body, do_trace=True):
# safety check, if we proceed ifmap-server reports error
# asking for update|delete in publish
if not oper_body:
return
self._queue.put((oper, oper_body, do_trace))
# end _publish_to_ifmap_enqueue
def _ifmap_dequeue_task(self):
while True:
try:
self._publish_to_ifmap_dequeue()
except Exception as e:
tb = detailed_traceback()
self.config_log(tb, level=SandeshLevel.SYS_ERR)
def _publish_to_ifmap_dequeue(self):
def _publish(requests, traces, publish_discovery=False):
if not requests:
return
ok = False
# Continue to trying publish requests until the queue is full.
# When queue is full, ifmap is totally resync from db
while not ok:
ok, err_msg = self._publish_to_ifmap(''.join(requests))
if ok:
trace_msg(traces, 'IfmapTraceBuf', self._sandesh)
else:
trace_msg(traces,
'IfmapTraceBuf',
self._sandesh,
error_msg=err_msg)
if publish_discovery and ok:
self._get_api_server().publish_ifmap_to_discovery()
self._is_ifmap_up = True
if not ok:
msg = ("%s. IF-MAP sending queue size: %d/%d" %
(err_msg, self._queue.qsize(),
self._get_api_server()._args.ifmap_queue_size))
self.config_log(msg, level=SandeshLevel.SYS_WARN)
gevent.sleep(1)
# end _publish
while True:
# block until there is data in the queue
(oper, oper_body, do_trace) = self._queue.get()
requests = []
requests_len = 0
traces = []
while True:
# drain the queue till empty or max message size
# or change of oper because ifmap does not like
# different operations in same message
if oper == 'publish_discovery':
_publish(requests, traces, True)
break
if do_trace:
trace = self._generate_ifmap_trace(oper, oper_body)
traces.append(trace)
requests.append(oper_body)
requests_len += len(oper_body)
if (requests_len >
self._get_api_server()._args.ifmap_max_message_size):
_publish(requests, traces)
break
old_oper = oper
try:
(oper, oper_body, do_trace) = self._queue.get_nowait()
if oper != old_oper:
_publish(requests, traces)
requests = []
requests_len = 0
traces = []
continue
except Empty:
_publish(requests, traces)
break
# end _publish_to_ifmap_dequeue
def _publish_to_ifmap(self, oper_body):
try:
not_published = True
retry_count = 0
resp_xml = None
while not_published:
sess_id = self._mapclient.get_session_id()
req_xml = PublishRequest(sess_id, oper_body)
resp_xml = self._mapclient.call('publish', req_xml)
resp_doc = etree.parse(StringIO.StringIO(resp_xml))
err_codes = resp_doc.xpath(
'/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode',
namespaces=self._NAMESPACES)
if err_codes:
if retry_count == 0:
log_str = 'Error publishing to ifmap, req: %s, resp: %s' \
%(req_xml, resp_xml)
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
ConnectionState.update(
conn_type=ConnType.IFMAP,
name='IfMap',
status=ConnectionStatus.INIT,
message='Session lost, renew it',
server_addrs=[
"%s:%s" %
(self._ifmap_srv_ip, self._ifmap_srv_port)
])
self._conn_state = ConnectionStatus.INIT
self._is_ifmap_up = False
retry_count = retry_count + 1
self._init_conn()
if self._ifmap_restarted():
msg = "IF-MAP servers restarted, re-populate it"
self.config_log(msg, level=SandeshLevel.SYS_ERR)
self.reset()
self._get_api_server().publish_ifmap_to_discovery(
'down', msg)
self._publish_config_root()
self._db_client_mgr.db_resync()
self._publish_to_ifmap_enqueue('publish_discovery', 1)
else: # successful publish
not_published = False
break
# end while not_published
if retry_count:
log_str = 'Success publishing to ifmap after %d tries' \
%(retry_count)
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
return True, resp_xml
except Exception as e:
# Failed to publish the operation due to unknown error.
# Probably a connection issue with the ifmap server.
msg = "Failed to publish request %s: %s" % (oper_body, str(e))
return False, msg
# end _publish_to_ifmap
def _build_request(self, id1_name, id2_name, meta_list, delete=False):
request = ''
id1 = unicode(
Identity(name=id1_name, type="other", other_type="extended"))
if id2_name != 'self':
id2 = unicode(
Identity(name=id2_name, type="other", other_type="extended"))
else:
id2 = None
for m in meta_list:
if delete:
filter = unicode(m) if m else None
op = PublishDeleteOperation(id1=id1, id2=id2, filter=filter)
else:
op = PublishUpdateOperation(id1=id1,
id2=id2,
metadata=unicode(m),
lifetime='forever')
request += unicode(op)
return request
def _delete_id_self_meta(self, self_imid, meta_name):
contrail_metaname = 'contrail:' + meta_name if meta_name else None
del_str = self._build_request(self_imid, 'self', [contrail_metaname],
True)
self._publish_to_ifmap_enqueue('delete', del_str)
try:
# del meta from cache and del id if this was last meta
if meta_name:
del self._id_to_metas[self_imid][meta_name]
if not self._id_to_metas[self_imid]:
del self._id_to_metas[self_imid]
else:
del self._id_to_metas[self_imid]
except KeyError:
# Case of delete received for an id which we do not know about.
# Could be a case of duplicate delete.
# There is nothing for us to do here. Just log and proceed.
msg = "Delete received for unknown imid(%s) meta_name(%s)." % \
(self_imid, meta_name)
self.config_log(msg, level=SandeshLevel.SYS_DEBUG)
# end _delete_id_self_meta
def _delete_id_pair_meta_list(self, id1, meta_list):
del_str = ''
for id2, metadata in meta_list:
contrail_metadata = 'contrail:' + metadata if metadata else None
del_str += self._build_request(id1, id2, [contrail_metadata], True)
self._publish_to_ifmap_enqueue('delete', del_str)
# del meta,id2 from cache and del id if this was last meta
def _id_to_metas_delete(id1, id2, meta_name):
if id1 not in self._id_to_metas:
return
if meta_name not in self._id_to_metas[id1]:
return
if not self._id_to_metas[id1][meta_name]:
del self._id_to_metas[id1][meta_name]
if not self._id_to_metas[id1]:
del self._id_to_metas[id1]
return
# if meta is prop, noop
if id2 in self._id_to_metas[id1][meta_name]:
del self._id_to_metas[id1][meta_name][id2]
#end _id_to_metas_delete
for id2, metadata in meta_list:
if metadata:
# replace with remaining refs
_id_to_metas_delete(id1, id2, metadata)
_id_to_metas_delete(id2, id1, metadata)
else: # no meta specified remove all links from id1 to id2
for meta_name in self._id_to_metas.get(id1, {}).keys():
_id_to_metas_delete(id1, id2, meta_name)
for meta_name in self._id_to_metas.get(id2, {}).keys():
_id_to_metas_delete(id2, id1, meta_name)
# end _delete_id_pair_meta_list
def _update_id_self_meta(self, update, meta):
""" update: dictionary of the type
update[<id> | 'self'] = list(metadata)
"""
mlist = update.setdefault('self', [])
mlist.append(meta)
# end _update_id_self_meta
def _update_id_pair_meta(self, update, to_id, meta):
mlist = update.setdefault(to_id, [])
mlist.append(meta)
# end _update_id_pair_meta
def _publish_update(self, self_imid, update):
requests = []
self_metas = self._id_to_metas.setdefault(self_imid, {})
for id2, metalist in update.items():
request = self._build_request(self_imid, id2, metalist)
# remember what we wrote for diffing during next update
old_metalist = []
for m in metalist:
meta_name = m._Metadata__name[9:]
# Objects have two types of members - Props and refs/links.
# Props are cached in id_to_metas as
# id_to_metas[self_imid][meta_name]['']
# (with empty string as key)
# Links are cached in id_to_metas as
# id_to_metas[self_imid][meta_name][id2]
# id2 is used as a key
if id2 == 'self':
self_metas[meta_name] = {'': m}
continue
if meta_name in self_metas:
old_metalist.append(self_metas[meta_name])
# Update the link/ref
self_metas[meta_name][id2] = m
else:
# Create a new link/ref
self_metas[meta_name] = {id2: m}
# Reverse linking from id2 to id1
self._id_to_metas.setdefault(id2, {})
if meta_name in self._id_to_metas[id2]:
self._id_to_metas[id2][meta_name][self_imid] = m
else:
self._id_to_metas[id2][meta_name] = {self_imid: m}
old_request = self._build_request(self_imid, id2, old_metalist)
if request != old_request:
requests.append(request)
upd_str = ''.join(requests)
self._publish_to_ifmap_enqueue('update', upd_str)
# end _publish_update
def _ifmap_restarted(self):
return not entity_is_present(self._mapclient, 'config-root', ['root'])
def _health_checker(self):
while True:
try:
# do the healthcheck only if we are connected
if self._conn_state == ConnectionStatus.DOWN:
continue
meta = Metadata('display-name',
'', {'ifmap-cardinality': 'singleValue'},
ns_prefix='contrail',
elements='')
request_str = self._build_request('healthcheck', 'self',
[meta])
self._publish_to_ifmap_enqueue('update',
request_str,
do_trace=False)
# Confirm the existence of the following default global entities in IFMAP.
search_list = [
('global-system-config', ['default-global-system-config']),
]
for type, fq_name in search_list:
if not entity_is_present(self._mapclient, type, fq_name):
raise Exception("%s not found in IFMAP DB" %
':'.join(fq_name))
# If we had unpublished the IFMAP server to discovery server earlier
# publish it back now since it is valid now.
if not self._is_ifmap_up:
self._get_api_server().publish_ifmap_to_discovery('up', '')
self._is_ifmap_up = True
ConnectionState.update(
conn_type=ConnType.IFMAP,
name='IfMap',
status=ConnectionStatus.UP,
message='',
server_addrs=[
"%s:%s" %
(self._ifmap_srv_ip, self._ifmap_srv_port)
])
except Exception as e:
log_str = 'IFMAP Healthcheck failed: %s' % (str(e))
self.config_log(log_str, level=SandeshLevel.SYS_ERR)
if self._is_ifmap_up:
self._get_api_server().publish_ifmap_to_discovery(
'down', 'IFMAP DB - Invalid state')
self._is_ifmap_up = False
ConnectionState.update(
conn_type=ConnType.IFMAP,
name='IfMap',
status=ConnectionStatus.DOWN,
message='Invalid IFMAP DB State',
server_addrs=[
"%s:%s" %
(self._ifmap_srv_ip, self._ifmap_srv_port)
])
finally:
gevent.sleep(
self._get_api_server().get_ifmap_health_check_interval())
class MuxSocketTransportSink(ClientMessageSink):
"""A transport sink for tmux servers.
This sink supports concurrent requests over its transport.
"""
SINK_LOG = ROOT_LOG.getChild('SocketTransportSink')
_EMPTY_DCT = {}
_CLOSE_INVOKED = "Close invoked"
class Varz(VarzBase):
"""
messages_sent - The number of messages sent over this sink.
messages_recv - The number of messages received over this sink.
active - 1 if the sink is open, else 0.
send_queue_size - The length of the send queue.
send_time - The aggregate amount of time spent sending data.
recv_time - The aggregate amount of time spend receiving data.
send_latency - The average amount of time taken to send a message.
recv_latency - The average amount of time taken to receive a message
(once a response has reached the client).
transport_latency - The average amount of time taken to perform a full
method call transaction (send data, wait for response,
read response).
"""
_VARZ_BASE_NAME = 'scales.thriftmux.SocketTransportSink'
_VARZ = {
'messages_sent': Rate,
'messages_recv': Rate,
'active': Gauge,
'send_queue_size': Gauge,
'send_time': AggregateTimer,
'recv_time': AggregateTimer,
'send_latency': AverageTimer,
'recv_latency': AverageTimer,
'transport_latency': AverageTimer
}
def __init__(self, socket, service):
super(MuxSocketTransportSink, self).__init__()
self._socket = socket
self._state = ChannelState.Idle
self._log = self.SINK_LOG.getChild('[%s.%s:%d]' % (
service, self._socket.host, self._socket.port))
self._socket_source = '%s:%d' % (self._socket.host, self._socket.port)
self._service = service
self._open_result = None
self._varz = self.Varz(Source(service=self._service,
endpoint=self._socket_source))
def _Init(self):
self._tag_map = {}
self._open_result = None
self._tag_pool = TagPool((2 ** 24) - 1, self._service, self._socket_source)
self._greenlets = []
self._send_queue = Queue()
@property
def isActive(self):
return self._state != ChannelState.Closed
@property
def state(self):
return self._state
def Open(self):
"""Initializes the dispatcher, opening a connection to the remote host.
This method may only be called once.
"""
if not self._open_result:
self._Init()
self._open_result = AsyncResult()
self._open_result.SafeLink(self._OpenImpl)
return self._open_result
def _SpawnNamedGreenlet(self, name, *args, **kwargs):
return NamedGreenlet.spawn(
'Scales %s for %s [%s]' % (name, self._service, self._socket_source),
*args,
**kwargs)
def _OpenImpl(self):
try:
self._log.debug('Opening transport.')
self._socket.open()
self._greenlets.append(self._SpawnNamedGreenlet('Recv Loop', self._RecvLoop))
self._greenlets.append(self._SpawnNamedGreenlet('Send Loop', self._SendLoop))
self._CheckInitialConnection()
self._log.debug('Open successful')
self._state = ChannelState.Open
self._varz.active(1)
except Exception as e:
self._log.error('Exception opening socket')
self._open_result.set_exception(e)
self._Shutdown('Open failed')
raise
@abstractmethod
def _CheckInitialConnection(self):
raise NotImplementedError()
def Close(self):
self._Shutdown(self._CLOSE_INVOKED, False)
def _Shutdown(self, reason, fault=True):
if not self.isActive:
return
self._state = ChannelState.Closed
if reason == self._CLOSE_INVOKED:
log_fn = self._log.debug
else:
log_fn = self._log.warning
log_fn('Shutting down transport [%s].' % str(reason))
self._varz.active(0)
self._socket.close()
[g.kill(block=False) for g in self._greenlets]
self._greenlets = []
if not isinstance(reason, Exception):
reason = Exception(str(reason))
if fault:
self.on_faulted.Set(reason)
msg = MethodReturnMessage(error=ClientError(reason))
for sink_stack, _, _ in self._tag_map.values():
sink_stack.AsyncProcessResponseMessage(msg)
if (self._open_result
and not self._open_result.ready()
and isinstance(reason, Exception)):
self._open_result.set_exception(reason)
self._tag_map = {}
self._open_result = None
self._send_queue = Queue()
@abstractmethod
def _OnTimeout(self, tag):
raise NotImplementedError()
def _HandleTimeout(self, msg_properties):
"""Determine if a message has timed out yet (because it waited in the queue
for too long). If it hasn't, initialize the timeout handler to fire if the
message times out in transit.
Args:
msg_properties - The properties of the message.
"""
timeout_event = msg_properties.get(Deadline.EVENT_KEY, None)
if timeout_event and timeout_event.Get():
# The message has timed out before it got out of the send queue
# In this case, we can discard it immediately without even sending it.
tag = msg_properties.pop(Tag.KEY, 0)
if tag != 0:
self._ReleaseTag(tag)
return True
elif timeout_event:
# The event exists but hasn't been signaled yet, hook up a
# callback so we can be notified on a timeout.
def timeout_proc():
timeout_tag = msg_properties.pop(Tag.KEY, 0)
if timeout_tag:
self._OnTimeout(timeout_tag)
timeout_event.Subscribe(lambda evt: timeout_proc(), True)
return False
else:
# No event was created, so this will never timeout.
return False
def _SendLoop(self):
"""Dispatch messages from the send queue to the remote server.
Note: Messages in the queue have already been serialized into wire format.
"""
while self.isActive:
try:
payload, dct = self._send_queue.get()
queue_len = self._send_queue.qsize()
self._varz.send_queue_size(queue_len)
# HandleTimeout sets up the transport level timeout handling
# for this message. If the message times out in transit, this
# transport will handle sending a Tdiscarded to the server.
if self._HandleTimeout(dct): continue
with self._varz.send_time.Measure():
with self._varz.send_latency.Measure():
self._socket.write(payload)
self._varz.messages_sent()
except Exception as e:
self._Shutdown(e)
break
def _RecvLoop(self):
"""Dispatch messages from the remote server to their recipient.
Note: Deserialization and dispatch occurs on a seperate greenlet, this only
reads the message off the wire.
"""
while self.isActive:
try:
sz, = unpack('!i', self._socket.readAll(4))
with self._varz.recv_time.Measure():
with self._varz.recv_latency.Measure():
buf = BytesIO(self._socket.readAll(sz))
self._varz.messages_recv()
gevent.spawn(self._ProcessReply, buf)
except Exception as e:
self._Shutdown(e)
break
@abstractmethod
def _ProcessReply(self, stream):
raise NotImplementedError()
def _ProcessTaggedReply(self, tag, stream):
tup = self._ReleaseTag(tag)
if tup:
reply_stack, start_time, props = tup
props[Tag.KEY] = None
self._varz.transport_latency(time.time() - start_time)
stream.seek(0)
reply_stack.AsyncProcessResponseStream(stream)
def _ReleaseTag(self, tag):
"""Return a tag to the tag pool.
Note: Tags are only returned when the server has ACK'd them (or NACK'd) with
and Rdispatch message (or similar). Client initiated timeouts do NOT return
tags to the pool.
Args:
tag - The tag to return.
Returns:
The ClientChannelSinkStack associated with the tag's response.
"""
tup = self._tag_map.pop(tag, None)
self._tag_pool.release(tag)
return tup
@abstractmethod
def _BuildHeader(self, tag, msg_type, data_len):
raise NotImplementedError()
def AsyncProcessRequest(self, sink_stack, msg, stream, headers):
if self._state == ChannelState.Idle and self._open_result:
self._log.debug('Waiting for channel to be open')
self._open_result.wait()
if ((self._state == ChannelState.Idle and not self._open_result)
or self._state == ChannelState.Closed) and sink_stack is not None:
err_msg = MethodReturnMessage(error=Exception('Sink not open.'))
sink_stack.AsyncProcessResponseMessage(err_msg)
return
if not msg.is_one_way:
tag = self._tag_pool.get()
msg.properties[Tag.KEY] = tag
self._tag_map[tag] = (sink_stack, time.time(), msg.properties)
else:
tag = 0
data_len = stream.tell()
header = self._BuildHeader(tag, headers[TransportHeaders.MessageType], data_len)
payload = header + stream.getvalue()
self._send_queue.put((payload, msg.properties))
def AsyncProcessResponse(self, sink_stack, context, stream, msg):
pass
class gwhatweb(object):
def __init__(self, url, webdata):
self.tasks = Queue()
self.url = url.rstrip("/")
self.cmsdict = {}
self.cmsname = None
for i in webdata:
self.tasks.put(i)
print("webdata total:%d" % len(webdata))
def _GetMd5(self, body):
m2 = hashlib.md5()
m2.update(body)
return m2.hexdigest()
def _clearQueue(self):
while not self.tasks.empty():
self.tasks.get()
def _worker(self):
data = self.tasks.get()
test_url = "{0}{1}".format(self.url, data["url"])
req = None
try:
req = requests.get(test_url, timeout=10)
except:
rtext = ''
if not req:
return False
result = checkcms(req, data)
if result:
if result > 100:
logger.info('web is {0} finger: {1}'.format(
data['name'], data['url']))
return data['name']
if data['name'] not in self.cmsdict:
logger.info('web look like {0}'.format(data['name']))
self.cmsdict[data['name']] = data['weight']
logger.info('cms weight:{}'.format(self.cmsdict[data['name']]))
else:
self.cmsdict[data['name']] += data['weight']
logger.info('cms weight:{}'.format(self.cmsdict[data['name']]))
if self.cmsdict[data['name']] > 100:
logger.info('web is {0} finger: {1}'.format(
data['name'], data['url']))
return data['name']
return False
def _boss(self):
while not self.tasks.empty():
flag = self._worker()
if flag:
self.cmsname = flag
self._clearQueue()
def whatweb(self, maxsize=5):
allr = [gevent.spawn(self._boss) for i in range(maxsize)]
gevent.joinall(allr)
return self.cmsname
def __init__(self, config):
super(CompetitiveDialogueDataBridge, self).__init__()
self.config = config
self.tenders_sync_client = TendersClientSync(
'',
host_url=self.config_get('public_tenders_api_server') or self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.client = TendersClientSync(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
def_queue_size = 500
def_watcher_delay = 15
self.initial_sync_point = {}
self.initialization_event = gevent.event.Event()
self.competitive_dialogues_queue = Queue(maxsize=def_queue_size) # Id tender which need to check
self.handicap_competitive_dialogues_queue = Queue(maxsize=def_queue_size)
self.dialogs_stage2_put_queue = Queue(maxsize=def_queue_size) # queue with new tender data
self.dialogs_stage2_retry_put_queue = Queue(maxsize=def_queue_size)
self.dialog_stage2_id_queue = Queue(maxsize=def_queue_size)
self.dialog_retry_stage2_id_queue = Queue(maxsize=def_queue_size)
self.dialogs_stage2_patch_queue = Queue(maxsize=def_queue_size)
self.dialogs_stage2_retry_patch_queue = Queue(maxsize=def_queue_size)
self.dialog_set_complete_queue = Queue(maxsize=def_queue_size)
self.dialog_retry_set_complete_queue = Queue(maxsize=def_queue_size)
self.jobs_watcher_delay = self.config_get('jobs_watcher_delay') or def_watcher_delay
def _Init(self):
self._tag_map = {}
self._open_result = None
self._tag_pool = TagPool((2 ** 24) - 1, self._service, self._socket_source)
self._greenlets = []
self._send_queue = Queue()
#!/usr/bin/env python
# encoding: utf-8
# SystemError: (libev) select: Unknown error
import socket
import gevent
from gevent.queue import Queue, Empty
tasks = Queue(maxsize=3)
def worker(name):
try:
while True:
task = tasks.get(timeout=1)
print(('Worker %s got task %s' % (name, task)))
gevent.sleep(0)
except Empty:
print(('Quitting time of %s!' % name))
def boss():
for i in range(1, 11):
tasks.put(i)
print('Assigned all work in iteration 1')
for i in range(11, 21):
tasks.put(i)
print('Assigned all work in iteration 2')
gevent.joinall([
class PAService(TradeService):
def __init__(self, name):
super(PAService, self).__init__(name)
self.active = False # 工作状态
self.queue = Queue() # 队列
self.thread = Thread(target=self._processData) # 线程
self.logger = instance.getLogger()
self.symbols = set() # 已经订阅的合约
# self.latest_ticks = {}
self.contracts = {ProductClass.Future: {}, ProductClass.Stock: {}}
self.generate_bars = []
self.kline_symbols = [] # 实时计算k线的合约名称
self.all_calc_mins = {}
def init(self, cfgs, **kwargs):
self.service_id = cfgs.get('id')
self.service_type = ServiceType.DataPAServer
super(PAService, self).init(cfgs)
# SymbolBarManager().init(cfgs.get('generate_bars'))
self.generate_bars = map(str.strip,
cfgs.get('generate_bars', '').split(','))
kline.mongodb_conn = instance.datasourceManager.get('mongodb').conn
self.contracts[ProductClass.Future] = get_all_contracts(
type_=ProductClass.Future)
symbols = self.cfgs.get('kline_symbols', '').split(',')
symbols = map(str.strip, symbols)
self.kline_symbols = symbols
def syncDownServiceConfig(self):
TradeService.syncDownServiceConfig(self)
def setupFanoutAndLogHandler(self):
from mantis.trade.log import TradeServiceLogHandler
self.initFanoutSwitchers(self.cfgs.get('fanout'))
handler = TradeServiceLogHandler(self)
self.logger.addHandler(handler)
def start(self, block=False):
self.setupFanoutAndLogHandler()
# 创建日志引擎
super(PAService, self).start()
self.active = True
# self.thread.start()
self.prepare_kline_calc_minutes()
self.registerTimedTask(self.make_kline, timeout=1)
def stop(self):
super(PAService, self).stop()
self.active = False
def join(self):
# self.thread.join()
pass
def onXminBar(self, scale, bar):
"""
:param scale:
:param bar: (VtBarData)
:return:
"""
symbol = bar.vtSymbol
self.logger.debug('onXminBar: {} - {}'.format(symbol, scale))
hashobj = copy.copy(bar.__dict__)
## obj.__dict__ 的使用方法，特别注意： 后续修改 hashobj将改变 bar对象的对应的属性值
hashobj['datetime'] = ''
hashobj['scale'] = scale
hashobj.has_key('_id')
del hashobj['_id']
jsondata = json.dumps(hashobj)
self.dataFanout('switch0', jsondata, symbol=symbol, scale=scale)
def onTick(self, symbol, tick):
"""
:param symbol:
:param tick: (VtTickData)
:return:
"""
# self.queue.put((symbol,tick))
# self.latest_ticks[symbol] = tick
pass
def _processData(self):
self.active = True
while self.active:
try:
data = self.queue.get(block=True, timeout=1)
if not data:
continue
symbol, tick = data
SymbolBarManager().onTick(symbol, tick)
except Exception as e:
# self.logger.error( str(e) )
# traceback.print_exc()
pass
def _make_kline(self, timer):
""" 定时执行所有合约的k线生成
生成k线bar发布到redis待写入mongodb
实时生成指定合约的k线数据，合约定义: ' kline_symbols'
"""
# symbols = self.contracts[ProductClass.Future].keys()
for symbol in self.kline_symbols:
# prefix = get_symbol_prefix(symbol)
# 定时生成 合约的1分钟 k线
bar = kline.make_lastest_min_bar(symbol)
if bar:
self.onXminBar('1m', bar)
for scale in self.generate_bars:
bar = kline.make_latest_nmin_bar(symbol, scale)
if bar:
self.onXminBar(scale, bar)
timer.start()
def prepare_kline_calc_minutes(self):
"""当天载入当日计算时间分钟点和跨日分钟点
凌晨2：40停止程序运行
"""
now = DateTime.now()
if now.time() < Time(3, 0): # 如果在凌晨启动的话，去前一天的计算分钟点
now = now - TimeDelta(days=1)
for symbol in self.kline_symbols:
self.all_calc_mins[symbol] = {}
for k in (1, 5, 15, 30, 60):
mins = kline.get_day_trade_calc_minutes_new(symbol, k, now)
self.all_calc_mins[symbol][k] = mins
def make_kline(self, timer):
""" 定时执行所有合约的k线生成
生成k线bar发布到redis待写入mongodb
实时生成指定合约的k线数据，合约定义: ' kline_symbols'
"""
# symbols = self.contracts[ProductClass.Future].keys()
for symbol in self.kline_symbols:
# for scale in self.generate_bars:
for k in (1, 5, 15, 30, 60):
scale = '{}m'.format(k)
calc_mins = self.all_calc_mins.get(symbol).get(k)
# 定时生成 合约的1分钟 k线
bar = kline.make_lastest_min_bar(symbol, scale, calc_mins)
if bar:
self.onXminBar(scale, bar) # 通知接收用户
timer.start()
def queue(self):
return Queue()
class CompetitiveDialogueDataBridge(object):
""" Competitive Dialogue Data Bridge """
copy_name_fields = (
'title', 'title_ru', 'title_en', 'description', 'description_en', 'description_ru',
'mode', 'procurementMethodDetails', 'submissionMethodDetails',
'minimalStep', 'value',
'procuringEntity', 'buyers',
)
rewrite_statuses = ['draft']
allowed_statuses = ['active.tendering', 'active.pre-qualification', 'active.pre-qualification.stand-still',
'active.auction', 'active.qualification', 'active.awarded', 'complete', 'cancelled',
'unsuccessful', STAGE2_STATUS]
def __init__(self, config):
super(CompetitiveDialogueDataBridge, self).__init__()
self.config = config
self.tenders_sync_client = TendersClientSync(
'',
host_url=self.config_get('public_tenders_api_server') or self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.client = TendersClientSync(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
def_queue_size = 500
def_watcher_delay = 15
self.initial_sync_point = {}
self.initialization_event = gevent.event.Event()
self.competitive_dialogues_queue = Queue(maxsize=def_queue_size) # Id tender which need to check
self.handicap_competitive_dialogues_queue = Queue(maxsize=def_queue_size)
self.dialogs_stage2_put_queue = Queue(maxsize=def_queue_size) # queue with new tender data
self.dialogs_stage2_retry_put_queue = Queue(maxsize=def_queue_size)
self.dialog_stage2_id_queue = Queue(maxsize=def_queue_size)
self.dialog_retry_stage2_id_queue = Queue(maxsize=def_queue_size)
self.dialogs_stage2_patch_queue = Queue(maxsize=def_queue_size)
self.dialogs_stage2_retry_patch_queue = Queue(maxsize=def_queue_size)
self.dialog_set_complete_queue = Queue(maxsize=def_queue_size)
self.dialog_retry_set_complete_queue = Queue(maxsize=def_queue_size)
self.jobs_watcher_delay = self.config_get('jobs_watcher_delay') or def_watcher_delay
def config_get(self, name):
return self.config.get('main').get(name)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def get_tender_credentials(self, tender_id):
self.client.headers.update({'X-Client-Request-ID': generate_req_id()})
logger.info("Getting credentials for tender {}".format(tender_id),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_CREDENTIALS},
{"TENDER_ID": tender_id}))
data = self.client.extract_credentials(tender_id)
logger.info("Got tender {} credentials".format(tender_id),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_GOT_CREDENTIALS},
{"TENDER_ID": tender_id}))
return data
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def initialize_sync(self, params=None, direction=None):
# TODO use gevent.Event to wake up forward sync instead of checking
# initial sync point
if direction == "backward":
assert params['descending']
response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
# set values in reverse order due to 'descending' option
self.initial_sync_point = {'forward_offset': response.prev_page.offset,
'backward_offset': response.next_page.offset}
self.initialization_event.set()
logger.info("Initial sync point {}".format(self.initial_sync_point))
return response
elif not self.initial_sync_point:
raise ValueError
else:
assert 'descending' not in params
gevent.wait([self.initialization_event])
self.initialization_event.clear()
params['offset'] = self.initial_sync_point['forward_offset']
logger.info("Starting forward sync from offset {}".format(params['offset']))
return self.tenders_sync_client.sync_tenders(params,
extra_headers={'X-Client-Request-ID': generate_req_id()})
def get_tenders(self, params, direction=""):
response = self.initialize_sync(params=params, direction=direction)
while not (params.get('descending') and not len(response.data) and params.get('offset') == response.next_page.offset):
tenders_list = response.data
params['offset'] = response.next_page.offset
delay = 101
if tenders_list:
delay = 15
logger.info("Client {} params: {}".format(direction, params))
for tender in tenders_list:
# Check, if we already work with this tender
if tender['id'] in dialog_work:
logger.info('WORK with tender {}'.format(tender['id']))
continue
if tender['procurementMethodType'] in [CD_UA_TYPE, CD_EU_TYPE] and tender['status'] == 'active.stage2.waiting':
logger.info('{0} sync: Found competitive dialogue (stage1), id={1} with status {2}'.format(direction.capitalize(), tender['id'], tender['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_NOLOT},
{"TENDER_ID": tender['id']}))
yield tender
else:
logger.debug('{0} sync: Skipping tender {1} in status {2} with procurementMethodType {3}'.format(direction.capitalize(), tender['id'], tender['status'], tender['procurementMethodType']),
extra=journal_context(params={"TENDER_ID": tender['id']}))
logger.info('Sleep {} sync...'.format(direction),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP}))
gevent.sleep(delay)
logger.info('Restore {} sync'.format(direction),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_RESUME}))
logger.debug('{} {}'.format(direction, params))
response = self.tenders_sync_client.sync_tenders(params,
extra_headers={'X-Client-Request-ID': generate_req_id()})
def get_competitive_dialogue_data(self):
while True:
try:
tender_to_sync = self.competitive_dialogues_queue.peek() # Get competitive dialogue which we want to sync
tender = self.tenders_sync_client.get_tender(tender_to_sync['id'])['data'] # Try get data by tender id
except Exception as e:
# If we have something problems then put tender back to queue
logger.exception(e)
logger.info('Putting tender {} back to tenders queue...'.format(tender_to_sync['id']),
extra=journal_context(params={"TENDER_ID": tender_to_sync['id']}))
self.competitive_dialogues_queue.get() # Remove erroneous item from the queue, to put it as the last item
# What if another thread put one more item after get but before put?
self.competitive_dialogues_queue.put(tender_to_sync)
logger.info('Tender {} put back to tenders queue. Que size: {}'.format(
tender_to_sync['id'], self.competitive_dialogues_queue.qsize()))
else:
if 'stage2TenderID' in tender:
try:
tender_stage2 = self.tenders_sync_client.get_tender(tender['stage2TenderID'])['data']
except:
logger.info('Tender stage 2 id={0} didn\'t exist, need create new'.format(tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_STAGE2_NOT_EXIST},
{"TENDER_ID": tender['id']}))
else:
if tender_stage2.get('status') in self.allowed_statuses:
if tender.get('status') == 'complete':
logger.warn('Dialog {0} already has complete status - silently removing from initial queue.'.format(tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_ONLY_PATCH},
{"TENDER_ID": tender['id']}))
self.competitive_dialogues_queue.get() # Remove from the queue
else:
logger.info('For dialog {0} tender stage 2 already exists, need only patch'.format(tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_ONLY_PATCH},
{"TENDER_ID": tender['id']}))
patch_data = {"id": tender['id'],
"status": "complete"}
self.competitive_dialogues_queue.get() # Remove from the queue
self.dialog_set_complete_queue.put(patch_data)
continue
elif tender_stage2.get('status') in self.rewrite_statuses:
logger.info('Tender stage 2 id={0} has bad status need to create new '.format(tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_CREATE_NEW_STAGE2},
{"TENDER_ID": tender['id']}))
logger.info('Copy competitive dialogue data, id={} '.format(tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_COPY_TENDER_ITEMS},
{"TENDER_ID": tender['id']}))
new_tender = dict(procurementMethod='selective', status='draft', dialogueID=tender['id'])
for field_name in self.copy_name_fields: # Copy fields from stage 1 competitive dialog
if field_name in tender:
new_tender[field_name] = tender[field_name]
if tender['procurementMethodType'].endswith('EU'):
new_tender['procurementMethodType'] = STAGE_2_EU_TYPE
else:
new_tender['procurementMethodType'] = STAGE_2_UA_TYPE
new_tender['tenderID'] = '{}.2'.format(tender['tenderID']) # set tenderID as in stage1 + '.2'
old_lots, items, short_listed_firms = dict(), list(), dict()
for qualification in tender['qualifications']:
if qualification['status'] == 'active': # check if qualification has status active
if qualification.get('lotID'):
if qualification['lotID'] not in old_lots: # check if lot id in local dict with new lots
lot = prepare_lot(tender, qualification['lotID'], items) # update lot with new id
if not lot: # Go next iter if not lot
continue
old_lots[qualification['lotID']] = lot # set new lot in local dict
bid = get_bid_by_id(tender['bids'], qualification['bidID'])
for bid_tender in bid['tenderers']:
if bid_tender['identifier']['id'] not in short_listed_firms:
short_listed_firms[bid_tender['identifier']['id']] = {"name": bid_tender['name'],
"identifier": bid_tender['identifier'],
"lots": [{"id": old_lots[qualification['lotID']]['id']}]}
else:
short_listed_firms[bid_tender['identifier']['id']]['lots'].append(
{"id": old_lots[qualification['lotID']]['id']})
else:
new_tender['items'] = copy.deepcopy(tender['items']) # add all items, with new id
bid = get_bid_by_id(tender['bids'], qualification['bidID'])
for bid_tender in bid['tenderers']:
if bid_tender['identifier']['id'] not in short_listed_firms:
short_listed_firms[bid_tender['identifier']['id']] = {"name": bid_tender['name'],
"identifier": bid_tender['identifier'],
"lots": []}
if items: # If we have lots, then add only related items
new_tender['items'] = items
new_tender['lots'] = old_lots.values()
if 'features' in tender:
new_tender['features'] = []
for feature in tender.get('features'):
if feature['featureOf'] == 'tenderer': # If feature related to tender, than just copy
new_tender['features'].append(feature)
elif feature['featureOf'] == 'item': # If feature related to item need check if it's actual
if feature['relatedItem'] in (item['id'] for item in new_tender['items']):
new_tender['features'].append(feature)
elif feature['featureOf'] == 'lot': # If feature related to lot need check if it's actual
if feature['relatedItem'] in old_lots.keys():
new_tender['features'].append(feature)
new_tender['shortlistedFirms'] = short_listed_firms.values()
self.competitive_dialogues_queue.get()
self.handicap_competitive_dialogues_queue.put(new_tender)
def prepare_new_tender_data(self):
while True:
new_tender = self.handicap_competitive_dialogues_queue.get()
try:
logger.info("Getting extra info for competitive dialogue, id={0}".format(new_tender['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_EXTRA_INFO},
{"TENDER_ID": new_tender['dialogueID']}))
tender_data = self.get_tender_credentials(new_tender['dialogueID'])
except Exception as e:
logger.exception(e)
logger.info("Can't get competitive dialogue credentials, id={0}".format(new_tender['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_MISSING_CREDENTIALS},
{"TENDER_ID": new_tender['dialogueID']}))
self.handicap_competitive_dialogues_queue.put(new_tender)
else:
logger.debug("Got extra info for competitive dialogue, id={}".format(new_tender['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_GOT_EXTRA_INFO},
{"TENDER_ID": new_tender['dialogueID']}))
data = tender_data.data
new_tender['owner'] = data['owner']
new_tender['dialogue_token'] = data['tender_token']
self.dialogs_stage2_put_queue.put(new_tender)
gevent.sleep(0)
def put_tender_stage2(self):
"""
Create tender for stage 2
"""
while True:
new_tender = self.dialogs_stage2_put_queue.get()
logger.info("Creating tender stage2 from competitive dialogue id={}".format(new_tender['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_CREATE_NEW_TENDER},
{"TENDER_ID": new_tender['dialogueID']}))
data = {"data": new_tender}
try:
res = self.client.create_tender(data)
except ResourceError as re:
if re.status_int == 412: # Update Cookie, and retry
self.client.headers['Cookie'] = re.response.headers['Set-Cookie']
elif re.status_int == 422: # WARNING and don't retry
logger.warn("Catch 422 status, stop create tender stage2",
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_CREATE},
{"TENDER_ID": new_tender['dialogueID']}))
logger.warn("Error response {}".format(re.message),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_CREATE},
{"TENDER_ID": new_tender['dialogueID']}))
continue
elif re.status_int == 404: # WARNING and don't retry
logger.warn("Catch 404 status, stop create tender stage2",
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_CREATE},
{"TENDER_ID": new_tender['dialogueID']}))
continue
logger.info("Unsuccessful put for tender stage2 of competitive dialogue id={0}".format(new_tender['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_CREATE},
{"TENDER_ID": new_tender['dialogueID']}))
logger.info("Schedule retry for competitive dialogue id={0}".format(new_tender['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_RETRY_CREATE},
{"TENDER_ID": new_tender['dialogueID']}))
self.dialogs_stage2_retry_put_queue.put(new_tender)
except Exception as e:
logger.info("Exception, schedule retry for competitive dialogue id={0}".format(new_tender['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_RETRY_CREATE},
{"TENDER_ID": new_tender['dialogueID']}))
self.dialogs_stage2_retry_put_queue.put(new_tender)
logger.exception(e)
else:
logger.info("Successfully created tender stage2 id={} from competitive dialogue id={}".format(res['data']['id'], res['data']['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_CREATED},
{"DIALOGUE_ID": res['data']['dialogueID'],
"TENDER_ID": res['data']['id']}))
# Put data in queue for patch dialog
dialog = {"id": res['data']['dialogueID'],
"stage2TenderID": res['data']['id']}
self.dialog_stage2_id_queue.put(dialog)
gevent.sleep(0)
def patch_dialog_add_stage2_id(self):
"""
Patch origin competitive dialogue - set tender id for stage 2 (field stage2TenderID)
"""
while True:
dialog = self.dialog_stage2_id_queue.get()
dialog_work.add(dialog['id'])
logger.info("Patch competitive dialogue id={} with stage2 tender id".format(dialog['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_CD_PATCH_STAGE2_ID},
{"TENDER_ID": dialog['id']}))
patch_data = {"data": dialog}
try:
res_patch = self.client.patch_tender(patch_data)
except Exception as e:
logger.exception(e)
logger.info("Unsuccessful patch competitive dialogue id={0} with stage2 tender id".format(dialog['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_CD_UNSUCCESSFUL_PATCH_STAGE2_ID},
{"TENDER_ID": dialog['id']}))
logger.info("Schedule retry for competitive dialogue id={0}".format(dialog['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_CD_RETRY_PATCH_STAGE2_ID},
{"TENDER_ID": dialog['id']}))
self.dialog_retry_stage2_id_queue.put(dialog)
else:
logger.info("Successful patch competitive dialogue id={0} with stage2 tender id".format(res_patch['data']['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_CD_PATCHED_STAGE2_ID},
{"DIALOGUE_ID": res_patch['data']['id'],
"TENDER_ID": res_patch['data']['stage2TenderID']}))
data = {"id": dialog['stage2TenderID'],
"status": STAGE2_STATUS,
"dialogueID": dialog['id']}
self.dialogs_stage2_patch_queue.put(data)
gevent.sleep(0)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def _patch_dialog_add_stage2_id_with_retry(self, dialog):
try:
data = {"data": dialog}
logger.info("Patch competitive dialogue id={0}".format(dialog['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_PATCH_DIALOG},
{"TENDER_ID": dialog['id']}))
self.client.patch_tender(data)
except Exception as e:
logger.exception(e)
raise
def retry_patch_dialog_add_stage2_id(self):
while True:
try:
dialog = self.dialog_retry_stage2_id_queue.get()
self._patch_dialog_add_stage2_id_with_retry(dialog)
except:
logger.warn("Can't patch competitive dialogue id={0}".format(dialog['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_CD_UNSUCCESSFUL_PATCH_STAGE2_ID},
{"TENDER_ID": dialog['id']}))
self.competitive_dialogues_queue.put({"id": dialog['id']})
else:
data = {"id": dialog['stage2TenderID'],
"status": STAGE2_STATUS,
"dialogueID": dialog['id']}
self.dialogs_stage2_patch_queue.put(data)
gevent.sleep(0)
def patch_new_tender_status(self):
while True:
patch_data = self.dialogs_stage2_patch_queue.get()
logger.info("Patch tender stage2 id={0} with status {1}".format(patch_data['id'], patch_data['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_PATCH_NEW_TENDER_STATUS},
{"TENDER_ID": patch_data["id"]}))
try:
res = self.client.patch_tender({"data": patch_data})
except Exception as e:
logger.exception(e)
logger.info("Unsuccessful path tender stage2 id={0} with status {1}".format(patch_data['id'], patch_data['status']))
logger.info("Schedule retry patch for tender stage2 {0}".format(patch_data['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_PATCH_NEW_TENDER_STATUS},
{"TENDER_ID": patch_data['id']}))
self.dialogs_stage2_retry_patch_queue.put(patch_data)
else:
logger.info("Successful patch tender stage2 id={0} with status {1}".format(patch_data['id'], patch_data['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESSFUL_PATCH_NEW_TENDER_STATUS}))
patch_data = {"id": res['data']['dialogueID'],
"status": "complete"}
self.dialog_set_complete_queue.put(patch_data)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def _patch_new_tender_status_with_retry(self, new_tender):
try:
data = {"data": new_tender}
logger.info("Patch new tender stage2 id={0} status".format(new_tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_PATCH_NEW_TENDER_STATUS},
{"TENDER_ID": new_tender['id']}))
self.client.patch_tender(data)
except Exception as e:
logger.exception(e)
raise
def path_dialog_status(self):
while True:
patch_data = self.dialog_set_complete_queue.get()
logger.info("Patch competitive dialogue id={0} with status {1}".format(patch_data['id'], patch_data['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_PATCH_DIALOG_STATUS},
{"TENDER_ID": patch_data["id"]}))
try:
self.client.patch_tender({"data": patch_data})
except Exception as e:
logger.exception(e)
logger.info("Unsuccessful path competitive dialogue id={0} with status {1}".format(patch_data['id'], patch_data['status']))
logger.info("Schedule retry patch for competitive dialogue id={0}".format(patch_data['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_PATCH_DIALOG_STATUS},
{"TENDER_ID": patch_data['id']}))
self.dialog_retry_set_complete_queue.put(patch_data)
else:
logger.info("Successful patch competitive dialogue id={0} with status {1}".format(patch_data['id'], patch_data['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESSFUL_PATCH_DIALOG_STATUS}))
try:
dialog_work.remove(patch_data['id'])
except KeyError:
pass
def retry_patch_dialog_status(self):
while True:
patch_data = self.dialog_retry_set_complete_queue.get()
try:
self._patch_dialog_status_with_retry(patch_data)
except:
logger.warn("Can't patch competitive dialogue id={0} with status {1}".format(patch_data['id'], patch_data['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_PATCH_DIALOG_STATUS,
"TENDER_ID": patch_data['id']}))
self.competitive_dialogues_queue.put({"id": patch_data['id']})
gevent.sleep(0)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def _patch_dialog_status_with_retry(self, patch_data):
try:
data = {"data": patch_data}
logger.info("Patch competitive dialogue id={0} with status {1}".format(patch_data['id'], patch_data['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_PATCH_DIALOG_STATUS},
{"TENDER_ID": patch_data['id']}))
self.client.patch_tender(data)
except Exception as e:
logger.exception(e)
raise
def retry_patch_new_tender_status(self):
while True:
patch_data = self.dialogs_stage2_retry_patch_queue.get()
try:
self._patch_new_tender_status_with_retry(patch_data)
except:
logger.warn("Can't patch tender stage2 id={0} with status {1}".format(patch_data['id'], patch_data['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_PATCH_NEW_TENDER_STATUS,
"TENDER_ID": patch_data['id']}))
self.competitive_dialogues_queue.put({"id": patch_data['dialogueID']})
gevent.sleep(0)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=10000)
def _put_with_retry(self, new_tender):
data = {"data": new_tender}
logger.info("Creating tender stage2 from competitive dialogue id={0}".format(new_tender['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_CREATE_NEW_TENDER},
{"TENDER_ID": new_tender['dialogueID']}))
try:
res = self.client.create_tender(data)
except ResourceError as re:
if re.status_int == 412: # Update Cookie, and retry
self.client.headers['Cookie'] = re.response.headers['Set-Cookie']
elif re.status_int == 422: # WARNING and don't retry
logger.warn("Catch 422 status, stop create tender stage2",
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_CREATE},
{"TENDER_ID": new_tender['dialogueID']}))
logger.warn("Error response {}".format(re.message),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_CREATE},
{"TENDER_ID": new_tender['dialogueID']}))
elif re.status_int == 404: # WARNING and don't retry
logger.warn("Catch 404 status, stop create tender stage2",
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_CREATE},
{"TENDER_ID": new_tender['dialogueID']}))
else:
logger.info("Unsuccessful put for tender stage2 of competitive dialogue id={0}".format(new_tender['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESSFUL_CREATE},
{"TENDER_ID": new_tender['dialogueID']}))
raise re
except Exception as e:
logger.exception(e)
raise
else:
# Put data in queue for patch dialog
dialog = {"id": res['data']['dialogueID'],
"stage2TenderID": res['data']['id']}
self.dialog_stage2_id_queue.put(dialog)
def retry_put_tender_stage2(self):
while True:
try:
new_tender = self.dialogs_stage2_retry_put_queue.get()
self._put_with_retry(new_tender)
except:
del new_tender['dialogue_token'] # do not reveal tender credentials in logs
logger.warn("Can't create tender stage2 from competitive dialogue id={0}".format(new_tender['dialogueID']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_CREATE_ERROR,
"TENDER_ID": new_tender['dialogueID']}))
self.competitive_dialogues_queue.put({"id": new_tender['dialogueID']})
else:
dialog = {"id": new_tender['dialogueID'],
"stage2TenderID": new_tender['id']}
self.dialog_stage2_id_queue.put(dialog)
gevent.sleep(0)
def get_competitive_dialogue_forward(self):
logger.info('Start forward data sync worker...')
params = {'opt_fields': 'status,procurementMethodType', 'mode': '_all_'}
try:
for tender_data in self.get_tenders(params=params, direction="forward"):
logger.info('Forward sync: Put competitive dialogue id={} to process...'.format(tender_data['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS},
{"TENDER_ID": tender_data['id']}))
self.competitive_dialogues_queue.put(tender_data)
except ResourceError as re:
if re.status_int == 412:
self.tenders_sync_client.headers['Cookie'] = re.response.headers['Set-Cookie']
logger.warn('Forward catch 412, update coockie and restart worker',extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {}))
logger.warn('Forward worker died!', extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {}))
logger.error("Error response {}".format(re.message))
raise re
except Exception as e:
# TODO reset queues and restart sync
logger.warn('Forward worker died!', extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {}))
logger.exception(e)
raise e
else:
logger.warn('Forward data sync finished!') # Should never happen!!!
def get_competitive_dialogue_backward(self):
logger.info('Start backward data sync worker...')
params = {'opt_fields': 'status,procurementMethodType', 'descending': 1, 'mode': '_all_'}
try:
for tender_data in self.get_tenders(params=params, direction="backward"):
logger.info('Backward sync: Put competitive dialogue id={} to process...'.format(tender_data['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS},
{"TENDER_ID": tender_data['id']}))
self.competitive_dialogues_queue.put(tender_data)
except ResourceError as re:
if re.status_int == 412:
self.tenders_sync_client.headers['Cookie'] = re.response.headers['Set-Cookie']
logger.warn('Backward catch 412, update coockie and restart worker', extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {}))
logger.warn('Backward worker died!', extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {}))
logger.error("Error response {}".format(re.message))
raise re
except Exception as e:
# TODO reset queues and restart sync
logger.warn('Backward worker died!', extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {}))
logger.exception(e)
raise e
else:
logger.info('Backward data sync finished.')
def catch_exception(self, exc, name):
"""Restarting job"""
logger.warning('Worker died! Restarting {}.'.format(name), extra=journal_context({"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {}))
if name == 'get_competitive_dialogue_data':
tender = self.competitive_dialogues_queue.get() # delete invalid tender from queue
logger.info('Remove invalid tender {}'.format(tender.id))
self.immortal_jobs[name] = gevent.spawn(getattr(self, name))
self.immortal_jobs[name].link_exception(partial(self.catch_exception, name=name))
def _start_competitive_sculptors(self):
logger.info('Start Competitive Dialogue Data Bridge')
self.immortal_jobs = {
'get_competitive_dialogue_data': gevent.spawn(self.get_competitive_dialogue_data),
'prepare_new_tender_data': gevent.spawn(self.prepare_new_tender_data),
'put_tender_stage2': gevent.spawn(self.put_tender_stage2),
'retry_put_tender_stage2': gevent.spawn(self.retry_put_tender_stage2),
'patch_dialog_add_stage2_id': gevent.spawn(self.patch_dialog_add_stage2_id),
'retry_patch_dialog_add_stage2_id': gevent.spawn(self.retry_patch_dialog_add_stage2_id),
'patch_new_tender_status': gevent.spawn(self.patch_new_tender_status),
'retry_patch_new_tender_status': gevent.spawn(self.retry_patch_new_tender_status),
'path_dialog_status': gevent.spawn(self.path_dialog_status),
'retry_patch_dialog_status': gevent.spawn(self.retry_patch_dialog_status)
}
for name, job in self.immortal_jobs.items():
job.link_exception(partial(self.catch_exception, name=name))
def _start_competitive_wokers(self):
self.jobs = [
gevent.spawn(self.get_competitive_dialogue_backward),
gevent.spawn(self.get_competitive_dialogue_forward),
]
def _restart_synchronization_workers(self):
logger.warn("Restarting synchronization", extra=journal_context({"MESSAGE_ID": DATABRIDGE_RESTART}, {}))
for j in self.jobs:
j.kill(timeout=5)
self._start_competitive_wokers()
def run(self):
self._start_competitive_sculptors()
self._start_competitive_wokers()
backward_worker, forward_worker = self.jobs
counter = 0
try:
while True:
gevent.sleep(self.jobs_watcher_delay)
if counter == 20:
logger.info(
"""Current state:
First stages in processing {competitive_dialogues_queue};
Prepared data for second stage {handicap_competitive_dialogues_queue};
Prepared data with owner and token {dialogs_stage2_put_queue};
Retry prepared data with owner and token {dialogs_stage2_retry_put_queue};
Data with second stage ID {dialog_stage2_id_queue};
Retry data with second stage ID {dialog_retry_stage2_id_queue};
Data with new status and first stage ID {dialogs_stage2_patch_queue}
Retry data with new status and first stage ID {dialogs_stage2_retry_patch_queue}
Data with new status for first stage {dialog_set_complete_queue}
Retry data with new status for first stage {dialog_retry_set_complete_queue}""".format(
competitive_dialogues_queue=self.competitive_dialogues_queue.qsize(),
handicap_competitive_dialogues_queue=self.handicap_competitive_dialogues_queue.qsize(),
dialogs_stage2_put_queue=self.dialogs_stage2_put_queue.qsize(),
dialogs_stage2_retry_put_queue=self.dialogs_stage2_retry_put_queue.qsize(),
dialog_stage2_id_queue=self.dialog_stage2_id_queue.qsize(),
dialog_retry_stage2_id_queue=self.dialog_retry_stage2_id_queue.qsize(),
dialogs_stage2_patch_queue=self.dialogs_stage2_patch_queue.qsize(),
dialogs_stage2_retry_patch_queue=self.dialogs_stage2_retry_patch_queue.qsize(),
dialog_set_complete_queue=self.dialog_set_complete_queue.qsize(),
dialog_retry_set_complete_queue=self.dialog_retry_set_complete_queue.qsize()),
extra={
'competitive_dialogues_queue': self.competitive_dialogues_queue.qsize(),
'handicap_competitive_dialogues_queue': self.handicap_competitive_dialogues_queue.qsize(),
'dialogs_stage2_put_queue': self.dialogs_stage2_put_queue.qsize(),
'dialogs_stage2_retry_put_queue': self.dialogs_stage2_retry_put_queue.qsize(),
'dialog_stage2_id_queue': self.dialog_stage2_id_queue.qsize(),
'dialog_retry_stage2_id_queue': self.dialog_retry_stage2_id_queue.qsize(),
'dialogs_stage2_patch_queue': self.dialogs_stage2_patch_queue.qsize(),
'dialogs_stage2_retry_patch_queue': self.dialogs_stage2_retry_patch_queue.qsize(),
'dialog_set_complete_queue': self.dialog_set_complete_queue.qsize(),
'dialog_retry_set_complete_queue': self.dialog_retry_set_complete_queue.qsize()})
counter = 0
counter += 1
if forward_worker.dead or (backward_worker.dead and not backward_worker.successful()):
self._restart_synchronization_workers()
backward_worker, forward_worker = self.jobs
except KeyboardInterrupt:
logger.info('Exiting...')
gevent.killall(self.jobs, timeout=5)
gevent.killall(self.immortal_jobs, timeout=5)
except Exception as e:
logger.exception(e)
logger.warn("Restarting synchronization", extra=journal_context({"MESSAGE_ID": DATABRIDGE_RESTART}))
#防止ssl未校验时出现提示信息
requests.packages.urllib3.disable_warnings()
#dict_mode的payloads
payloads.dict_mode_dict = set()
#crawl_mode的payloads
payloads.crawl_mode_dynamic_fuzz_temp_dict = set()
payloads.similar_urls_set = set()
payloads.crawl_mode_dynamic_fuzz_dict = list()
#blast_mode的payload
payloads.blast_mode_custom_charset_dict = list()
#fuzz_mode的payload
payloads.fuzz_mode_dict = list()
#创建all_tasks队列
tasks.all_task = Queue()
tasks.task_length = 0
tasks.task_count = 0
#假性404页面md5列表
conf.autodiscriminator_md5 = set()
bar.log = progressbar.ProgressBar()
def saveResults(domain, msg):
'''
@description: 结果保存，以"域名.txt"命名，url去重复
@param {domain:域名,msg:保存的信息}
@return: null
'''
import gevent
from gevent.queue import Queue
tasks = Queue()
def worker(user):
while not tasks.empty():
task = tasks.get()
print('%s got task %s' % (user, task))
gevent.sleep(0)
print('Quitting worker!')
def boss():
for i in xrange(4):
tasks.put_nowait(i)
gevent.spawn(boss).join()
gevent.joinall([
gevent.spawn(worker, 'steve'),
gevent.spawn(worker, 'john'),
])
