def run_worker(context):
poller = Poller()
liveness = HEARTBEAT_LIVENESS
interval = INTERVAL_INIT
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
worker = yield worker_socket(context, poller)
cycles = 0
while True:
socks = yield poller.poll(HEARTBEAT_INTERVAL * 1000)
socks = dict(socks)
# Handle worker activity on backend
if socks.get(worker) == zmq.POLLIN:
# Get message
# - 3-part envelope + content -> request
# - 1-part HEARTBEAT -> heartbeat
frames = yield worker.recv_multipart()
if not frames:
break # Interrupted
if len(frames) == 3:
# Simulate various problems, after a few cycles
cycles += 1
if cycles > 3 and randint(0, 5) == 0:
print("I: Simulating a crash")
break
if cycles > 3 and randint(0, 5) == 0:
print("I: Simulating CPU overload")
yield gen.sleep(3)
print("I: Normal reply")
yield worker.send_multipart(frames)
liveness = HEARTBEAT_LIVENESS
yield gen.sleep(1) # Do some heavy work
elif len(frames) == 1 and frames[0] == PPP_HEARTBEAT:
print("I: Queue heartbeat")
liveness = HEARTBEAT_LIVENESS
else:
print("E: Invalid message: %s" % frames)
interval = INTERVAL_INIT
else:
liveness -= 1
if liveness == 0:
print("W: Heartbeat failure, can't reach queue")
print("W: Reconnecting in %0.2fs..." % interval)
yield gen.sleep(interval)
if interval < INTERVAL_MAX:
interval *= 2
poller.unregister(worker)
worker.setsockopt(zmq.LINGER, 0)
worker.close()
worker = yield worker_socket(context, poller)
liveness = HEARTBEAT_LIVENESS
if time.time() > heartbeat_at:
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
print("I: Worker heartbeat")
yield worker.send(PPP_HEARTBEAT)
def run_client(context):
print("I: Connecting to server...")
client = context.socket(zmq.REQ)
client.connect(SERVER_ENDPOINT)
poll = Poller()
poll.register(client, zmq.POLLIN)
sequence = 0
retries_left = REQUEST_RETRIES
while retries_left:
sequence += 1
request = str(sequence)
print("I: Sending (%s)" % request)
yield client.send_string(request)
expect_reply = True
while expect_reply:
socks = yield poll.poll(REQUEST_TIMEOUT)
socks = dict(socks)
if socks.get(client) == zmq.POLLIN:
reply = yield client.recv()
if not reply:
break
if int(reply) == sequence:
print("I: Server replied OK (%s)" % reply)
retries_left = REQUEST_RETRIES
expect_reply = False
else:
print("E: Malformed reply from server: %s" % reply)
else:
print("W: No response from server, retrying...")
# Socket is confused. Close and remove it.
print('W: confused')
client.setsockopt(zmq.LINGER, 0)
client.unbind(SERVER_ENDPOINT)
#client.close()
poll.unregister(client)
retries_left -= 1
if retries_left == 0:
print("E: Server seems to be offline, abandoning")
return
print("I: Reconnecting and resending (%s)" % request)
# Create new connection
client = context.socket(zmq.REQ)
client.connect(SERVER_ENDPOINT)
poll.register(client, zmq.POLLIN)
yield client.send_string(request)
def run_client(context):
print("I: Connecting to server...")
client = context.socket(zmq.REQ)
client.connect(SERVER_ENDPOINT)
poll = Poller()
poll.register(client, zmq.POLLIN)
sequence = 0
retries_left = REQUEST_RETRIES
while retries_left:
sequence += 1
request = str(sequence)
print("I: Sending (%s)" % request)
yield client.send_string(request)
expect_reply = True
while expect_reply:
socks = yield poll.poll(REQUEST_TIMEOUT)
socks = dict(socks)
if socks.get(client) == zmq.POLLIN:
reply = yield client.recv()
if not reply:
break
if int(reply) == sequence:
print("I: Server replied OK (%s)" % reply)
retries_left = REQUEST_RETRIES
expect_reply = False
else:
print("E: Malformed reply from server: %s" % reply)
else:
print("W: No response from server, retrying...")
# Socket is confused. Close and remove it.
print('W: confused')
client.setsockopt(zmq.LINGER, 0)
client.unbind(SERVER_ENDPOINT)
#client.close()
poll.unregister(client)
retries_left -= 1
if retries_left == 0:
print("E: Server seems to be offline, abandoning")
return
print("I: Reconnecting and resending (%s)" % request)
# Create new connection
client = context.socket(zmq.REQ)
client.connect(SERVER_ENDPOINT)
poll.register(client, zmq.POLLIN)
yield client.send_string(request)
class TornadoRdsBusClient(object):
ASC = 1
DESC = -1
def __init__(self,
url: str,
logger,
request_timeout: int = None,
database: str = None):
self._logger = logger
self._database = database
self._context = Context.instance()
self._poller = Poller()
self._request = self._context.socket(zmq.DEALER)
self._request_timeout = request_timeout or 60
self._rds_bus_url = url
self._request.connect(self._rds_bus_url)
self._request_dict = dict()
self._io_loop = ioloop.IOLoop.current()
self._io_loop.add_callback(self.start)
@classmethod
def pack(cls,
database: str,
key: str,
parameter: dict,
is_query: bool = False,
order_by: list = None,
page_no: int = None,
per_page: int = None,
found_rows: bool = False):
"""
打包请求数据
:param database: RDS-BUS的数据库类名
:param key: 数据库类所持有的实例名
:param parameter: 参数字典
:param is_query: 是否为查询操作
:param order_by: 排序信息 [{"column": "字段名", "order": TornadoRdsBusClient.ASC/TornadoRdsBusClient.DESC}]
:param page_no: 当前页（范围[0-n) n指第n页）
:param per_page: 每页记录数
:param found_rows: 是否统计总数
:return:
"""
if is_query:
amount = int(per_page) if per_page else None
offset = int(page_no) * amount if page_no else None
limit = (dict(amount=amount, offset=offset) if offset else dict(
amount=amount)) if amount else None
result = dict(command="{}/{}".format(database, key),
data=dict(var=parameter,
order_by=order_by,
limit=limit,
found_rows=found_rows))
else:
result = dict(command="{}/{}".format(database, key),
data=dict(var=parameter))
return result
@gen.coroutine
def query(self,
key: str,
parameter: dict,
order_by: list = None,
page_no: int = None,
per_page: int = None,
found_rows: bool = False,
database: str = None,
execute: bool = True):
"""
查询接口
:param database: RDS-BUS的数据库类名
:param key: 数据库类所持有的语句实例名
:param parameter: 参数字典
:param order_by: 排序信息 [{"column": "字段名", "order": TornadoRdsBusClient.ASC/TornadoRdsBusClient.DESC}]
:param page_no: 当前页（范围[0-n) n指第n页）
:param per_page: 每页记录数
:param found_rows: 是否统计总数
:param execute: 是否执行
:return:
"""
_database = database or self._database
argument = self.pack(database=_database,
key=key,
parameter=parameter,
is_query=True,
order_by=order_by,
page_no=page_no,
per_page=per_page,
found_rows=found_rows)
if execute:
response = yield self._send(operation=OperationType.QUERY,
argument=argument)
result = RdsData(response)
else:
result = argument
return result
@gen.coroutine
def insert(self,
key: str,
parameter: dict,
database: str = None,
execute: bool = True):
"""
新增接口
:param database: RDS-BUS的数据库类名
:param key: 数据库类所持有的语句实例名
:param parameter: 参数字典
:param execute: 是否执行
:return:
"""
_database = database or self._database
argument = self.pack(database=_database, key=key, parameter=parameter)
if execute:
response = yield self._send(operation=OperationType.INSERT,
argument=argument)
result = RdsData(response)
else:
result = argument
return result
@gen.coroutine
def update(self,
key: str,
parameter: dict,
database: str = None,
execute: bool = True):
"""
更新接口
:param database: RDS-BUS的数据库类名
:param key: 数据库类所持有的语句实例名
:param parameter: 参数字典
:param execute: 是否执行
:return:
"""
_database = database or self._database
argument = self.pack(database=_database, key=key, parameter=parameter)
if execute:
response = yield self._send(operation=OperationType.UPDATE,
argument=argument)
result = RdsData(response)
else:
result = argument
return result
@gen.coroutine
def delete(self,
key: str,
parameter: dict,
database: str = None,
execute: bool = False):
"""
删除接口
:param database: RDS-BUS的数据库类名
:param key: 数据库类所持有的语句实例名
:param parameter: 参数字典
:param execute: 是否执行
:return:
"""
_database = database or self._database
argument = self.pack(database=_database, key=key, parameter=parameter)
if execute:
response = yield self._send(operation=OperationType.DELETE,
argument=argument)
result = RdsData(response)
else:
result = argument
return result
@gen.coroutine
def transaction(self, data: list, database: str = None):
"""
事务接口
:param database: RDS-BUS的数据库类名
:param data: 操作列表
:return:
"""
_database = database or self._database
result = yield self._send(
operation=OperationType.TRANSACTION,
argument=dict(command="{}/transaction".format(_database),
data=data))
return RdsListData(result)
@gen.coroutine
def batch(self, data: list, database: str = None):
"""
批量接口
:param database: RDS-BUS的数据库类名
:param data: 操作列表
:return:
"""
_database = database or self._database
result = yield self._send(operation=OperationType.BATCH,
argument=dict(
command="{}/batch".format(_database),
data=data))
return RdsListData(result)
@gen.coroutine
def start(self):
self._poller.register(self._request, zmq.POLLIN)
while True:
events = yield self._poller.poll()
if self._request in dict(events):
response = yield self._request.recv_json()
self._logger.debug("received {}".format(response))
if response["id"] in self._request_dict:
future = self._request_dict.pop(response["id"])
if HttpResult.is_duplicate_data_failure(response["code"]):
future.set_exception(
DuplicateDataException.new_exception(
response["desc"]))
elif HttpResult.is_failure(response["code"]):
future.set_exception(
CallServiceException(method="ZMQ",
url=self._rds_bus_url,
errmsg=response["desc"]))
else:
future.set_result(response["data"])
else:
self._logger.warning(
"unknown response {}".format(response))
def stop(self):
self._poller.unregister(self._request)
def shutdown(self):
self.stop()
self._request.close()
def _send(self, operation, argument):
"""
:param operation:
:param argument:
:return:
"""
request_id = get_unique_id()
self._request_dict[request_id] = Future()
self._io_loop.call_later(60, self._session_timeout, request_id)
self._request.send_multipart([
json.dumps(
dict(id=request_id,
operation=operation.value,
argument=argument)).encode("utf-8")
])
return self._request_dict[request_id]
def _session_timeout(self, request_id):
if request_id in self._request_dict:
future = self._request_dict.pop(request_id)
future.set_exception(
ServerTimeoutException(method="ZMQ", url=self._rds_bus_url))
