class Client(object):
def __init__(self, data):
self.corr_id = str(uuid4())
self.json_input = data
self.response = None
url = environ.get('CLOUDAMQP_URL',
'amqps://*****:*****@woodpecker.rmq.cloudamqp.com/ahsmnsum')
params = URLParameters(url)
self.connection = BlockingConnection(params)
self.channel = self.connection.channel()
result = self.channel.queue_declare(queue='', exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(
queue=self.callback_queue,
on_message_callback=self.on_response,
auto_ack=True)
def on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = body
def call(self):
self.channel.basic_publish(
exchange='',
routing_key='send',
properties=BasicProperties(
reply_to=self.callback_queue,
correlation_id=self.corr_id,
),
body=self.json_input)
while self.response is None:
self.connection.process_data_events()
return self.response
def process_step(self, form):
if self.steps.current == '0':
text_type = form.data['0-text_type']
text = form.data['0-text']
# Prepare message
uima_response = {}
uima_response['response'] = None
uima_corr_id = str(uuid.uuid4())
uima_body = json.dumps({
'text': text,
'mode': text_type,
})
def uima_on_response(channel, method, props, body):
if uima_corr_id == props.correlation_id:
uima_response['response'] = body
# Call UIMA
uima_connection = BlockingConnection(
ConnectionParameters(host=RABBITMQ_SERVER))
uima_channel = uima_connection.channel()
uima_result = uima_channel.queue_declare(exclusive=True)
uima_callback_queue = uima_result.method.queue
uima_channel.basic_consume(uima_on_response,
no_ack=True,
queue=uima_callback_queue)
uima_channel.basic_publish(exchange='',
routing_key='uima_plain_worker',
properties=BasicProperties(
reply_to=uima_callback_queue,
content_type='application/json',
correlation_id=uima_corr_id,
),
body=uima_body)
while uima_response['response'] is None:
uima_connection.process_data_events()
# Transform result into HTML
result = uima_response['response']
result = xmlmap.load_xmlobject_from_string(result,
xmlclass=RocheTEI)
result = result.body.xsl_transform(xsl=XSL_TRANSFORM_1).serialize()
self.uima_result = result
return self.get_form_step_data(form)
class Client(object):
def __init__(self):
credentials = PlainCredentials(RABBIT_SERVER['USER'],
RABBIT_SERVER['PASS'])
parameters = ConnectionParameters(RABBIT_SERVER['HOST'],
RABBIT_SERVER['PORT'],
credentials=credentials)
self.connection = BlockingConnection(parameters)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=QUEUE_MESSAGES)
result = self.channel.queue_declare(queue=EXCHANGE_MESSAGES,
exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(queue=self.callback_queue,
on_message_callback=self.on_response,
auto_ack=True)
def on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = True
self.response_body = body
self.response_status = props.headers['Status']
def call(self, request_path, request_method, request_body):
self.response = False
self.corr_id = str(uuid.uuid4())
self.channel.basic_publish(exchange=EXCHANGE_MESSAGES,
routing_key=QUEUE_MESSAGES,
properties=BasicProperties(
reply_to=self.callback_queue,
correlation_id=self.corr_id,
content_type='application/json',
headers={
'Method': request_method,
'Path': request_path
}),
body=json.dumps(request_body))
while not self.response:
self.connection.process_data_events()
return self.response_body, self.response_status
def process_step(self, form):
if self.steps.current == '0':
text_type = form.data['0-text_type']
text = form.data['0-text']
# Prepare message
uima_response = {}
uima_response['response'] = None
uima_corr_id = str(uuid.uuid4())
uima_body = json.dumps({'text': text, 'mode': text_type, })
def uima_on_response(channel, method, props, body):
if uima_corr_id == props.correlation_id:
uima_response['response'] = body
# Call UIMA
uima_connection = BlockingConnection(ConnectionParameters(host=RABBITMQ_SERVER))
uima_channel = uima_connection.channel()
uima_result = uima_channel.queue_declare(exclusive=True)
uima_callback_queue = uima_result.method.queue
uima_channel.basic_consume(uima_on_response, no_ack=True, queue=uima_callback_queue)
uima_channel.basic_publish(exchange='',
routing_key='uima_plain_worker',
properties=BasicProperties(reply_to=uima_callback_queue,
content_type='application/json',
correlation_id=uima_corr_id, ),
body=uima_body)
while uima_response['response'] is None:
uima_connection.process_data_events()
# Transform result into HTML
result = uima_response['response']
result = xmlmap.load_xmlobject_from_string(result, xmlclass=RocheTEI)
result = result.body.xsl_transform(xsl=XSL_TRANSFORM_1).serialize()
self.uima_result = result
return self.get_form_step_data(form)
def _PushHeartbeat(self,
connection: pika.BlockingConnection,
channelId: str,
heartbeatInterval: int,
nextHeartbeat: int = -1):
if nextHeartbeat < 0:
nextHeartbeat = time.time() + heartbeatInterval
heartBeatStart = time.time()
if heartBeatStart >= nextHeartbeat:
try:
self._logger.debug(
f'Triggering connection heartbeat with channel {channelId}'
)
connection.process_data_events()
self._logger.debug(
f'Connection heartbeat triggered with channel {channelId} after {time.time() - heartBeatStart} seconds.'
)
except Exception as exception:
self._logger.debug(
f'Heartbeat failure with channel {channelId}: {str(type(exception))}: {str(exception)}'
)
nextHeartbeat = time.time() + heartbeatInterval
return nextHeartbeat
class RpcClient(object):
def __init__(self):
self.connection = BlockingConnection(
ConnectionParameters(host='localhost'))
self.channel = self.connection.channel()
result = self.channel.queue_declare(exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(self.on_response,
no_ack=True,
queue=self.callback_queue)
def on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = body
def call(self, name):
'''
接收参数name作为被调用的远程函数的名字，通过app_id传给服务端程序
:param name:
:return:
'''
self.response = None
self.corr_id = str(uuid4())
self.channel.basic_publish(exchange='',
routing_key='rpc_queue',
properties=BasicProperties(
reply_to=self.callback_queue,
correlation_id=self.corr_id,
app_id=str(name),
),
body="request")
while self.response is None:
self.connection.process_data_events()
return str(self.response)
class ServiceBus():
def __init__(self):
self.__channel = None
self.__queues = []
def init_connection(self):
self.__connection = BlockingConnection(
ConnectionParameters(host=SERVICEBUS_HOST))
self.__channel = self.__connection.channel()
def add_queue(self, queue_name, emitter):
event_object = {'queue_name': queue_name, 'emitter': emitter}
self.__queues.append(event_object)
def receive(self, channel_name, send=''):
self.__channel_name = channel_name
self.__start_one_channel(self.__on_response)
self.response = None
self.corr_id = str(uuid.uuid4())
self.__publish_channel(self.__channel, self.__channel_name,
self.corr_id, send, self.callback_queue)
while self.response is None:
self.__connection.process_data_events()
return self.response
def send(self):
self.__start_multiple_channel()
def status(self):
if self.__channel:
return len(self.__channel.consumer_tags)
return 0
def start(self):
self.__channel.start_consuming()
def stop(self):
self.__channel.stop_consuming()
def close_connection(self):
self.__connection.close()
def __start_one_channel(self, on_event):
result = self.__queue_declaration('', True)
self.callback_queue = result.method.queue
self.__consume_channel(self.callback_queue, on_event, True)
def __start_multiple_channel(self):
self.__channel.basic_qos(prefetch_count=1)
for queue in self.__queues:
result = self.__queue_declaration(queue['queue_name'])
callback_queue = result.method.queue
self.__consume_channel(callback_queue, self.__on_request)
def __on_request(self, ch, method, props, body):
request = contains(
self.__queues,
lambda queue: queue['queue_name'] == method.routing_key)
request = request['emitter'](
self.__transform_body(body)) if request != None else None
self.__publish_channel(ch, props.reply_to, props.correlation_id,
request)
ch.basic_ack(delivery_tag=method.delivery_tag)
def __on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = json.loads(body.decode('utf8'))
def __queue_declaration(self, queue, exclusive=False):
result = self.__channel.queue_declare(queue=queue, exclusive=exclusive)
return result
def __consume_channel(self, callback_queue, on_event, auto_ack=False):
self.__channel.basic_consume(queue=callback_queue,
on_message_callback=on_event,
auto_ack=auto_ack)
def __publish_channel(self,
channel,
routing_key,
correlation_id,
body,
reply_to=None):
channel.basic_publish(exchange='',
routing_key=routing_key,
properties=BasicProperties(
reply_to=reply_to,
correlation_id=correlation_id,
content_type='application/json'),
body=json.dumps(body))
def __transform_body(self, data):
dict_str = data.decode('UTF-8')
final_data = ast.literal_eval(dict_str)
return final_data
def _heartbeat(connection: pika.BlockingConnection):
while True:
logger.debug('SENDING HEARTBEAT')
connection.add_callback_threadsafe(
lambda: connection.process_data_events(time_limit=10))
time.sleep(10)
class ServiceBus(threading.Thread):
def __init__(self, *args, **kwargs):
super(ServiceBus, self).__init__(*args, **kwargs)
self.__connection = BlockingConnection(
ConnectionParameters(host='localhost'))
self.__channel = self.__connection.channel()
def __start_channel(self, channel_type, on_event):
exclusive = False
auto_ack = False
queue = ''
if channel_type == 'receive':
exclusive = True
auto_ack = True
if channel_type == 'send':
self.__channel.basic_qos(prefetch_count=1)
queue = self.__channel_name
result = self.__channel.queue_declare(queue=queue, exclusive=exclusive)
self.callback_queue = result.method.queue
self.__channel.basic_consume(queue=self.callback_queue,
on_message_callback=on_event,
auto_ack=auto_ack)
def __on_request(self, ch, method, props, body):
request = self.travel_event()
ch.basic_publish(exchange='',
routing_key=props.reply_to,
properties=BasicProperties(
correlation_id=props.correlation_id,
content_type='application/json'),
body=json.dumps(request))
ch.basic_ack(delivery_tag=method.delivery_tag)
def __on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = json.loads(body.decode('utf8'))
def receive(self, channel_name, send=''):
self.__channel_name = channel_name
self.__start_channel('receive', self.__on_response)
self.response = None
self.corr_id = str(uuid.uuid4())
self.__channel.basic_publish(exchange='',
routing_key=self.__channel_name,
properties=BasicProperties(
reply_to=self.callback_queue,
correlation_id=self.corr_id,
content_type='application/json'),
body=json.dumps(send))
while self.response is None:
self.__connection.process_data_events()
return self.response
def send(self, channel_name, receive=None):
self.__channel_name = channel_name
self.travel_event = receive
self.__start_channel('send', self.__on_request)
def run(self):
self.__channel.start_consuming()
def stop(self):
self.__channel.stop_consuming()
def close_connection(self):
self.__connection.close()
def keep_rabbit_connection_online(connection: pika.BlockingConnection) -> None:
while not STOP_HEARTBEAT:
logging.info("RabbitMQ heartbeat.")
connection.process_data_events()
time.sleep(30)
def annotate_text(request, text, function, lemma, juan=-1):
"""
Annotate a single text. Run UIMA through a remote procedure call.
"""
from .models import Annotation
collection_path = None
#
# Find the collection path this text belongs to
#
os.chdir('/docker/dublin-store/')
for (dirpath, dirnames, filenames) in os.walk(u'浙江大學圖書館'):
if dirpath.endswith(unicode(text)):
collection_path = '/docker/dublin-store/' + dirpath
break
#
# RPC UIMA worker
#
uima_response = {}
uima_response['response'] = None
uima_corr_id = str(uuid.uuid4())
uima_body = json.dumps({'text': text,
'function': function,
'collection_path': collection_path,
'juan': juan,
'lemma': lemma})
def uima_on_response(channel, method, props, body):
if uima_corr_id == props.correlation_id:
uima_response['response'] = body
uima_connection = BlockingConnection(ConnectionParameters(host=RABBITMQ_SERVER))
uima_channel = uima_connection.channel()
uima_result = uima_channel.queue_declare(exclusive=True)
uima_callback_queue = uima_result.method.queue
uima_channel.basic_consume(uima_on_response, no_ack=True, queue=uima_callback_queue)
uima_channel.basic_publish(exchange='',
routing_key='uima_worker',
properties=BasicProperties(reply_to=uima_callback_queue,
content_type='application/json',
correlation_id=uima_corr_id, ),
body=uima_body)
while uima_response['response'] is None:
uima_connection.process_data_events()
response = uima_response['response']
if response == 'ERROR':
pass
# Invalidate page?
# TODO
annotation = Annotation()
annotation.tei_tag = function
annotation.lemma = lemma
annotation.ip = request.META['REMOTE_ADDR']
annotation.save()
return HttpResponse(response)
class Publisher:
def __init__(
self,
host: str = Default.RABBITMQ_HOST,
port: int = Default.RABBITMQ_PORT,
task_queue: str = Default.TASK_QUEUE,
response_queue: str = Default.RESPONSE_QUEUE,
):
"""
Init rabbitmq publisher
:param host: rabbitmq host
:param port: rabbitmq port
:param task_queue: queue name
:param response_queue: response queue name
"""
self.task_queue_name = task_queue
self.response_queue_name = response_queue
self.connection = BlockingConnection(
ConnectionParameters(
host=host,
port=port,
))
self.channel = self.connection.channel()
self.task_queue = self.channel.queue_declare(
queue=self.task_queue_name)
self.response_queue = self.channel.queue_declare(
queue=self.response_queue_name, exclusive=True)
self.channel.basic_consume(
queue=self.response_queue_name,
on_message_callback=self.task_response,
auto_ack=True,
)
@staticmethod
def task_response(
channel: BlockingChannel,
method: Basic.Deliver,
properties: BasicProperties,
body: bytes,
) -> None:
"""
Process tasks response
:param channel: channel
:param method: method
:param properties: task properties
:param body: task body
:return: None
"""
logger.info(msg=f"Done task {properties.correlation_id}")
def publish_task(self, task: TaskItem, cases: list) -> None:
"""
Publish task
:param task: task item
:param cases: list of cases
:return: None
"""
task_body = dumps({
"task": task.as_json(),
"cases": cases
}).encode(encoding="utf-8")
self.channel.basic_publish(
exchange="",
routing_key=self.task_queue_name,
properties=BasicProperties(
reply_to=self.response_queue_name,
correlation_id=task.task_id,
),
body=task_body,
)
def process_data_events(self, time_limit: int = 1) -> None:
"""
Process data events
:param time_limit: limit time of processing (in seconds)
:return: None
"""
logger.info(
msg=f"Check for new events: "
f"{self.task_queue.method.message_count} tasks in queue, "
f"{self.response_queue.method.message_count} responses in queue")
self.connection.process_data_events(time_limit=time_limit)
def __del__(self):
"""
Force close connection
:return: None
"""
try:
self.connection.close()
except:
pass
