def get_url_for_company_name(ch, method, properties, body):
""" lookup URL for a company using fortune's profile on the company """
company_data = json.loads(body)
company_name = company_data["name"]
print " [x] Received %r" % (company_name)
# hacks for slugfiying
company_name_slug = company_name.lower().replace("& ", "").replace(".", "").replace(",", "").replace(" ", "-")
company_profile_url = "http://www.inc.com/inc5000/profile/%s" % company_name_slug
r = requests.get(company_profile_url)
soup = BeautifulSoup(r.text)
detail_section = soup.find("div", "inc5000companydata")
try:
company_url = detail_section.find("a")["href"]
except:
company_url = None
company_data["url"] = company_url
to_send = json.dumps(company_data)
ch.basic_ack(delivery_tag = method.delivery_tag)
# now that we've gotten the company's URL, send the data to another worker to get the MG score
connection = BlockingConnection(ConnectionParameters('localhost'))
channel = connection.channel()
channel.exchange_declare(exchange=EXCHANGE_NAME, type='direct')
channel.basic_publish(exchange=EXCHANGE_NAME, routing_key='mg', body=to_send)
class Client(object):
def __init__(self, data):
self.corr_id = str(uuid4())
self.json_input = data
self.response = None
url = environ.get('CLOUDAMQP_URL',
'amqps://*****:*****@woodpecker.rmq.cloudamqp.com/ahsmnsum')
params = URLParameters(url)
self.connection = BlockingConnection(params)
self.channel = self.connection.channel()
result = self.channel.queue_declare(queue='', exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(
queue=self.callback_queue,
on_message_callback=self.on_response,
auto_ack=True)
def on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = body
def call(self):
self.channel.basic_publish(
exchange='',
routing_key='send',
properties=BasicProperties(
reply_to=self.callback_queue,
correlation_id=self.corr_id,
),
body=self.json_input)
while self.response is None:
self.connection.process_data_events()
return self.response
class Publisher(object):
connection = None
channel = None
exchange = None
def __init__(self, props):
self.props = props
def start(self, exchange):
self.exchange = exchange
self.connection = BlockingConnection()
self.connection.set_backpressure_multiplier(self.props.backpressure)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=exchange, durable=True, exclusive=False, auto_delete=False)
def publish(self, status):
self.channel.basic_publish(
exchange="",
routing_key=self.exchange,
body=status,
properties=BasicProperties(content_type="text/plain", delivery_mode=1),
)
def close(self):
self.connection.close()
def __init__(self,
queue,
rabhost,
user,
passwd,
msg,
routing_key='py-fxp-publisher'):
self.queue = queue
self.host = rabhost
self.topic_type = 'direct'
self.user = user
self.passwd = passwd
self.result_exchange = 'py-fxp-publisher'
self.publish_connection = BlockingConnection(
ConnectionParameters(host=self.host,
port=5672,
virtual_host='/',
retry_delay=3,
connection_attempts=60,
credentials=PlainCredentials(
self.user, self.passwd)))
self.publish_channel = self.publish_connection.channel()
self.result_routing = routing_key
self.msg = msg
self.publish_channel.queue_declare(queue=self.queue,
passive=False,
durable=True,
exclusive=False,
auto_delete=False)
def send_msg_example(queue: str, message: str):
"""Пример отправки сообщения в очередь
:param queue: название очереди
:param message: тело сообщения
"""
# подключаемся к серверу
connection = BlockingConnection(ConnectionParameters('localhost'))
channel = connection.channel()
# проверяем, что очередь сущетсвует, или создаем новую
# method = channel.queue_declare('') создаст временную очередь со случайным именем
channel.queue_declare(
queue=queue, # название
durable=True, # объявить устойчивой
)
# необязательно: создаём обменник и связываем с очередью
# channel.exchange_declare('logs', exchange_type='fanout')
# channel.queue_bind(method.method.queue, 'logs')
# с типом fanout при отправке сообщения routing_key можно не указывать
# отправляем сообщение
channel.basic_publish(
exchange='', # точка обмена
routing_key=queue, # имя очереди
body=message, # сообщение
properties=BasicProperties(
delivery_mode=2, # объявить устойчивым
)
)
connection.close()
def __init__(self, decider: BoboDecider, exchange_name: str, user_id: str,
parameters: ConnectionParameters,
max_sync_attempts: int) -> None:
super().__init__()
connection = BlockingConnection(parameters=parameters)
channel = connection.channel()
self.decider = decider
self.exchange_name = exchange_name
self.user_id = user_id
self.parameters = parameters
self.sync_id = None
self.max_sync_attempts = max_sync_attempts
self._queue_transition = Queue()
self._queue_clone = Queue()
self._queue_halt = Queue()
self._queue_final = Queue()
self._queue_action = Queue()
self._subs = []
self._connection = connection
self._channel = channel
self._is_synced = False
self._sync_response = None
def main():
def on_message_receive_callback(channel, method, properties, body):
log("Mensaje recibido.")
message = json.loads(body)
operation = message["operation"]
image_processor.run(operation)
log("Todas las tareas asociadas a la operación '{}' han finalizado.".
format(operation))
log("Inicializando conexión a RabbitMQ en '{}:{}'...".format(
RABBITMQ_HOST, RABBITMQ_PORT))
connection = BlockingConnection(
ConnectionParameters(host=RABBITMQ_HOST,
port=RABBITMQ_PORT,
credentials=PlainCredentials(
RABBITMQ_USER, RABBITMQ_PASS),
virtual_host="/"))
channel = connection.channel()
channel.exchange_declare(exchange="XPyxel-Nodes",
exchange_type="direct",
durable=True)
channel.queue_declare(queue=WORKER_ID, durable=True)
channel.basic_consume(queue=WORKER_ID,
on_message_callback=on_message_receive_callback,
auto_ack=True)
log("Pyxel [{}] ha iniciado correctamente y está esperando peticiones. Presione [Ctrl]+C para salir..."
.format(WORKER_ID))
channel.start_consuming()
def _run_amqp(self):
url = self.output_id.amqp_host_id.connection
connection = BlockingConnection(URLParameters(url))
channel = connection.channel()
result = channel.basic_publish(**self._generate_amqp_data())
_logger.debug(result)
connection.close()
def start_channel(self):
connection = BlockingConnection(ConnectionParameters(self.broker_host))
channel = connection.channel()
channel.basic_qos(prefetch_count=1)
channel.queue_declare(queue=self.queue, durable=True)
return channel
def direct_exchange():
connection = BlockingConnection(ConnectionParameters("localhost"))
channel = connection.channel()
# connect to exchange and queue
channel.exchange_declare(exchange="direct_logs", type="direct")
# create multiple queues to handle each type of severity
severities = ["critical", "error", "warning", "info", "debug"]
for severity in severities:
channel.queue_declare(queue=severity)
channel.queue_bind(exchange="direct_logs", queue=severity, routing_key=severity)
# pick a random severity level queue to attach this worker to
from random import choice
severities = ["critical", "error", "warning", "info", "debug"]
severity = choice(severities)
severity = "info"
# don't send tasks to this worker until it's ready, keep them in the queue for other workers
# THIS IS IMPORTANT, otherwise adding more workers won't help anything cause
# all msgs will already have been sent to the currently 'busy' worker(s)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(callback, queue=severity)
print " [*] Waiting for messages in %s queue. To exit press CTRL+C" % (severity)
channel.start_consuming()
def __init__(
self,
host: str = Default.RABBITMQ_HOST,
port: int = Default.RABBITMQ_PORT,
task_queue: str = Default.TASK_QUEUE,
response_queue: str = Default.RESPONSE_QUEUE,
):
"""
Init rabbitmq publisher
:param host: rabbitmq host
:param port: rabbitmq port
:param task_queue: queue name
:param response_queue: response queue name
"""
self.task_queue_name = task_queue
self.response_queue_name = response_queue
self.connection = BlockingConnection(
ConnectionParameters(
host=host,
port=port,
))
self.channel = self.connection.channel()
self.task_queue = self.channel.queue_declare(
queue=self.task_queue_name)
self.response_queue = self.channel.queue_declare(
queue=self.response_queue_name, exclusive=True)
self.channel.basic_consume(
queue=self.response_queue_name,
on_message_callback=self.task_response,
auto_ack=True,
)
class Publisher(object):
connection = None
channel = None
exchange = None
def __init__(self, props):
self.props = props
def start(self, exchange):
self.exchange = exchange
self.connection = BlockingConnection()
self.connection.set_backpressure_multiplier(self.props.backpressure)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=exchange,
durable=True,
exclusive=False,
auto_delete=False)
def publish(self, status):
self.channel.basic_publish(exchange="",
routing_key=self.exchange,
body=status,
properties=BasicProperties(
content_type="text/plain",
delivery_mode=1))
def close(self):
self.connection.close()
class Consumer():
connection = None
channel = None
queue_name = None
on_message_callback = None
def __init__(self, props):
self.props = props
def start(self, on_message_callback, exchange):
self.on_message_callback = on_message_callback
self.connection = BlockingConnection()
self.connection.set_backpressure_multiplier(self.props.backpressure)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=exchange,
durable=True,
exclusive=False,
auto_delete=False)
self.channel.basic_consume(self.on_message, exchange, no_ack=True)
self.channel.start_consuming()
def on_message(self, a, b, c, message):
self.on_message_callback(message)
def __init__(self, broker: Broker, rabbit_host: str):
self._logger = logging.getLogger(__name__)
self._rabbit_host = rabbit_host
self._broker = broker
self._broker.subscribe_broker(self)
self._broker_subscribers = []
# Init rabbit mq
self._logger.info(f"Connecting to rabbit host {rabbit_host}")
self._rabbit_connection = BlockingConnection(
ConnectionParameters(rabbit_host))
# self._rabbit_connection = pika.connection.Connection(pika.ConnectionParameters(rabbit_host))
self._rabbit_channel = self._rabbit_connection.channel()
for q in [
QueueName.TRADE_ACCOUNT, QueueName.ORDERS, QueueName.TRADES,
QueueName.MONEY_LIMITS, QueueName.STOCK_LIMITS,
QueueName.MSG_REPLY
]:
self._logger.info(f"Declaring rabbit queue {q}")
self._rabbit_channel.queue_declare(queue=q, durable=True)
# Subscribe to buy/sell events in new thread because pika consumes synchronously only
self._consumer_rabbit_connection = None
self._consumer_rabbit_channel = None
Thread(target=self.listen_commands).start()
self._logger.info("Initialized")
def _connect(self):
try:
logger.info("attempt to open connection",
server="primary",
category="rabbitmq")
return BlockingConnection(URLParameters(self.rabbitmq_url))
except AMQPError as e:
logger.error(
"unable to open connection",
exc_info=e,
server="primary",
category="rabbitmq",
)
try:
logger.info(
"attempt to open connection",
server="secondary",
category="rabbitmq",
)
return BlockingConnection(
URLParameters(self.rabbitmq_secondary_url))
except AMQPError as err:
logger.error(
"unable to open connection",
exc_info=e,
server="secondary",
category="rabbitmq",
)
raise err
class Service( object ):
def __init__( self, groups, host = 'localhost', verbose = True, port = 5672, user = '', password = '' ):
credentials = PlainCredentials(user, password)
self._connection = BlockingConnection( ConnectionParameters( host, port, vhost, credentials ) )
self._channel = self._connection.channel()
self._channel.exchange_declare( exchange = exchange_name, type = 'topic' )
self._queueID = self._channel.queue_declare( exclusive = True ).method.queue
for topic in groups:
self._channel.queue_bind(exchange = exchange_name, queue = self._queueID, routing_key = topic)
def _handle( self, c, m, p, b):
print b
def close( self ):
self._channel.stop_consuming()
print 'done', datetime.datetime.now()
self._connection.close()
def run( self ):
self._channel.basic_consume( self._handle, queue = self._queueID, no_ack = True )
self._channel.start_consuming()
def _get_consumer_rmq_objects(cls, host, port, creds, exchange, queue_name, prefetch_count):
connection = BlockingConnection(
ConnectionParameters(host=host, port=port, credentials=creds),
)
channel = connection.channel()
channel.basic_qos(prefetch_count=prefetch_count)
cls._declare_exchange(channel, exchange)
channel.queue_declare(queue_name, durable=True, exclusive=False)
for cqrs_id, replica_model in ReplicaRegistry.models.items():
channel.queue_bind(exchange=exchange, queue=queue_name, routing_key=cqrs_id)
# Every service must have specific SYNC routes
channel.queue_bind(
exchange=exchange,
queue=queue_name,
routing_key='cqrs.{}.{}'.format(queue_name, cqrs_id),
)
channel.basic_consume(
queue=queue_name,
on_message_callback=cls._consume_message,
auto_ack=False,
exclusive=False,
)
return connection, channel
def __init__( self, host = 'localhost', port = 5672, user = '', password = '', vhost = '/', routingKey = ''):
credentials = PlainCredentials( user, password )
self._connection = BlockingConnection( ConnectionParameters( host, port, vhost, credentials ) )
#self._connection = SelectConnection( ConnectionParameters( host, port, vhost, credentials ) )
self._channel = self._connection.channel()
self._channel.exchange_declare( exchange = exchange_name, type = 'topic' )
self.rkey = routingKey
def connect(self, consumer):
log.info("Connecting to RabbitMQ...")
user = self.settings["rabbit_user"]
password = self.settings["rabbit_password"]
queue = self.settings["backend_queue"]
parameters = dict(
host=self.settings["rabbit_host"],
port=self.settings["rabbit_port"],
socket_timeout=self.settings["rabbit_connecion_timeout"],
)
try:
credentials = PlainCredentials(user, password)
param = ConnectionParameters(**parameters)
self.connection = BlockingConnection(param)
self.channel = self.connection.channel()
self.channel.basic_consume(
queue=queue, auto_ack=True, on_message_callback=consumer
)
except Exception as e:
log.error("Something went wrong with connection to RabbitMQ... %s", e)
def main():
connection = BlockingConnection(ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='calc', durable=True)
print ' [*] Waiting for messages. To exit press CTRL+C'
def callback(ch, method, properties, body):
print " [x] Persistence {}".format(properties.delivery_mode == 2)
print " [x] Received: {}".format(body)
try:
print " [.] Caclulated: {} = {}".format(body, eval(body))
except SyntaxError:
print " [.] Invalid syntax"
print " [x] Done"
ch.basic_ack(delivery_tag=method.delivery_tag)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(callback,
queue='calc')
try:
channel.start_consuming()
except KeyboardInterrupt:
print "[*] Exit"
return
def main():
connection = BlockingConnection(
parameters=URLParameters("amqp://*****:*****@localhost:5672/%2F")
)
props = BasicProperties(
content_type="application/json",
content_encoding="utf8",
delivery_mode=2,
)
ch = connection.channel()
ch.exchange_declare(exchange="worker.mm", exchange_type="topic")
ch.queue_declare(queue="splitter_action_request_q")
ch.queue_bind(
queue="splitter_action_request_q",
exchange="worker.mm",
routing_key="splitter.*.request",
)
ch.basic_publish(
exchange="worker.mm",
routing_key="splitter.split.request",
properties=props,
body=json.dumps(
{
"cache_dir": "/home/xy/misc/mtm/cache/youtube/好葉怎樣練習冥想完整教學 動畫講解-NLJcwbpkiJ0",
"action.type": "split",
"partial_duration": 2 * 60,
}
),
)
def __init__(self):
self.conn = BlockingConnection(
ConnectionParameters(host=self.host, port=self.port))
self.channel = self.conn.channel()
self.channel.exchange_declare(exchange=self.exchange, type='topic')
def main():
args = get_arguments()
credentials = None
if args.username and args.password:
credentials = PlainCredentials(args.username, args.password)
parameters = ConnectionParameters(host=args.host,
port=args.port,
credentials=credentials)
connection = BlockingConnection(parameters)
channel = connection.channel()
response = channel.queue_declare(exclusive=True, auto_delete=True)
queue = response.method.queue
channel.queue_bind(exchange=args.exchange,
queue=queue,
routing_key=args.routing_key)
channel.basic_consume(on_message, queue)
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
connection.close()
def connect(self):
""" Connect to broker """
logger.debug("Connecting to AMQP broker {}:{}".format(self.host, self.port))
try:
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
context.verify_mode = ssl.CERT_REQUIRED
context.load_verify_locations(cafile=self.caCertsFile)
context.load_cert_chain(self.certFile, keyfile=self.keyFile)
ssloptions = SSLOptions(context)
except Exception:
ssloptions = None
conn_params = ConnectionParameters(
host=self.host,
port=self.port,
ssl_options=ssloptions
)
if self.connection is not None:
logger.debug("Connect called on {}:{}, but connection already defined. Disconnecting and reconnecting".format(self.host, self.port))
self.disconnect()
self.connection = BlockingConnection(conn_params)
self.channel = self.connection.channel()
logger.debug("Connected to AMQP broker {}:{}".format(self.host, self.port))
def connection_thread(url, results, hide_password=False):
from oslo_config import cfg
from oslo_messaging.transport import TransportURL
from pika import exceptions as pika_exceptions
from pika import URLParameters as PikaUrlParameters
from pika import BlockingConnection as PikaBlockingConnection
try:
parsed_url = TransportURL.parse(cfg.CONF, url)
if hide_password:
url = re.sub(':+[^:@]+@', ':******@', url)
except Exception as e:
results.append({'url': url, 'exception': e})
else:
test_url, driver = parse_test_url(parsed_url)
try:
if driver == 'kombu':
connection = Connection(test_url)
connection.connect()
connection.close()
elif driver == 'pika':
params = PikaUrlParameters(test_url)
params.socket_timeout = 5
conn = PikaBlockingConnection(params)
conn.close()
except (OSError, pika_exceptions.ConnectionClosed):
results.append({'url': url, 'exception': _('Url not reachable')})
except (AccessRefused, pika_exceptions.ProbableAuthenticationError):
results.append({
'url': url,
'exception': _('Credentials incorrect')
})
except Exception as e:
results.append({'url': url, 'exception': force_text(e)})
else:
results.append({'url': url})
class Covid19MQUtils:
MQ_EXCHANGE_TYPE = "direct"
MQ_EXCHANGE_NAME = '1606875806'
MQ_HOST = "152.118.148.95"
MQ_PORT = 5672
MQ_USERNAME = "******"
MQ_PASSWORD = "******"
MQ_VIRTUAL_HOST = "/0806444524"
def __init__(self, routing_key, is_in_production=False):
self.routing_key = routing_key
self.pika_connection = BlockingConnection(
ConnectionParameters(host=self.MQ_HOST,
virtual_host=self.MQ_VIRTUAL_HOST,
port=self.MQ_PORT,
credentials=PlainCredentials(
self.MQ_USERNAME, self.MQ_PASSWORD)))
self.connection_channel = self.pika_connection.channel()
self.connection_channel.exchange_declare(
exchange=self.MQ_EXCHANGE_NAME,
exchange_type=self.MQ_EXCHANGE_TYPE)
def send_message(self, message):
self.connection_channel.basic_publish(exchange=self.MQ_EXCHANGE_NAME,
routing_key=self.routing_key,
body=message)
def close_connection(self):
self.pika_connection.close()
class TopicClient(object):
host = config["RABBIT_HOST"]
port = config["RABBIT_PORT"]
exchange = config["EXCHANGE"]
def __init__(self):
self.conn = BlockingConnection(ConnectionParameters(host=self.host, port=self.port))
self.channel = self.conn.channel()
self.channel.exchange_declare(exchange=self.exchange, type="topic")
def subscribe_to_topic(self, callback, topic):
result = self.channel.queue_declare(exclusive=True)
queue_name = result.method.queue
self.channel.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=topic)
self.channel.basic_consume(callback, queue=queue_name, no_ack=True)
def publish_to_topic(self, message, topic):
self.channel.basic_publish(exchange=self.exchange, routing_key=topic, body=message)
def run(self):
print "Start something"
try:
self.channel.start_consuming()
except KeyboardInterrupt:
print "Closing"
self.channel.stop_consuming()
self.conn.close()
class Service( object ):
def __init__( self, topics, name, host = 'localhost', verbose = True, port = 5672, user = '', password = '' ):
credentials = PlainCredentials(user, password)
self.connection = BlockingConnection( ConnectionParameters( host, port, vhost, credentials ) )
self.channel = self.connection.channel()
self.channel.exchange_declare( exchange = exchange_name, type = 'topic' )
self.queueID = self.channel.queue_declare( exclusive = True ).method.queue
self.name = name
for topic in topics:
self.channel.queue_bind( exchange = exchange_name, queue = self.queueID, routing_key = topic)
def _handle( self, c, m, p, b ):
routingKey = self.name #'.'.join( [ 'SS', self.name ] )
print routingKey, b
self.channel.basic_publish( exchange = exchange_name, routing_key = routingKey, body = b)
def close( self ):
self.channel.stop_consuming()
print 'done', datetime.datetime.now()
#for key, val in self._deposit.iteritems():
# print key, len( val )
self.connection.close()
def run( self ):
#_callback = lambda c, m, p, d: self._handle( d )
self.channel.basic_consume( self._handle, queue = self.queueID, no_ack = True )
self.channel.start_consuming()
class RabbitMQConn(object):
"""
RabbitMQ client used to manager queues
"""
__metaclass__ = Singleton
def __init__(self):
self.params = ConnectionParameters()
def connect(self):
self.conn = BlockingConnection(parameters=self.params)
self.ch = self.conn.channel()
def remove_queue(self, queue_name):
try:
logging.info('Removing queue: ' + queue_name)
self.ch.queue_purge(queue=queue_name)
self.ch.queue_delete(queue=queue_name)
except (exceptions.ChannelWrongStateError,
exceptions.ChannelClosed) as error:
logging.error('Err: ' + str(error))
def remove_all_queues(self):
queues = [
'knot-fog-message', 'knot-cloud-message', 'knot-control-message'
]
for queue in queues:
self.remove_queue(queue)
def close(self):
if self.ch.is_open:
self.ch.close()
if self.conn.is_open:
self.conn.close()
def send_msg_to_MQ(
msg_data): # Build connection -> build channel -> send message
#建立连接，然后发起通道，然后再发送信息
connection = BlockingConnection(
ConnectionParameters(host=HOST_NAME,
port=HOST_PORT,
virtual_host='/',
credentials=credentials))
channel = connection.channel()
result = channel.queue_declare(
queue='judged') # 声明消息队列，消息将在这个队列传递，如不存在，则创建
"""
data:
msg_data;
@key='TaskID' value->str # 任务ID
@key='studentNumber' value->str # 学号
@key='result' value->str # 代码评判和编译结果
@key='time' value->str # 代码提交的时间
"""
message = json.dumps({
'TaskID': msg_data['TaskID'],
'result': msg_data['result'],
'time': msg_data['time'],
'studentNumber': msg_data['studentNumber']
}) # build msg
channel.basic_publish(exchange='', routing_key='judged',
body=message) # 向队列插入数值 routing_key是队列名
connection.close()
class RabbitBroker:
def __init__(self, config):
self._config = config
self._exchange = config.RABBITMQ_EXCHANGE_NAME
self._init_connection(config)
def produce(self, event):
try:
self._get_channel().basic_publish(exchange=self._exchange,
routing_key='',
body=event)
except AMQPConnectionError:
self._init_connection(self._config)
self._get_channel().basic_publish(exchange=self._exchange,
routing_key='',
body=event)
def _init_connection(self, config):
params = ConnectionParameters(
host=config.RABBITMQ_HOST,
heartbeat_interval=int(config.RABBITMQ_HEARTBEAT_INTERVAL),
blocked_connection_timeout=int(config.RABBITMQ_HEARTBEAT_INTERVAL))
self._connection = BlockingConnection(params)
self._channel = self._connection.channel()
self._channel.exchange_declare(self._exchange, exchange_type='fanout')
def _get_channel(self):
return self._connection.channel()
def reprocess():
# To use this script, fill in the credentials for rabbit. They should be found in app/settings.py
rabbit_url = 'amqp://<user>:<pass>@<host>:<port>/%2f'
connection = BlockingConnection(URLParameters(rabbit_url))
channel = connection.channel()
method, properties, body = channel.basic_get('Seft.Responses.Quarantine')
if method:
try:
print("Recovered quarantine message")
print("Headers:")
print(properties.headers)
# Uncomment if extra information is needed (payload is encrypted so it's not likely to be useful)
# print("Body:")
# print(body)
publisher = QueuePublisher(urls=[rabbit_url],
queue='Seft.Responses')
publisher.publish_message(body, headers=properties.headers)
print("Message successfully reprocessed")
channel.basic_ack(method.delivery_tag)
print("Message ACK")
except PublishMessageError as e:
print(e)
channel.basic_nack(method.delivery_tag)
else:
print('No message found on quarantine queue')
def amq_channel(exchange):
conn = BlockingConnection(ConnectionParameters(get_broker_address()))
ch = conn.channel()
ch.confirm_delivery()
ch.exchange_declare(exchange=exchange, exchange_type='topic', durable=True)
yield ch
conn.close()
def create_connection():
connection = BlockingConnection(connection_parameters)
try:
yield connection
finally:
connection.close()
def send_msg_to_MQ(
msg_data): # Build connection -> build channel -> send message
#建立连接，然后发起通道，然后再发送信息
connection = BlockingConnection(
ConnectionParameters(host=HOST_NAME,
port=HOST_PORT,
virtual_host='/',
credentials=credentials))
channel = connection.channel()
result = channel.queue_declare(
queue='un_judged') # 声明消息队列，消息将在这个队列传递，如不存在，则创建
"""
data:
msg_data;
@key='TaskID' value->str # 自动生成唯一的任务ID (自动生成)
@key='studentNumber' value->str # 学号
@key='code' value->str # 需要评判的代码
@key='time' value->str # 当前的时间 (自动生成)
"""
TID = gen_task_ID(msg_data['studentNumber'])
message = json.dumps({
'TaskID': TID,
'code': msg_data['code'],
'time': current_datetime(),
'studentNumber': msg_data['studentNumber']
}) # build msg
channel.basic_publish(exchange='', routing_key='un_judged',
body=message) # 向队列插入数值 routing_key是队列名
connection.close()
return TID
def __init__(self,
connection_string: str,
queue: str,
tasks=None,
loglevel="WARNING"):
self._queue = queue
self._tasks = tasks or {}
self._logger = logging.getLogger(queue)
stream_handler = logging.StreamHandler()
# TODO: сделать кастомнй форматер чтобы добавлять пробелы (https://stackoverflow.com/questions/6692248/python-logging-string-formatting/22707429)
stream_format = logging.Formatter(
'[NokiMQ] - %(asctime)s [%(name)s] %(levelname)s %(message)s'
)
stream_handler.setFormatter(stream_format)
self._logger.addHandler(stream_handler)
self._logger.setLevel(logging.getLevelName(loglevel))
self._logger.info(f'Connecting to "{connection_string}"')
connection = None
try:
connection = BlockingConnection(URLParameters(connection_string))
except:
self._logger.critical('Connection failed')
quit(-1)
self._logger.info('Connected!')
self._channel = connection.channel()
self._channel.queue_declare(queue)
def rabbit_connect(self):
if 'queue_host' in self.config:
host = self.config['queue_host']
else:
host = 'localhost'
conn = BlockingConnection(ConnectionParameters(host, connection_attempts=3))
self.rabbit = conn.channel()
self.rabbit.queue_declare(queue=RABBIT_QUEUE)
def __init__(self, url, context, marshal, unmarshal):
self._context = context
self.marshal, self.unmarshal = marshal, unmarshal
host = match(r"amqp://([\w\d\.]+)", url).group(1)
connection = BlockingConnection(ConnectionParameters(host))
self._channel = connection.channel()
self._channel.exchange_declare(exchange=EXCHANGE, type='topic')
self._init_docstore()
def _get_channel():
global _channel
assert 'kompaq' in settings.INSTALLED_APPS
if not _channel:
connection = BlockingConnection(URLParameters(settings.KOMPAQ_URL))
_channel = connection.channel()
return _channel
class RabbitInput(object):
input_name = 'rabbitmq'
def __init__(self, host, port=5672, vhost='/', username='******', password='******', queue='default', prefetch=10):
credentials = PlainCredentials(username, password)
self.connection_params = ConnectionParameters(host=host, port=port, virtual_host=vhost, credentials=credentials)
self.queue = queue
self.prefetch = prefetch
def _worker(self, ch, method, properties, body):
# TODO: find out why rabbitmq sucks
if not body:
logger.warning('empty message received from rabbitmq - skipping')
ch.basic_ack(delivery_tag = method.delivery_tag)
return
try:
data = json.loads(body)
except Exception as err:
logger.debug('unable to decode json: %s' % (str(err), ))
else:
for fmt in self.format_modules:
if fmt.bind and self.input_name in fmt.bind:
data = fmt.decode(data)
self.output_threads.write(data)
finally:
ch.basic_ack(delivery_tag = method.delivery_tag)
def handle_input(self):
try:
self.connection = BlockingConnection(self.connection_params)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.queue, durable=True)
self.channel.basic_qos(prefetch_count=self.prefetch)
self.channel.basic_consume(self._worker, queue=self.queue)
self.connection.channel()
self.channel.start_consuming()
except (AMQPConnectionError, socket.error) as err:
raise EveConnectionError(err)
def run(self, format_modules, output_modules):
# Start output threads
self.format_modules = format_modules
self.output_modules = output_modules
self.output_threads = OutputThreads(self.output_modules, self.format_modules)
while True:
try:
self.handle_input()
except EveConnectionError as err:
logger.error('connection error in input handler %s: %r - retrying in 1 second' % (self.input_name, err))
sleep(1)
def auto_inspect(self):
neid = '100000000013414'
inspect_session_id = INSPECT_SESSION_ID
# inspect_msg = Message(channel, properties={
# 'content-type': 'application/json'
# })
# conn = Connection(
# userid='guest', password='******',
# virtual_host='/', host='192.168.35.172',
# port=20555)
items = [{"executive": "kmScript", "ctrlColId": "1000", "function": "LINUX_MAINFRAME_linux_mainframe_cmd_check",
"configneid": "", "param": ""}, {"executive": "kmScript", "ctrlColId": "1005",
"function": "LINUX_LOGICALVOLUME_linux_logicalvolume_cmd_check",
"configneid": "", "param": ""},
{"executive": "kmScript", "ctrlColId": "1009",
"function": "LINUX_SWAPPARTITION_linux_swapparttion_cmd_check", "configneid": "", "param": ""},
{"executive": "kmScript", "ctrlColId": "1010",
"function": "LINUX_VOLUMEGROUP_linux_volumegroup_cmd_check", "configneid": "", "param": ""}]
body = {"msgtype": "request",
"business": "kmCheckScript",
"body":
{
"neid": "900000012103258",
"inspectSessionId": "{0}".format(inspect_session_id),
"items": items
},
"replyto": "itm.tom"
}
# body = {"msgtype":"request",
# "business":"cmdCtrl",
# "body":
# {
# "neid":"{0}".format(neid),
# "inspectSessionId":"{0}".format(inspect_session_id),
# "items": "[{0}]".format(items)
# }
# }
conn = BlockingConnection(self.params)
channel = conn.channel()
for i in xrange(10):
result = channel.basic_publish(exchange=AMQ_EXCHANGE, routing_key=AMQ_ROUTINGKEY, body=json.dumps(body),
properties=self.properties)
if result:
logging.info('delivery comfirmed')
logging.info('publish result: {0}'.format(result))
else:
logging.info('delivery not confirmed')
# time.sleep(40)
channel.close()
conn.close()
def rabbit_connect(self, callback):
conn = BlockingConnection(ConnectionParameters('localhost'))
self.rabbit = conn.channel()
self.rabbit.queue_declare(queue=RABBIT_QUEUE)
self.rabbit.basic_consume(
callback,
queue=RABBIT_QUEUE,
no_ack=True
)
print "connected"
def work_queue():
message = ' '.join(sys.argv[1:]) or "Hello World!"
connection = BlockingConnection(ConnectionParameters('localhost'))
channel = connection.channel()
queue_name = 'work_queue'
# create a work queue and send a message directly to it, bypassing the exchange
channel.queue_declare(queue='work_queue', durable=True)
channel.basic_publish(exchange='', routing_key='work_queue', body=message, properties=BasicProperties(delivery_mode=2))
print " [x] Sent '%s'" % (message)
connection.close()
def post(self, message: str, queue_name: str):
"""
Posts the given message to a queue with the given name via the message broker's default exchange.
:param message: the message to post
:param queue_name: the name of the queue to post to
"""
connection = BlockingConnection(self._connection_parameters)
try:
channel = connection.channel()
channel.basic_publish(exchange="", routing_key=queue_name, body=message)
finally:
connection.close()
class Service( object ):
def __init__( self, host = 'localhost', port = 5672, user = '', password = '', vhost = '/', routingKey = ''):
credentials = PlainCredentials( user, password )
self._connection = BlockingConnection( ConnectionParameters( host, port, vhost, credentials ) )
#self._connection = SelectConnection( ConnectionParameters( host, port, vhost, credentials ) )
self._channel = self._connection.channel()
self._channel.exchange_declare( exchange = exchange_name, type = 'topic' )
self.rkey = routingKey
def close( self ):
self._connection.close()
def run( self ):
#message = raw_input("Message : ")
while True:
message = """
XKRX-CS-KR-000252,13:30:48.023942,7,290.9,123.19,90.82,79.62,937.15
XKRX-CS-KR-000253,13:30:48.024171,7,28.84,93.29,67.13,234.64,149.7
XKRX-CS-KR-000254,13:30:48.024337,7,248.17,118.49,1489.54,118.45,117.42
XKRX-CS-KR-000255,13:30:48.024497,7,70.67,170.82,65.45,152.11,420.7
XKRX-CS-KR-000256,13:30:48.034801,7,160.74,82.36,260.87,104.42,384.35
XKRX-CS-KR-000257,13:30:48.034973,7,123.39,150.31,60.78,201.21,181.55
XKRX-CS-KR-000100,13:30:48.035137,8,166.66,87.45,252.83,82.03,44.02
XKRX-CS-KR-000101,13:30:48.045434,8,114.86,1023.0,37.92,65.76,61.82
XKRX-CS-KR-000102,13:30:48.045586,8,159.16,97.96,60.07,75.29,690.15
XKRX-CS-KR-000103,13:30:48.045730,8,23.52,133.91,44.0,107.83,533.96
XKRX-CS-KR-000104,13:30:48.045901,8,76.62,274.25,166.57,116.48,149.1
XKRX-CS-KR-000250,13:30:48.056203,8,105.32,254.87,158.97,21.0,59.72
XKRX-CS-KR-000251,13:30:48.056364,8,192.7,226.26,76.02,72.7,40.53
XKRX-CS-KR-000252,13:30:48.056520,8,138.58,138.76,89.68,41.78,175.83
XKRX-CS-KR-000253,13:30:48.066883,8,88.67,41.84,126.81,222.26,8.98
XKRX-CS-KR-000254,13:30:48.067103,8,156.14,126.11,46.24,24.03,57.94
XKRX-CS-KR-000255,13:30:48.067259,8,136.01,35.25,25.29,275.88,50.33
XKRX-CS-KR-000256,13:30:48.067416,8,136.89,10.51,197.03,200.62,238.65
XKRX-CS-KR-000257,13:30:48.077776,8,47.36,41.77,101.75,105.99,64.56
XKRX-CS-KR-000100,13:30:48.078006,9,26.76,231.9,104.19,117.87,24.69
XKRX-CS-KR-000101,13:30:48.078187,9,57.14,84.92,73.62,33.72,47.86
XKRX-CS-KR-000102,13:30:48.088561,9,21.85,120.6,538.69,58.24,1685.93
XKRX-CS-KR-000103,13:30:48.088819,9,450.32,417.01,210.68,121.41,27.18
XKRX-CS-KR-000104,13:30:48.088998,9,80.61,69.15,132.51,98.67,226.2
XKRX-CS-KR-000250,13:30:48.089161,9,107.44,11.22,80.1,85.93,125.1
XKRX-CS-KR-000251,13:30:48.099518,9,43.86,51.79,282.43,101.35,946.29
XKRX-CS-KR-000252,13:30:48.099705,9,170.75,242.6,74.15,323.43,28.48
XKRX-CS-KR-000253,13:30:48.099871,9,53.27,36.47,81.75,50.96,46.73
XKRX-CS-KR-000254,13:30:48.110195,9,136.93,17.66,77.64,253.57,66.8
XKRX-CS-KR-000255,13:30:48.110408,9,65.49,72.59,39.59,63.07,74.31
XKRX-CS-KR-000256,13:30:48.110575,9,63.16,44.29,36.04,119.36,21.78
XKRX-CS-KR-000257,13:30:48.110733,9,125.17,54.65,374.91,219.27,136.63
"""
self._channel.basic_publish( exchange = exchange_name, routing_key = self.rkey, body = message )
print 'Done', datetime.datetime.now(), ", Message :", message
self.close()
def get_mg_score_for_url(ch, method, properties, body):
""" take a URL and return the overall marketing grader score """
company_data = json.loads(body)
company_url = company_data["url"]
print " [x] Received %r" % (company_url)
if company_url == None:
print " [-] no url to lookup"
company_data["mg"] = "no url to lookup"
else:
hostname = company_url.split("://")[1] # remove the protocol from the url
init_url = "http://marketing.grader.com/report/init/%s" % hostname
try:
init = requests.get(init_url)
response_json = json.loads(init.text)
report_guid = response_json["report"]["guid"]
except:
print " [-] error initing MG report"
company_data["mg"] = "error initing MG report"
if not company_data.get("mg", False): # if we haven't already hit an error, try fetching the partial report
while True:
partial_url = "http://marketing.grader.com/report/partial/%s" % (report_guid)
partial = requests.get(partial_url)
response_json = json.loads(partial.text)
if response_json["success"] == True and response_json["report"]["finished"]:
try:
company_data["mg"] = response_json["report"]["finalGrade"]
print " [x] Got final score for %s" % company_data["name"]
break
except KeyError:
company_data["mg"] = "malformed final grade"
print " [x] Error getting final score for %s" % company_data["name"]
break
else:
time.sleep(1) # wait and try polling the API again
print " [-] Not ready yet, trying again"
to_send = json.dumps(company_data)
ch.basic_ack(delivery_tag = method.delivery_tag)
# now that we've gotten the company's MG score, send the data to another worker to get it written to disk
connection = BlockingConnection(ConnectionParameters('localhost'))
channel = connection.channel()
channel.exchange_declare(exchange=EXCHANGE_NAME, type='direct')
channel.basic_publish(exchange=EXCHANGE_NAME, routing_key='write', body=to_send)
def rabbit_connect(self, callback):
if 'queue_host' in self.config:
host = self.config['queue_host']
else:
host = 'localhost'
conn = BlockingConnection(ConnectionParameters(host, connection_attempts=3))
self.rabbit = conn.channel()
self.rabbit.queue_declare(queue=RABBIT_QUEUE)
self.rabbit.basic_consume(
callback,
queue=RABBIT_QUEUE,
no_ack=True
)
def fanout_exchange():
message = ' '.join(sys.argv[1:]) or "Hello World!"
connection = BlockingConnection(ConnectionParameters('localhost'))
channel = connection.channel()
exchange_name = 'analytics'
# create a fanout exchange
channel.exchange_declare(exchange=exchange_name, type='fanout')
# send a task
channel.basic_publish(exchange=exchange_name, routing_key='', body=message)
print " [x] Sent '%s'" % (message)
connection.close()
def fanout_exchange():
connection = BlockingConnection(ConnectionParameters("localhost"))
channel = connection.channel()
# connect to exchange and queue
channel.exchange_declare(exchange="analytics", type="fanout")
result = channel.queue_declare(exclusive=True)
queue_name = result.method.queue
exchange_name = "analytics"
# bind to the queue and start consuming
channel.queue_bind(exchange=exchange_name, queue=queue_name)
channel.basic_consume(callback, queue=queue_name)
print " [*] Waiting for messages. To exit press CTRL+C"
channel.start_consuming()
def direct_exchange():
severity = sys.argv[1]
message = ' '.join(sys.argv[2:]) or "Hello World!"
message = severity + ": " + message
connection = BlockingConnection(ConnectionParameters('localhost'))
channel = connection.channel()
# create a direct exchange
channel.exchange_declare(exchange='direct_logs', type='direct')
# send a task
channel.basic_publish(exchange='direct_logs', routing_key=severity, body=message)
print " [x] Sent '%s'" % (message)
connection.close()
class BlockingChannel:
"""Provides raw blocking Pika channel object for given connection id.
The created connection is closed automatically after 'with' statement has
been closed.
Usage::
from collective.zamqp.connection import BlockingChannel
with BlockingChannel(connection_id) as channel:
frame = channel.declare_queue(auto_delete=True)
# ...
Refer to Pika's API on how to use the raw channel-object. E.g. you
could use it to declare a channel to retrieve the amount of messages
still waiting on that channel.
"""
def __init__(self, connection_id, timeout=60):
# Look up the given connection
connection = getUtility(IBrokerConnection, name=connection_id)
# Prepare a new one-shot blocking connection
credentials = PlainCredentials(
connection.username, connection.password, erase_on_connect=False)
parameters = ConnectionParameters(
connection.hostname, connection.port, connection.virtual_host,
credentials=credentials, heartbeat=True)
# AMQP-heartbeat timeout must be set manually due to bug in pika 0.9.5:
if parameters.heartbeat:
parameters.heartbeat = timeout
# Create the connection and reset channel
self.connection = BlockingConnection(parameters=parameters)
self.channel = None
def __enter__(self):
# Open and return a new channel
self.channel = self.connection.channel()
return self.channel
def __exit__(self, type, value, traceback):
from pika import spec
self.connection.callbacks.add(
self.channel.channel_number,
spec.Channel.CloseOk,
self.channel.transport._on_rpc_complete
)
self.channel.close()
self.connection.close()
class MyClient(object):
def __init__(self):
self.conn = BlockingConnection(ConnectionParameters(host="localhost"))
self.chan = self.conn.channel()
def handler(self, ch, method, props, body):
ch.basic_ack(delivery_tag = method.delivery_tag)
ch.stop_consuming()
def reverse(self, s):
self.chan.queue_declare(queue="request", durable=True)
self.chan.basic_publish(
exchange="",
routing_key="request",
body=s,
properties = BasicProperties(
delivery_mode = 2
)
)
self.chan.queue_declare(queue="response", durable=True)
self.chan.basic_consume(
consumer_callback = self.handler,
queue = "response"
)
self.chan.start_consuming()
def start(self, exchange):
self.exchange = exchange
self.connection = BlockingConnection()
self.connection.set_backpressure_multiplier(self.props.backpressure)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=exchange,
durable=True, exclusive=False, auto_delete=False)
def work_queue():
# http://www.rabbitmq.com/tutorials/tutorial-two-python.html
connection = BlockingConnection(ConnectionParameters("localhost"))
channel = connection.channel()
queue_name = "work_queue"
# don't send tasks to this worker until it's ready, keep them in the queue for other workers
# THIS IS IMPORTANT, otherwise adding more workers won't help anything cause
# all msgs will already have been sent to the currently 'busy' worker(s)
channel.basic_qos(prefetch_count=1)
# create the work queue and start consuming
channel.queue_declare(queue=queue_name, durable=True)
channel.basic_consume(callback, queue=queue_name)
print " [*] Waiting for messages. To exit press CTRL+C"
channel.start_consuming()
def main(a, b):
connection = BlockingConnection(ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='calc', durable=True)
message = "{}+{}".format(a, b)
channel.basic_publish(exchange='',
routing_key='calc',
body=message,
properties=BasicProperties(
delivery_mode=2, # make message persistent
))
print " [x] Sent {}".format(message)
class MyServer(object):
def __init__(self):
self.conn = BlockingConnection(ConnectionParameters(host="localhost"))
self.chan = self.conn.channel()
def handler(self, ch, method, props, body):
print body
nums = len(body)
sleep(nums)
ch.queue_declare(queue="response", durable=True)
ch.basic_publish(
exchange = "",
routing_key = "response",
body = body[::-1],
properties = BasicProperties(
delivery_mode = 2
)
)
ch.basic_ack(delivery_tag=method.delivery_tag)
def start(self):
self.chan.queue_declare(queue="request", durable=True)
self.chan.basic_consume(consumer_callback=self.handler, queue="request")
print "开始监听..."
self.chan.start_consuming()
def main(full=False):
""" bootstraps the processing by getting the list of 5000 companies and queuing them to be processed """
if full == False: # limit the number of companies for testing the system
companies = get_company_name_and_rank(limit=5, pages=1)
else:
companies = get_company_name_and_rank()
# now that we've got the full list of companies, queue them for processing
connection = BlockingConnection(ConnectionParameters('localhost'))
channel = connection.channel()
channel.exchange_declare(exchange=EXCHANGE_NAME, type='direct')
for company in companies: # send company names and ranks to be worked on asychronously
to_send = json.dumps(company)
channel.basic_publish(exchange=EXCHANGE_NAME, routing_key="url", body=to_send)
print " [x] Sent '%s'" % (company)
def open_channel(self, exchange_name, exchange_type):
logger.debug("Connecting to the RabbitMQ server.")
connection = BlockingConnection(parameters=self.connection_parameters)
channel = connection.channel()
try:
channel.exchange_declare(exchange=exchange_name,
exchange_type=exchange_type)
yield channel
except Exception as e:
logger.error(e)
finally:
connection.close()
logger.debug("Connection closed.")
