#coding=utf-8
#Version:python3.5.2
#Tools:Pycharm 2017.1
import pika
conn=pika.BlockingConnection(
pika.ConnectionParameters('localhost')
)
channel=conn.channel()
channel.queue_declare(queue='Q1')
channel.basic_publish(
exchange='',
routing_key='Q1',
body='Hello world!',
properties=pika.BasicProperties(
delivery_mode=2,#消息持久化
)
)
print("Send hello world!")
conn.close()
def task_generate_pdf(pdf_request_dict):
properties = pika.BasicProperties(app_id="docdigit-api",
content_type="application/json")
channel.basic_publish("", "generate_pdf",
str(jsonable_encoder(pdf_request_dict)), properties)
ACTION_TAGS = 'track-tags'
ACTION_INFO = 'track-info'
MESSAGE_FORMAT = '{"action": "%s", "track": %s}'
DEFAULT_TRACK = '{"title":"Pjanoo", "artist":"Eric Prydz"}'
def _parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('--action', default=ACTION_TAGS,
choices=[ACTION_TAGS, ACTION_INFO])
parser.add_argument('track', nargs='?', default=DEFAULT_TRACK)
return parser.parse_args()
if __name__ == '__main__':
args = _parse_args()
host = CONFIG['queue']['host']
queue = CONFIG['queue']['name']
# Set up the connection.
conn = pika.BlockingConnection(pika.ConnectionParameters(host))
channel = conn.channel()
# Creates the queue if it doesn't exist yet.
channel.queue_declare(queue=queue, durable=True)
# Send a message to the queue.
message = MESSAGE_FORMAT % (args.action, args.track)
channel.basic_publish(exchange='', routing_key='lastfm', body=message,
properties=pika.BasicProperties(delivery_mode=2))
print "Sent message %r" % message
# Closing the connection flushes all the messages.
conn.close()
def publish(method, body):
properties = pika.BasicProperties(method)
channel.basic_publish(exchange='',
routing_key='admin',
body=json.dumps(body),
properties=properties)
#!/usr/bin/env python
import pika
from flask import json
import time
import sys
queue_name = "8a6084dc970f4cbfb49e44eb02c7659f"
# establish connection with rabbitmq server
connection = pika.BlockingConnection(pika.ConnectionParameters(host="localhost"))
channel = connection.channel()
channel.queue_declare(queue=queue_name, durable=True) # create a queue
control_signal = {"u": int(sys.argv[1])}
channel.basic_publish("", routing_key=queue_name,
body=json.dumps({"control": control_signal}),
properties=pika.BasicProperties(content_type="application/json"))
print("control signal sent!")
connection.close()
def run(self, experimentId, trialsCount, trialRepetitionsCount):
mqConn, mqCh = initPika()
mqCh.exchange_declare(exchange='mlAgentCommands',
exchange_type='fanout')
mqCh.queue_purge(queue='workerAssignments')
mqCh.queue_purge(queue='workerResponses')
def sendMLAgentCommand(cmd, data={}):
msg = dict(data)
msg['command'] = cmd
msg['targetInstanceType'] = None
mqCh.basic_publish(
exchange='mlAgentCommands',
routing_key='',
body=json.dumps(msg),
properties=pika.BasicProperties(
content_type='application/json'),
)
def sendShutdown():
sendMLAgentCommand('stopService')
def sendStart():
sendMLAgentCommand(
'startService', {
'count': self.workerCountPerInstance,
'args': ['/usr/bin/python3', self.workerSourceFile]
})
def handleResponse(ch, method, properties, body):
msg = json.loads(body)
experimentId = msg['experimentId']
trialId = msg['trialId']
repetition = msg['repetition']
status = msg['status']
results = msg['results']
trialKey = (experimentId, trialId, repetition)
if status == 'accepted':
try:
self.trialsPending.remove(trialKey)
self.trialsInProcessing.add(trialKey)
except KeyError:
logging.warning(
f'Dropping {status} response for {experimentId}:{trialId}:{repetition}'
)
elif status == 'completed' or status == 'error':
try:
self.trialsInProcessing.remove(trialKey)
if status == 'completed':
logging.info(
f'Trial {experimentId}:{trialId}:{repetition} finished with status {status}. Results: train loss = {results["trainLoss"]}, train acc = {results["trainAccuracy"] * 100}, val loss = {results["valLoss"]}, val acc = {results["valAccuracy"] * 100}'
)
else:
logging.info(
f'Trial {experimentId}:{trialId}:{repetition} finished with status {status}.'
)
except KeyError:
logging.warning(
f'Dropping {status} response for {experimentId}:{trialId}:{repetition}'
)
def sigIntHandler(sig, frame):
logging.warning(
'SIGNINT detected. Sending shutdown to all workers and exiting...'
)
sendShutdown()
time.sleep(3)
mqCh.queue_purge(queue='workerAssignments')
mqConn.close()
sys.exit(0)
signal.signal(signal.SIGINT, sigIntHandler)
logging.info('Shutting down dangling workers...')
sendShutdown()
time.sleep(3)
logging.info('Starting workers ...')
sendStart()
mqCh.basic_consume(queue='workerResponses',
auto_ack=True,
on_message_callback=handleResponse)
for trialId in range(trialsCount):
logging.info(f'Starting suggestConfig')
trialConfig = self.suggestConfig()
logging.info(f'Received trialConfig from suggestConfig')
logging.info(f'Starting with trial {experimentId}:{trialId}')
logging.debug(trialConfig)
resultsDir = resultsBaseDir / self.resultsSubDir / str(
experimentId) / str(trialId)
resultsDir.mkdir(parents=True, exist_ok=True)
trialTemplate = {
'experimentId': experimentId,
'trialId': trialId,
'config': trialConfig
}
with open(resultsDir / f'trial.json', 'wt') as trialFile:
json.dump(trialTemplate, trialFile, indent=4)
for repetition in range(trialRepetitionsCount):
while len(self.trialsPending) > 0:
logging.debug('Ensuring that services are running')
sendStart()
mqConn.process_data_events(time_limit=10)
logging.debug(
f'Submitting trial {experimentId}:{trialId}:{repetition}')
trial = dict(trialTemplate)
trial['repetition'] = repetition
self.trialsPending.add((experimentId, trialId, repetition))
mqCh.basic_publish(
exchange='',
routing_key='workerAssignments',
body=json.dumps(trial),
properties=pika.BasicProperties(
content_type='application/json',
delivery_mode=2, # make message persistent
),
)
mqConn.process_data_events()
while len(self.trialsPending) > 0 or len(self.trialsInProcessing) > 0:
mqConn.process_data_events()
sendShutdown()
mqConn.close()
def call_elect(self):
self.channel.basic_publish(exchange='fanout_start_urls',
properties=pika.BasicProperties(
correlation_id='elect', ),
routing_key='',
body=APP_CONF['config']['localhost'])
write(json.dumps(keys), fileadd)
return json.dumps({'response_code': '250'}) #成功
def on_request(ch, method, props, body):
try:
keys = json.loads(body)
print keys
fileadd = keys['fileadd']
ipadd = keys['ipadd']
username = keys['username']
password = keys['password']
response = do_command(fileadd, ipadd, username, password)
print "拯救地球 !!!"
except Exception, e:
print "[Error]: ", e
response = json.dumps({'response_code': '1748'}) #操作错
ch.basic_publish(exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(correlation_id=\
props.correlation_id),
body=str(response))
ch.basic_ack(delivery_tag=method.delivery_tag)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(on_request, queue='key_queue')
print " [x] Awaiting RPC requests"
channel.start_consuming()
channel.basic_consume(queue=info_queue, on_message_callback=on_info, auto_ack=True)
def consumer_routine():
try:
channel.start_consuming()
except KeyboardInterrupt:
pass
Thread(target=consumer_routine, daemon=True).start()
try:
for order_num in count(1):
service_id = input()
routing_key = service_id
order_id = agency_id + '-' + str(order_num)
print(colored(f'Agency ID: {agency_id}\nOrder ID: {order_id}\nService ID: {service_id}', 'cyan'))
channel.basic_publish(
exchange='order',
routing_key=routing_key,
properties=pika.BasicProperties(
reply_to=f'agency.{agency_id}',
correlation_id=order_id,
delivery_mode=2),
body=f'Agency {agency_id} places order {order_id} for service {service_id}')
except KeyboardInterrupt:
pass
# no connection.close() here because there is channel.start_consuming() in another thread
import pika
import sys
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
# RabbitMQ has priority queue implementation in the core as of version 3.5.0.
# You can declare priority queues using the x-max-priority argument.
# This argument should be an integer indicating the maximum priority the queue should support.
args = {'x-max-priority': 10}
channel.queue_declare(queue='priority_queue', durable=True, arguments=args)
if len(sys.argv) != 3:
print 'Usage: %s message priority' % sys.argv[0]
sys.exit(1)
message = sys.argv[1]
priority = int(sys.argv[2])
# publish prioritised messages using the priority field of basic.properties.
# Larger numbers indicate higher priority.
channel.basic_publish(exchange='',
routing_key='priority_queue',
body=message,
properties=pika.BasicProperties(delivery_mode=2,
priority=priority))
print '[x] Sent %r' % message
connection.close()
def _wrapper(*args, **kwargs):
logger.debug("request \'{}\'.*args: {}".format(
on_request_callback.__name__, args))
# process request arguments
deliver = args[1]
properties = args[2]
message = args[3]
# convert message body to string
# output.decode('string-escape').strip('"')
if isinstance(message, bytes):
message = message.decode("utf-8")
if not isinstance(message, str):
logger.warning(
"_profiler_wrapper_request, type: {}, body: {}".format(
type(message), message))
logger.debug("Starting request \'{}\'".format(
on_request_callback.__name__))
# response = on_request_callback(*args, **kwargs)
start = time.time()
rpc_exception = None
try:
response = {
# message is text, it should be converted in dictionary at request func
"result":
on_request_callback(exchange=deliver.exchange,
routing_key=deliver.routing_key,
headers=properties.headers,
body=message),
}
except Exception as e:
logger.warning(
"Exception in request \'{}\', routing_key: {}\n{}".format(
on_request_callback.__name__, deliver.routing_key,
traceback.format_exc()))
rpc_exception = e
response = {"error": utils._ensure_utf8(rpc_exception)}
elapsed = time.time() - start
logger.debug('Request \'{}\' finished. Time elapsed: {}'.format(
on_request_callback.__name__, elapsed))
# sending response back
channel = self._callbacks[on_request_callback].channel
exchange = self._callbacks[on_request_callback].exchange
routing_key = properties.reply_to
logger.debug(
'Sending RPC response to routing key: {}'.format(routing_key))
try:
publish_result = channel.basic_publish(
exchange=exchange,
routing_key=routing_key,
properties=pika.BasicProperties(
correlation_id=properties.correlation_id,
content_encoding='utf-8',
content_type='application/json'),
body=json.dumps(response,
ensure_ascii=False).encode('utf8'),
mandatory=True)
if not publish_result:
raise amqppy.PublishNotRouted(
"Request response was not routed")
except amqppy.PublishNotRouted:
# don't raise it
logger.warning(
"RPC response it has not been published, it might be due to a response waiting timeout"
)
except Exception:
logger.error(
'Exception on publish message to routing_key: {}. Exception message: {}'
.format(routing_key, traceback.format_exc()))
logger.debug('RPC response sent.')
# throw exception to the rpc server/consumer
if rpc_exception:
raise rpc_exception
#coding:utf-8
__author__ = 'xuefeng'
"""
rpm -ivh http://dl.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm
yum -y install erlang
yum -y install rabbitmq-server
service rabbitmq-server start/stop
"""
import pika
conn = pika.BlockingConnection(pika.ConnectionParameters(host='192.168.1.12'))
chan = conn.channel()
chan.queue_declare(queue="fm107.8",durable=True) #声明一个队列，durable=True表示持久化存储
chan.basic_publish(exchange='',
routing_key="fm107.8",
body="Hello world!",
properties=pika.BasicProperties(delivery_mode=2) #代表此条消息持久化
)
print "[x] Sent 'Hello world!'"
conn.close()
def callback(ch, method, properties, body):
# for converting str to dict
cam = literal_eval(body)
# should go through checking here
# initialize 'exist' and 'active' to false
exist = 'unchecked'
active = 'unchecked'
# camera active checking
# initialize duration for checking to run (if checking ran too long, it raises error)
duration = 10
cam = check_active(cam, duration)
active = cam['is_active_image']
# camera_exist checking
# this fucntion should check if a new camera exist in db.
# the returned value is None if already exist
# returning a camera without camera id means that we add new camera
# returning a camera with camera id means that we update old camera
res = check_exist(**cam)
# if returning an old camera
if len(res) > 1:
op = 'error'
cam = None
exist = True
elif len(res) == 1 and res[0].hasKey('camera_ID') is True:
op = 'write' # combine add and update to one fucntion
cam = res[0]
exist = True
elif len(res) == 1 and res[0].hasKey('camera_ID') is False:
# new camera that does not exist on db
# camera health checkng by retrieving images
exist = False
active = False
if active is True:
op = 'write'
else:
op = 'ignore'
else:
exist = False
op = 'ignore'
# after the checking, send processed result to another queue
result = {
'health': {
'exist': exist,
'active': active,
},
'cam': cam,
'op': op,
'cat': 'new'
}
result_connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
result_channel = result_connection.channel()
result_channel.queue_declare(queue='v1-2', durable=True)
channel.basic_publish(
exchange='',
routing_key='v1-2',
body=str(result),
properties=pika.BasicProperties(
delivery_mode=2, # make message persistent
))
print(" [x] Done")
ch.basic_ack(delivery_tag=method.delivery_tag)
__author__ = "xiaoyu hao"
import pika
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
# 声明一个管道
channel = connection.channel()
#声明queue
channel.queue_declare(queue='hello', durable=True) #durable=True 让队列持久化
channel.basic_publish(
exchange='',
routing_key='hello',
body='Hello World!',
properties=pika.BasicProperties(
delivery_mode=2, #消息持久化，进程挂掉消息不会丢失
))
print("[x] Sent 'Hello World!'")
connection.close()
# import pika
#
# queue_name = 'scrape'
# connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost')) # 连接rabbitmq
# channel = connection.channel() # 频道对象
# channel.queue_declare(queue=queue_name) # 声明一个队列，队列名叫scrape
# channel.basic_publish(exchange='', routing_key=queue_name, body=b'test') # routing_key是队列名，body是消息
# 优先级队列
import pika
queue_name = 'scrape1'
max_priority = 100
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost')) # 连接rabbitmq
channel = connection.channel() # 频道对象
# durable 队列持久化
channel.queue_declare(queue=queue_name,
arguments={'x-max-priority': max_priority},
durable=True)
while True:
data, priority = input().split()
channel.basic_publish(exchange='',
routing_key=queue_name,
properties=pika.BasicProperties(
priority=int(priority), ),
body=data)
print(f'put {data}')
# )
# print('waiting...')
# channel.start_consuming()
import pika
import sys
import time
# 远程rabbitmq服务的配置信息
username = '******'
pwd = 'admin'
# ip_addr = '10.1.7.7'
ip_addr = '127.0.0.1'
port_num = 5672
# 信息队列服务的连接和队列的创建
credentials = pika.PlainCredentials(username,pwd)
connection = pika.BlockingConnection(pika.ConnectionParameters(ip_addr, port_num, '/', credentials))
channel = connection.channel()
channel.queue_declare(queue='balance', durable=True)
message_str = 'Hello World!'
for i in range(1000000000):
channel.basic_publish(
exchange='',
routing_key='balance', # 写明将信息发送给队列balance
body=message_str, # 要发送的信息
properties=pika.BasicProperties(delivery_mode=2, ) # 设置消息持久化（持久化第二部），将要发送的消息的属性标记为2，表示该信息持久化
) # 向消息队列发送一条信息
print(" [%s] Sent 'Hello World'"%i)
connection.close()
def services_running(self, target):
"""
Check all services are running
:param target: Target to check
:return: Boolean
"""
try:
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4()))
value = str(time.time())
if target == 'framework':
# Volatile
self.log_message(target, 'Testing volatile store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
try:
logging.disable(logging.WARNING)
from ovs.extensions.storage.volatilefactory import VolatileFactory
VolatileFactory.store = None
volatile = VolatileFactory.get_client()
volatile.set(key, value)
if volatile.get(key) == value:
volatile.delete(key)
break
volatile.delete(key)
finally:
logging.disable(logging.NOTSET)
except Exception as message:
self.log_message(
target,
' Error during volatile store test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(target,
' Volatile store not working correctly',
2)
return False
self.log_message(
target,
' Volatile store OK after {0} tries'.format(tries), 0)
# Persistent
self.log_message(target, 'Testing persistent store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
try:
logging.disable(logging.WARNING)
persistent = PersistentFactory.get_client()
persistent.set(key, value)
if persistent.get(key) == value:
persistent.delete(key)
break
persistent.delete(key)
finally:
logging.disable(logging.NOTSET)
except Exception as message:
self.log_message(
target,
' Error during persistent store test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(
target, ' Persistent store not working correctly', 2)
return False
self.log_message(
target,
' Persistent store OK after {0} tries'.format(tries), 0)
if target == 'volumedriver':
# Arakoon, voldrv cluster
self.log_message(target, 'Testing arakoon (voldrv)...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.db.etcd.configuration import EtcdConfiguration
from ovs.extensions.storage.persistent.pyrakoonstore import PyrakoonStore
cluster_name = str(
EtcdConfiguration.get(
'/ovs/framework/arakoon_clusters|voldrv'))
client = PyrakoonStore(cluster=cluster_name)
client.set(key, value)
if client.get(key) == value:
client.delete(key)
break
client.delete(key)
except Exception as message:
self.log_message(
target,
' Error during arakoon (voldrv) test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(
target, ' Arakoon (voldrv) not working correctly', 2)
return False
self.log_message(target, ' Arakoon (voldrv) OK', 0)
if target in ['framework', 'volumedriver']:
# RabbitMQ
self.log_message(target, 'Test rabbitMQ...', 0)
import pika
from ovs.extensions.db.etcd.configuration import EtcdConfiguration
rmq_servers = EtcdConfiguration.get(
'/ovs/framework/messagequeue|endpoints')
good_node = False
for server in rmq_servers:
try:
connection_string = '{0}://{1}:{2}@{3}/%2F'.format(
EtcdConfiguration.get(
'/ovs/framework/messagequeue|protocol'),
EtcdConfiguration.get(
'/ovs/framework/messagequeue|user'),
EtcdConfiguration.get(
'/ovs/framework/messagequeue|password'),
server)
connection = pika.BlockingConnection(
pika.URLParameters(connection_string))
channel = connection.channel()
channel.basic_publish(
'', 'ovs-watcher', str(time.time()),
pika.BasicProperties(content_type='text/plain',
delivery_mode=1))
connection.close()
good_node = True
except Exception as message:
self.log_message(
target,
' Error during rabbitMQ test on node {0}: {1}'.
format(server, message), 2)
if good_node is False:
self.log_message(
target, ' No working rabbitMQ node could be found', 2)
return False
self.log_message(target, ' RabbitMQ test OK', 0)
self.log_message(target, 'All tests OK', 0)
return True
except Exception as ex:
self.log_message(target, 'Unexpected exception: {0}'.format(ex), 2)
return False
import sys
'''
Отправляет одно сообщение в очередь
'''
# Подключаемся к брокеру на локальном хосте. Для подключения к брокеру на другой машине нужно указать её ip вместо localhost
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='127.0.0.1'))
channel = connection.channel()
# создаёт точку обмена
channel.exchange_declare(exchange='direct_logs', exchange_type='direct')
# Создаём очередь. Если послать сообщение в несуществующую очередь, то рэбит его проигнорирует (создаёт очередь с этим именем только один раз), urable=True делает очередь устойчивой
channel.queue_declare(queue='one', durable=True)
message = sys.argv[1]
# Отправляем сообщение в очередь
channel.basic_publish(
exchange='direct_logs',
routing_key='black',
body=message,
properties=pika.BasicProperties(
# позволяет сделать сообщения устойчивыми
delivery_mode=2, ))
print('sended message')
# Закрыли соединение с брокером
connection.close()
def call_notify(self):
self.channel.basic_publish(exchange='fanout_start_urls',
properties=pika.BasicProperties(
correlation_id='notify', ),
routing_key='',
body='')
def send_msg(self, msg, queue, priority):
properties = pika.BasicProperties(delivery_mode=2, priority=priority)
self.channel.basic_publish(exchange=self.exchange,
routing_key=queue,
body=msg,
properties=properties)
def services_running(self, target):
"""
Check all services are running
:param target: Target to check
:return: Boolean
"""
try:
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4()))
value = str(time.time())
if target in ['config', 'framework']:
self.log_message(target, 'Testing configuration store...', 0)
from ovs.extensions.generic.configuration import Configuration
try:
Configuration.list('/')
except Exception as ex:
self.log_message(
target,
' Error during configuration store test: {0}'.format(
ex), 2)
return False
from ovs.extensions.db.arakooninstaller import ArakoonInstaller, ArakoonClusterConfig
from ovs_extensions.db.arakoon.pyrakoon.pyrakoon.compat import NoGuarantee
from ovs.extensions.generic.configuration import Configuration
with open(Configuration.CACC_LOCATION) as config_file:
contents = config_file.read()
config = ArakoonClusterConfig(cluster_id='cacc',
load_config=False)
config.read_config(contents=contents)
client = ArakoonInstaller.build_client(config)
contents = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY,
consistency=NoGuarantee())
if Watcher.LOG_CONTENTS != contents:
try:
config.read_config(
contents=contents
) # Validate whether the contents are not corrupt
except Exception as ex:
self.log_message(
target,
' Configuration stored in configuration store seems to be corrupt: {0}'
.format(ex), 2)
return False
temp_filename = '{0}~'.format(Configuration.CACC_LOCATION)
with open(temp_filename, 'w') as config_file:
config_file.write(contents)
config_file.flush()
os.fsync(config_file)
os.rename(temp_filename, Configuration.CACC_LOCATION)
Watcher.LOG_CONTENTS = contents
self.log_message(target, ' Configuration store OK', 0)
if target == 'framework':
# Volatile
self.log_message(target, 'Testing volatile store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
try:
logging.disable(logging.WARNING)
from ovs.extensions.storage.volatilefactory import VolatileFactory
VolatileFactory.store = None
volatile = VolatileFactory.get_client()
volatile.set(key, value)
if volatile.get(key) == value:
volatile.delete(key)
break
volatile.delete(key)
finally:
logging.disable(logging.NOTSET)
except Exception as message:
self.log_message(
target,
' Error during volatile store test: {0}'.format(
message), 2)
key = 'ovs-watcher-{0}'.format(str(
uuid.uuid4())) # Get another key
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(target,
' Volatile store not working correctly',
2)
return False
self.log_message(
target,
' Volatile store OK after {0} tries'.format(tries), 0)
# Persistent
self.log_message(target, 'Testing persistent store...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
try:
logging.disable(logging.WARNING)
persistent = PersistentFactory.get_client()
persistent.nop()
break
finally:
logging.disable(logging.NOTSET)
except Exception as message:
self.log_message(
target,
' Error during persistent store test: {0}'.format(
message), 2)
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(
target, ' Persistent store not working correctly', 2)
return False
self.log_message(
target,
' Persistent store OK after {0} tries'.format(tries), 0)
if target == 'volumedriver':
# Arakoon, voldrv cluster
self.log_message(target, 'Testing arakoon (voldrv)...', 0)
max_tries = 5
tries = 0
while tries < max_tries:
try:
from ovs.extensions.generic.configuration import Configuration
from ovs_extensions.storage.persistent.pyrakoonstore import PyrakoonStore
cluster_name = str(
Configuration.get(
'/ovs/framework/arakoon_clusters|voldrv'))
configuration = Configuration.get(
'/ovs/arakoon/{0}/config'.format(cluster_name),
raw=True)
client = PyrakoonStore(cluster=cluster_name,
configuration=configuration)
client.nop()
break
except Exception as message:
self.log_message(
target,
' Error during arakoon (voldrv) test: {0}'.format(
message), 2)
time.sleep(1)
tries += 1
if tries == max_tries:
self.log_message(
target, ' Arakoon (voldrv) not working correctly', 2)
return False
self.log_message(target, ' Arakoon (voldrv) OK', 0)
if target in ['framework', 'volumedriver']:
# RabbitMQ
self.log_message(target, 'Test rabbitMQ...', 0)
import pika
from ovs.extensions.generic.configuration import Configuration
messagequeue = Configuration.get('/ovs/framework/messagequeue')
rmq_servers = messagequeue['endpoints']
good_node = False
for server in rmq_servers:
try:
connection_string = '{0}://{1}:{2}@{3}/%2F'.format(
messagequeue['protocol'], messagequeue['user'],
messagequeue['password'], server)
connection = pika.BlockingConnection(
pika.URLParameters(connection_string))
channel = connection.channel()
channel.basic_publish(
'', 'ovs-watcher', str(time.time()),
pika.BasicProperties(content_type='text/plain',
delivery_mode=1))
connection.close()
good_node = True
except Exception as message:
self.log_message(
target,
' Error during rabbitMQ test on node {0}: {1}'.
format(server, message), 2)
if good_node is False:
self.log_message(
target, ' No working rabbitMQ node could be found', 2)
return False
self.log_message(target, ' RabbitMQ test OK', 0)
self.log_message(target, 'All tests OK', 0)
return True
except Exception as ex:
self.log_message(target, 'Unexpected exception: {0}'.format(ex), 2)
return False
import time, json, pika
creds_broker = pika.PlainCredentials("guest", "rabbitmqguest")
conn_params = pika.ConnectionParameters("137.140.3.151",
virtual_host="/",
credentials=creds_broker)
conn_broker = pika.BlockingConnection(conn_params)
channel = conn_broker.channel()
msg = json.dumps({"client_name": "RPC Client 1.0", "time": time.time()})
result = channel.queue_declare(exclusive=True, auto_delete=True)
msg_props = pika.BasicProperties()
msg_props.reply_to = result.method.queue
channel.basic_publish(body=msg,
exchange="rpc",
properties=msg_props,
routing_key="ping")
print "Sent 'ping' RPC call. Waiting for reply..."
def reply_callback(channel, method, header, body):
"""Receives RPC server replies."""
print "RPC Reply --- " + body
channel.stop_consuming()
channel.basic_consume(reply_callback,
def __publish_raw(self, msg):
self._channel.basic_publish(exchange="", routing_key=self._queue_name,
body=msg,
properties=pika.BasicProperties(
content_type="application/json"))
def on_request(ch, method, props, body):
mylist = []
print("Received in slave from orchestrator %r" % json.loads(body))
sys.stdout.flush()
received_data = json.loads(body)
response_data = 0
if received_data["module"] == "add_user":
for keys, values in received_data.items():
mylist.append(values)
condition = (mylist[0], )
query = """SELECT username,password FROM users WHERE username = ?"""
cursor.execute(query, condition)
result = cursor.fetchall()
connection_db.commit()
print('the cursor.rowcount is %r' % cursor.rowcount)
if len(result) == 0:
print('rowcount is 0')
sys.stdout.flush()
response_data = 201
else:
print('rowcount is not 0')
sys.stdout.flush()
response_data = 400
print("status code to return is %r" % response_data)
sys.stdout.flush()
if received_data["module"] == "list_all_users":
query = """SELECT username FROM users"""
cursor.execute(query)
result = cursor.fetchall()
connection_db.commit()
if len(result) == 0:
# 204 indicates that the request was successfull but there was no content found!
response_data = 204
else:
response_data = result
print("status code to return is %r" % response_data)
sys.stdout.flush()
if received_data["module"] == "delete_user":
print('inside the if condition of delete_data in slave')
sys.stdout.flush()
for keys, values in received_data.items():
mylist.append(values)
condition = (mylist[0], )
query = """SELECT username FROM users WHERE username = ?"""
cursor.execute(query, condition)
result = cursor.fetchall()
connection_db.commit()
print('the cursor.rowcount is %r' % cursor.rowcount)
if len(result) == 0:
print('rowcount is 0')
sys.stdout.flush()
response_data = 400
else:
print('rowcount is not 0')
sys.stdout.flush()
response_data = 200
print("status code to return is %r" % response_data)
sys.stdout.flush()
if received_data["module"] == "upcoming_ride":
mylist = []
for keys, values in received_data.items():
mylist.append(values)
condition1 = mylist[0]
condition2 = mylist[1]
query = """SELECT rowid,username,time FROM ride WHERE source = ? AND destination = ?"""
recordTuple = (condition1, condition2)
cursor.execute(query, recordTuple)
result = cursor.fetchall()
connection_db.commit()
if len(result) == 0:
# 400 is bad request!
response_data = 400
else:
response_data = result
print("status code to return is %r" % response_data)
sys.stdout.flush()
if received_data["module"] == "ride_details":
print('inside ride_details of slave')
sys.stdout.flush()
mylist = []
for keys, values in received_data.items():
mylist.append(values)
condition = mylist[0]
query1 = """SELECT rowid,username,time, source, destination FROM ride WHERE rowid = ?"""
recordTuple = (condition, )
cursor.execute(query1, recordTuple)
result = cursor.fetchall()
connection_db.commit()
print('select from ride table is executed %r' % result)
sys.stdout.flush()
if cursor.rowcount != 0:
query2 = """SELECT name FROM associated_riders WHERE rowid = ?"""
recordTuple = (condition, )
cursor.execute(query2, recordTuple)
result2 = cursor.fetchall()
connection_db.commit()
print('select from associated_riders fetch value is %r' % result2)
sys.stdout.flush()
if cursor.rowcount == 0:
response_data = result
else:
result = result + result2
response_data = result
else:
response_data = 405
if received_data["module"] == "join_ride":
# r = json.dumps(received_data)
print('inside if condition of join_ride in slave')
sys.stdout.flush()
mylist = []
for keys, values in received_data.items():
mylist.append(values)
condition1 = mylist[0]
query = """SELECT rowid FROM ride WHERE rowid =?"""
cursor.execute(query, condition1)
print('selecting rideid query executed in slave')
sys.stdout.flush()
result = cursor.fetchall()
connection_db.commit()
if cursor.rowcount == 0:
# 405 is method not allowed!
response_data = 405
else:
response_data = 200
print("status code to return from slave is %r" % response_data)
sys.stdout.flush()
if received_data["module"] == "rides_count":
query = """SELECT COUNT(*) FROM ride"""
cursor.execute(query)
result = cursor.fetchall()
connection_db.commit()
if len(result) == 0:
response_data = 405
else:
response_data = result
print("status code to return from slave is %r" % response_data)
sys.stdout.flush()
ch.basic_publish(exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(correlation_id = \
props.correlation_id),
body=json.dumps(response_data))
ch.basic_ack(delivery_tag=method.delivery_tag)
print(
'response_data was wrote on the response_queue and returned successfully %r'
% response_data)
# wait for SynQ from master!
channel.exchange_declare(exchange='logs', exchange_type='fanout')
result = channel.queue_declare(queue='', exclusive=True)
queue_name = result.method.queue
channel.queue_bind(exchange='logs', queue=queue_name)
print('Waiting for SynQ')
sys.stdout.flush()
def callback(ch, method, properties, body):
mylist = []
print("%r" % json.loads(body))
sys.stdout.flush()
print(
'The payload from the master is received by slave to write into the database'
)
sys.stdout.flush()
payload_to_update = json.loads(body)
# after getting the payload from master write the data to the
# slaves also for synchronization and to stay updated!
if payload_to_update["module"] == "add_user":
for keys, values in payload_to_update.items():
mylist.append(values)
condition1 = mylist[0]
condition2 = mylist[1]
query = """INSERT INTO users(username,password)VALUES(?,?)"""
recordTuple = (condition1, condition2)
try:
cursor.execute(query, recordTuple)
connection_db.commit()
except sqlite3.IntegrityError:
print("An Exception was Caught!!")
sys.stdout.flush()
print('Slave is updated!')
sys.stdout.flush()
if payload_to_update["module"] == "delete_user":
for keys, values in received_data.items():
mylist.append(values)
condition = mylist[0]
query = """DELETE FROM users WHERE username =?"""
recordTuple = (condition, )
cursor.execute(query, recordTuple)
result = cursor.fetchall()
connection_db.commit()
if len(result) == 0:
response_data = 400
else:
response_data = 200
print("deleted code %r" % response_data)
sys.stdout.flush()
if payload_to_update["module"] == "clear_database":
print('in clear database of slave to update')
sys.stdout.flush()
query = """DELETE FROM users"""
cursor.execute(query)
result = cursor.fetchall()
connection_db.commit()
query1 = """DELETE FROM associated_riders"""
cursor.execute(query1)
connection_db.commit()
result2 = cursor.fetchall()
query2 = """DELETE FROM ride"""
cursor.execute(query2)
result3 = cursor.fetchall()
connection_db.commit()
if len(result) == 0 and len(result2) == 0 and len(result3) == 0:
response_data = 200
else:
response_data = 405
print("status code to return is %r" % response_data)
sys.stdout.flush()
if payload_to_update["module"] == "create_ride":
mylist = []
print('inside if condition of create_ride in slave')
for keys, values in payload_to_update.items():
mylist.append(values)
condition1 = mylist[0]
condition2 = mylist[1]
condition3 = int(mylist[2])
condition4 = int(mylist[3])
query = """INSERT INTO ride(username,time,source,destination)VALUES(?, ?,?,?)"""
recordTuple = (condition1, condition2, condition3, condition4)
cursor.execute(query, recordTuple)
result = cursor.fetchall()
connection_db.commit()
if cursor.rowcount == 0:
response_data = 400
else:
response_data = 200
print("status code to return is %r" % response_data)
sys.stdout.flush()
if payload_to_update["module"] == "join_ride":
print('inside if condition of join_ride in slave to update it!')
sys.stdout.flush()
mylist = []
for keys, values in payload_to_update.items():
mylist.append(values)
condition1 = mylist[0]
condition2 = mylist[1]
query = """INSERT INTO associated_riders(rideid,name)VALUES(?,?)"""
recordTuple = (condition1, condition2)
cursor.execute(query, recordTuple)
print('insert query executed in slave')
sys.stdout.flush()
result = cursor.fetchall()
connection_db.commit()
if len(result) == 0:
response_data = 200
else:
response_data = 405
print('slave is updated!')
sys.stdout.flush()
channel.basic_consume(queue=queue_name,
on_message_callback=callback,
auto_ack=True)
import pika
import sys
"""
new_task.py
Scheduler for tasks being sent to work queues
"""
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue='task_hello', durable=True)
message = ' '.join(sys.argv[1:]) or "Hello World!"
channel.basic_publish(exchange='', routing_key='hello', body=message, properties=pika.BasicProperties(delivery_mode=2,))
print(" [x] Sent {0}".format(message))
connection.close()
callback_queue_name = callback_queue.method.queue
def on_response(ch,method,props,body):
global response
response = body.decode()
if props.correlation_id == corr_id:
print("From RPC Server: {}".format(response))
n = 1
msg = 'hello world'
while n<11:
response = ''
corr_id = str(uuid.uuid4())
channel.basic_publish(exchange='',routing_key='rpc_request',body=msg,properties=pika.BasicProperties(delivery_mode=2,correlation_id=corr_id,\
reply_to=callback_queue_name,))
channel.basic_consume(queue=callback_queue_name,on_message_callback=on_response,auto_ack=True)
while not response:
# 该方法可选填time_limit参数表示最长的阻塞时间. 参数默认为0, 即有消息需要发送或接收时会立即结束阻塞并进行处理.
# 阻塞期间连接不会被消息代理断开, 该方法起到了保活作用.
connection.process_data_events()
# channel.start_consuming()
n += 1
channel.close()
import pika
import ConnectLocal as locale
def consume_response(ch, method, props, body):
# consume on temp queue
print("consume the response!")
print(body)
ch.basic_ack(delivery_tag=method.delivery_tag)
# connect
with locale.do_connect() as channel:
# create (temporary), exclusive anonymous queue
ok_result = channel.queue_declare("",
exclusive=True,
arguments={"x-expires": 30000})
reply_to = ok_result.method.queue
print("queue name: {0}".format(reply_to))
# set message header
properties = pika.BasicProperties(reply_to=reply_to)
# publish
channel.basic_publish("amq.direct",
routing_key="request",
properties=properties,
body="our request waits on {0} ".format(reply_to))
channel.basic_consume(reply_to, consume_response)
channel.start_consuming()
# close
def on_request(self, ch, method, props, body):
print(" I received: {}".format(body))
print("my correlation id is: {}".format(props.correlation_id))
x = body.decode().split("#")
response = ""
if x[0] == "admin":
subprocess.Popen([
'python3', '/root/minion/server_broker.py',
str(x[1]) + "_queue"
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
response = 1
print("The response is equal to: {}".format(response))
elif x[0] == "available_resource_creation":
print(
"***********The Host Node Server Broker -- verify_resource_creation--***********"
)
cpu, ram, disk = system_driver.resource_availability()
response = str(system_driver.get_ip()) + "#" + str(
cpu) + "#" + str(ram) + "#" + str(disk)
elif x[0] == "create":
print(
"***********The Host Node Server Broker -- create_container --***********"
)
system_driver.default_bridge()
print("calling the creation library .....")
response = sdn.create(x[1], x[2], x[3], x[4], x[5], x[6], x[7],
x[8])
print("The response is equal to: {}".format(response))
elif x[0] == "container_resources":
print(
"***********The Host Node Server Broker -- get_container_resources --***********"
)
print("getting the cpu, ram information .....")
cpu, ram = lxc_driver.get_container_resources(x[1])
response = str(
system_driver.get_ip()) + "#" + str(cpu) + "#" + str(ram)
print("The response is equal to: {}".format(response))
elif x[0] == "available_resource_migration":
print(
"***********The Host Node Server Broker -- verify_resource_migration --***********"
)
if str(system_driver.get_ip()) != x[1]:
cpu, ram, disk = system_driver.resource_availability()
response = str(system_driver.get_ip()) + "#" + str(
cpu) + "#" + str(ram) + "#" + str(disk)
else:
response = str(system_driver.get_ip()) + "#" + str(
0) + "#" + str(0) + "#" + str(0)
elif x[0] == "container_image":
print(
"***********The Host Node Server Broker -- container_image --***********"
)
print("getting the image name .....")
response = migration.container_image(x[1])
elif x[0] == "part_migration_check":
print(
"***********The Host Node Server Broker -- part_migration_check --***********"
)
print("searching for partial migration action .....")
if migration.target_container_image(x[1]):
response = migration.partial_migration_preparation(x[1], x[2])
elif x[0] == "migration":
print(
"***********The Host Node Server Broker -- full_migration --***********"
)
print("Full-Migration Process .....")
response = migration.migrate(x[1], x[2], x[3])
elif x[0] == "validate_migration":
print(
"***********The Host Node Server Broker -- validate_migration --***********"
)
print("Validate-Migration Process .....")
response = migration.validate_migration(x[1], x[2])
elif x[0] == "rat_trigger":
print(
"***********The Host Node Server Broker -- rat_trigger --***********"
)
response = rat.container_list()
elif x[0] == "sct_trigger":
print(
"***********The Host Node Server Broker -- sct_trigger --***********"
)
response = sct.container_live_resources()
elif x[0] == "scale_up_cpu_ram":
print(
"***********The Host Node Server Broker -- scale_up_cpu_ram --***********"
)
response = scale_up.scale_up_cpu_ram(x[1], x[2], x[3])
elif x[0] == "scale_up_cpu":
print(
"***********The Host Node Server Broker -- scale_up_cpu --***********"
)
response = scale_up.scale_up_cpu_full(x[1], x[2])
elif x[0] == "scale_up_ram":
print(
"***********The Host Node Server Broker -- scale_up_ram --***********"
)
response = scale_up.scale_up_ram_full(x[1], x[2])
elif x[0] == "container_dashboard_resources":
print(
"***********The Host Node Server Broker -- container_dashboard_resources --***********"
)
container_ip, cpu, ram, disk = lxc_driver.container_dashboard_resources(
x[1])
response = str(container_ip) + "#" + str(cpu) + "#" + str(
ram) + "#" + str(disk)
elif x[0] == "environment_cleaner":
print(
"***********The Host Node Server Broker -- environment_cleaner --***********"
)
subprocess.Popen(
['python3', '/root/minion/environment_cleaner.py'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
response = 1
ch.basic_publish(exchange='',
routing_key=props.reply_to,
properties=pika.BasicProperties(
correlation_id=props.correlation_id),
body=str(response))
ch.basic_ack(delivery_tag=method.delivery_tag)
import sys
import pika
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
# Message durability
# 雖然有ack機制可以確保接收者收到message,但是要是server直接掛掉,
# 消息還是會直接掰掉,所以可以在declare的時候,加上durable=True(worker也要加)
# 所以現在task_queue 這個queue 即使server重啟, 訊息也不會不見
# it may be just saved to cache and not really written to the disk!!
# If you need a stronger guarantee about message persistence
# then you can use publisher confirms.
# channel.queue_declare(queue='task_queue')
channel.queue_declare(queue='task_queue', durable=True)
message = ' '.join(sys.argv[1:]) or "Hello World!"
channel.basic_publish(
exchange='',
routing_key='task_queue',
body=message,
properties=pika.BasicProperties(
delivery_mode=2, # make message persistent
))
print(" [x] Sent %r" % message)
def on_request(self, channel, method, properties, body):
i, genes, fitness, last_location, best_fitness, memory, new_location, additional_parameters, exp_no, algo, dataset = json.loads(
body)
print(additional_parameters)
# If an additional parameter is received as a list, convert to tuple
for param in additional_parameters.keys():
if isinstance(additional_parameters[param], list):
additional_parameters[param] = tuple(additional_parameters[param])
self.algorithm = algo
self.dataset = dataset
if self.dataset == "mnist":
from keras.datasets import mnist
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
elif self.dataset == "cifar10":
from keras.datasets import cifar10
(train_images, train_labels), (test_images, test_labels) = cifar10.load_data()
else:
raise Exception("Currently only mnist and cifar10 datasets are supported")
n = train_images.shape[0]
test_n=test_images.shape[0]
self.input_shape = train_images[0].shape
lb = LabelBinarizer()
lb.fit(range(10))
# selection = random.sample(range(n), 10000) # Use only a subsample
# test_selection = random.sample(range(test_n), 1000)
self.y_train = lb.transform(train_labels)#[selection]) # One-hot encodings
self.y_test = lb.transform(test_labels)#[test_selection])
if len(train_images.shape) < 4:
new_shape = (*train_images.shape, 1)
new_shape_test = (*test_images.shape, 1)
else:
new_shape = train_images.shape
new_shape_test = test_images.shape
x_train = train_images.reshape(new_shape)#[selection]
x_test = test_images.reshape(new_shape_test)#[test_selection]
self.x_train = x_train / 255 # Normalize train data
self.x_test = x_test / 255
(unique_labels, nb_classes) = np.unique(train_labels, return_counts=True)
print(" [.] Evaluating individual {}".format(i))
# print(" ... Genes: {}".format(str(genes)))
# print(" ... Other: {}".format(str(additional_parameters)))
# Run model and return fitness metric
if self.algorithm == "ga":
self.individual = GeneticCnnIndividual
individual = self.individual(self.x_train, self.y_train, genes=genes, **additional_parameters)
fitness = individual.get_fitness()
# Prepare response for master and send it
response = json.dumps([i, fitness])
elif self.algorithm == "csa":
self.individual = CrowIndividual
individual = self.individual(self.gpu, self.x_train, self.y_train, self.x_test, self.y_test, id=i,
space=genes, location=new_location, memory=memory, best_fitness=best_fitness,
fitness=fitness, last_location=last_location, **additional_parameters)
import time
start_time = time.time()
fitness = individual.evaluate_fitness()
training_time = time.time() - start_time
best_fitness = individual.get_best_fitness()
memory = individual.get_memory()
location = individual.get_location()
last_location = individual.get_last_location()
self.system_info["device"]=individual.gpu_device
# Prepare response for master and send it
loss=individual.loss
mae=individual.mae
mse=individual.mse
msle=individual.msle
training_history=individual.training_history
epochs_history=individual.epochs_history
model_json=individual.model_json
response = json.dumps(
[i, last_location, fitness, memory, best_fitness, location, training_time, loss,
mae, mse, msle,training_history,epochs_history,model_json,self.system_info])
else:
raise Exception("Only Genetic Algorithm and Crow Search Algorithm are supported")
channel.basic_publish(
exchange='', routing_key=properties.reply_to,
properties=pika.BasicProperties(correlation_id=properties.correlation_id), body=response
)
channel.basic_ack(delivery_tag=method.delivery_tag)
print("hello")
