def update_celery_config_with_queue_head_ip(queue_head_ip, agent_type):
'''
Method used for updating celery config file. It should have the correct IP
of the queue head node, which should already be running.
Args
queue_head_ip The ip that is going to be written in the celery configuration file
'''
logging.debug("queue_head_ip = {0}".format(queue_head_ip))
with open(CeleryConfig.CONFIG_TEMPLATE_FILENAME,
'r') as celery_config_file:
celery_config_lines = celery_config_file.readlines()
with open(CeleryConfig.get_config_filename(agent_type=agent_type),
'w') as celery_config_file:
for line in celery_config_lines:
if line.strip().startswith('BROKER_URL'):
celery_config_file.write(
'BROKER_URL = "amqp://*****:*****@{0}:5672/"\n'.format(
queue_head_ip))
else:
celery_config_file.write(line)
# Now update the actual Celery app....
# TODO: Doesnt seem to work in GAE until next request comes in to server
tasks.CelerySingleton().configure()
def config_celery_queues(agent_type, instance_types):
exchange = "exchange = Exchange('{0}', type='direct')".format(
CeleryConfig.get_exchange_name(agent_type=agent_type))
logging.debug(exchange)
queue_list = map(
lambda instance_type: "Queue('{0}', exchange, routing_key='{1}')".
format(
CeleryConfig.get_queue_name(agent_type=agent_type,
instance_type=instance_type),
CeleryConfig.get_routing_key_name(agent_type=agent_type,
instance_type=instance_type)),
instance_types)
agent_queue_name = CeleryConfig.get_queue_name(agent_type=agent_type)
agent_routing_key = CeleryConfig.get_routing_key_name(
agent_type=agent_type)
queue_list.insert(
0, "Queue('{0}', exchange, routing_key='{1}')".format(
agent_queue_name, agent_routing_key))
logging.debug(pprint.pformat(queue_list))
queues_string = 'CELERY_QUEUES = ({0})'.format(', '.join(queue_list))
logging.debug(queues_string)
with open(CeleryConfig.get_config_filename(agent_type=agent_type),
'r') as f:
lines = f.readlines()
f = open(CeleryConfig.get_config_filename(agent_type=agent_type), 'w')
clear_following = False
for line in lines:
if clear_following:
f.write("")
elif line.strip().startswith('exchange'):
f.write(exchange + "\n")
elif line.strip().startswith('CELERY_QUEUES'):
f.write(queues_string + "\n")
clear_following = True
else:
f.write(line)
f.close()
# reload the celery configuration
tasks.CelerySingleton().configure()
def copy_celery_config_to_vm(instance_type, ip, key_file, agent_type, username):
celery_config_filename = CeleryConfig.get_config_filename(agent_type=agent_type)
if not os.path.exists(celery_config_filename):
raise Exception("celery config file not found: {0}".format(celery_config_filename))
config_celery_queues(agent_type=agent_type, instance_types=[instance_type])
cmd = get_scp_command(keyfile=key_file, source=celery_config_filename,
target="{user}@{ip}:~/celeryconfig.py".format(user=username, ip=ip))
logging.info(cmd)
success = os.system(cmd)
if success == 0:
logging.info("scp success!")
logging.info(" {0} transfered to {1}".format(celery_config_filename, ip))
else:
raise Exception("scp failure: {0} not transfered to {1}".format(celery_config_filename, ip))
def config_celery_queues(agent_type, instance_types):
exchange = "exchange = Exchange('{0}', type='direct')".format(CeleryConfig.get_exchange_name(agent_type=agent_type))
logging.debug(exchange)
queue_list = map(lambda instance_type: "Queue('{0}', exchange, routing_key='{1}')".format(
CeleryConfig.get_queue_name(agent_type=agent_type, instance_type=instance_type),
CeleryConfig.get_routing_key_name(agent_type=agent_type, instance_type=instance_type)),
instance_types)
agent_queue_name = CeleryConfig.get_queue_name(agent_type=agent_type)
agent_routing_key = CeleryConfig.get_routing_key_name(agent_type=agent_type)
queue_list.insert(0, "Queue('{0}', exchange, routing_key='{1}')".format(agent_queue_name, agent_routing_key))
logging.debug(pprint.pformat(queue_list))
queues_string = 'CELERY_QUEUES = ({0})'.format(', '.join(queue_list))
logging.debug(queues_string)
with open(CeleryConfig.get_config_filename(agent_type=agent_type), 'r') as f:
lines = f.readlines()
f = open(CeleryConfig.get_config_filename(agent_type=agent_type), 'w')
clear_following = False
for line in lines:
if clear_following:
f.write("")
elif line.strip().startswith('exchange'):
f.write(exchange + "\n")
elif line.strip().startswith('CELERY_QUEUES'):
f.write(queues_string + "\n")
clear_following = True
else:
f.write(line)
f.close()
# reload the celery configuration
tasks.CelerySingleton().configure()
def update_celery_config_with_queue_head_ip(queue_head_ip, agent_type):
'''
Method used for updating celery config file. It should have the correct IP
of the queue head node, which should already be running.
Args
queue_head_ip The ip that is going to be written in the celery configuration file
'''
logging.debug("queue_head_ip = {0}".format(queue_head_ip))
with open(CeleryConfig.CONFIG_TEMPLATE_FILENAME, 'r') as celery_config_file:
celery_config_lines = celery_config_file.readlines()
with open(CeleryConfig.get_config_filename(agent_type=agent_type), 'w') as celery_config_file:
for line in celery_config_lines:
if line.strip().startswith('BROKER_URL'):
celery_config_file.write('BROKER_URL = "amqp://*****:*****@{0}:5672/"\n'.format(queue_head_ip))
else:
celery_config_file.write(line)
# Now update the actual Celery app....
# TODO: Doesnt seem to work in GAE until next request comes in to server
tasks.CelerySingleton().configure()
def copy_celery_config_to_vm(instance_type, ip, key_file, agent_type,
username):
celery_config_filename = CeleryConfig.get_config_filename(
agent_type=agent_type)
if not os.path.exists(celery_config_filename):
raise Exception(
"celery config file not found: {0}".format(celery_config_filename))
config_celery_queues(agent_type=agent_type, instance_types=[instance_type])
cmd = get_scp_command(keyfile=key_file,
source=celery_config_filename,
target="{user}@{ip}:~/celeryconfig.py".format(
user=username, ip=ip))
logging.info(cmd)
success = os.system(cmd)
if success == 0:
logging.info("scp success!")
logging.info(" {0} transfered to {1}".format(celery_config_filename,
ip))
else:
raise Exception("scp failure: {0} not transfered to {1}".format(
celery_config_filename, ip))
