def update_celery_config_with_queue_head_ip(queue_head_ip, agent_type): ''' Method used for updating celery config file. It should have the correct IP of the queue head node, which should already be running. Args queue_head_ip The ip that is going to be written in the celery configuration file ''' logging.debug("queue_head_ip = {0}".format(queue_head_ip)) with open(CeleryConfig.CONFIG_TEMPLATE_FILENAME, 'r') as celery_config_file: celery_config_lines = celery_config_file.readlines() with open(CeleryConfig.get_config_filename(agent_type=agent_type), 'w') as celery_config_file: for line in celery_config_lines: if line.strip().startswith('BROKER_URL'): celery_config_file.write( 'BROKER_URL = "amqp://*****:*****@{0}:5672/"\n'.format( queue_head_ip)) else: celery_config_file.write(line) # Now update the actual Celery app.... # TODO: Doesnt seem to work in GAE until next request comes in to server tasks.CelerySingleton().configure()
def config_celery_queues(agent_type, instance_types): exchange = "exchange = Exchange('{0}', type='direct')".format( CeleryConfig.get_exchange_name(agent_type=agent_type)) logging.debug(exchange) queue_list = map( lambda instance_type: "Queue('{0}', exchange, routing_key='{1}')". format( CeleryConfig.get_queue_name(agent_type=agent_type, instance_type=instance_type), CeleryConfig.get_routing_key_name(agent_type=agent_type, instance_type=instance_type)), instance_types) agent_queue_name = CeleryConfig.get_queue_name(agent_type=agent_type) agent_routing_key = CeleryConfig.get_routing_key_name( agent_type=agent_type) queue_list.insert( 0, "Queue('{0}', exchange, routing_key='{1}')".format( agent_queue_name, agent_routing_key)) logging.debug(pprint.pformat(queue_list)) queues_string = 'CELERY_QUEUES = ({0})'.format(', '.join(queue_list)) logging.debug(queues_string) with open(CeleryConfig.get_config_filename(agent_type=agent_type), 'r') as f: lines = f.readlines() f = open(CeleryConfig.get_config_filename(agent_type=agent_type), 'w') clear_following = False for line in lines: if clear_following: f.write("") elif line.strip().startswith('exchange'): f.write(exchange + "\n") elif line.strip().startswith('CELERY_QUEUES'): f.write(queues_string + "\n") clear_following = True else: f.write(line) f.close() # reload the celery configuration tasks.CelerySingleton().configure()
def copy_celery_config_to_vm(instance_type, ip, key_file, agent_type, username): celery_config_filename = CeleryConfig.get_config_filename(agent_type=agent_type) if not os.path.exists(celery_config_filename): raise Exception("celery config file not found: {0}".format(celery_config_filename)) config_celery_queues(agent_type=agent_type, instance_types=[instance_type]) cmd = get_scp_command(keyfile=key_file, source=celery_config_filename, target="{user}@{ip}:~/celeryconfig.py".format(user=username, ip=ip)) logging.info(cmd) success = os.system(cmd) if success == 0: logging.info("scp success!") logging.info(" {0} transfered to {1}".format(celery_config_filename, ip)) else: raise Exception("scp failure: {0} not transfered to {1}".format(celery_config_filename, ip))
def config_celery_queues(agent_type, instance_types): exchange = "exchange = Exchange('{0}', type='direct')".format(CeleryConfig.get_exchange_name(agent_type=agent_type)) logging.debug(exchange) queue_list = map(lambda instance_type: "Queue('{0}', exchange, routing_key='{1}')".format( CeleryConfig.get_queue_name(agent_type=agent_type, instance_type=instance_type), CeleryConfig.get_routing_key_name(agent_type=agent_type, instance_type=instance_type)), instance_types) agent_queue_name = CeleryConfig.get_queue_name(agent_type=agent_type) agent_routing_key = CeleryConfig.get_routing_key_name(agent_type=agent_type) queue_list.insert(0, "Queue('{0}', exchange, routing_key='{1}')".format(agent_queue_name, agent_routing_key)) logging.debug(pprint.pformat(queue_list)) queues_string = 'CELERY_QUEUES = ({0})'.format(', '.join(queue_list)) logging.debug(queues_string) with open(CeleryConfig.get_config_filename(agent_type=agent_type), 'r') as f: lines = f.readlines() f = open(CeleryConfig.get_config_filename(agent_type=agent_type), 'w') clear_following = False for line in lines: if clear_following: f.write("") elif line.strip().startswith('exchange'): f.write(exchange + "\n") elif line.strip().startswith('CELERY_QUEUES'): f.write(queues_string + "\n") clear_following = True else: f.write(line) f.close() # reload the celery configuration tasks.CelerySingleton().configure()
def update_celery_config_with_queue_head_ip(queue_head_ip, agent_type): ''' Method used for updating celery config file. It should have the correct IP of the queue head node, which should already be running. Args queue_head_ip The ip that is going to be written in the celery configuration file ''' logging.debug("queue_head_ip = {0}".format(queue_head_ip)) with open(CeleryConfig.CONFIG_TEMPLATE_FILENAME, 'r') as celery_config_file: celery_config_lines = celery_config_file.readlines() with open(CeleryConfig.get_config_filename(agent_type=agent_type), 'w') as celery_config_file: for line in celery_config_lines: if line.strip().startswith('BROKER_URL'): celery_config_file.write('BROKER_URL = "amqp://*****:*****@{0}:5672/"\n'.format(queue_head_ip)) else: celery_config_file.write(line) # Now update the actual Celery app.... # TODO: Doesnt seem to work in GAE until next request comes in to server tasks.CelerySingleton().configure()
def copy_celery_config_to_vm(instance_type, ip, key_file, agent_type, username): celery_config_filename = CeleryConfig.get_config_filename( agent_type=agent_type) if not os.path.exists(celery_config_filename): raise Exception( "celery config file not found: {0}".format(celery_config_filename)) config_celery_queues(agent_type=agent_type, instance_types=[instance_type]) cmd = get_scp_command(keyfile=key_file, source=celery_config_filename, target="{user}@{ip}:~/celeryconfig.py".format( user=username, ip=ip)) logging.info(cmd) success = os.system(cmd) if success == 0: logging.info("scp success!") logging.info(" {0} transfered to {1}".format(celery_config_filename, ip)) else: raise Exception("scp failure: {0} not transfered to {1}".format( celery_config_filename, ip))