def __configure_celery(self, queue_head):
commands = []
commands.append('source ~/.bashrc')
commands.append('export AWS_ACCESS_KEY_ID={0}'.format(
str(self.aws_credentials['AWS_ACCESS_KEY_ID'])))
commands.append('export AWS_SECRET_ACCESS_KEY={0}'.format(
str(self.aws_credentials['AWS_SECRET_ACCESS_KEY'])))
for machine in self.machines:
logging.info(
"Starting celery on {ip}".format(ip=machine["public_ip"]))
success = helper.start_celery_on_vm(
instance_type=self.INSTANCE_TYPE,
ip=machine["public_ip"],
username=machine["username"],
key_file=machine["keyfile"],
prepend_commands=commands,
agent_type=self.AGENT_TYPE)
if success != 0:
raise Exception("Failure to start celery on {0}".format(
machine["public_ip"]))
# get all intstance types and configure the celeryconfig.py locally
instance_types = [self.INSTANCE_TYPE]
helper.config_celery_queues(agent_type=self.AGENT_TYPE,
instance_types=instance_types)
def __configure_celery(self, params):
'''
Private method used for uploading the current celery configuration to each instance
that is running and ssh connectable.
Args
parameters A dictionary of parameters
'''
# Update celery config file...it should have the correct IP
# of the Queue head node, which should already be running.
# Pass it line by line so theres no weird formatting errors from
# trying to echo a multi-line file directly on the command line
logging.debug('__configure_celery() params={0}'.format(params))
flex_cloud_machine_info = params[self.PARAM_FLEX_CLOUD_MACHINE_INFO]
instance_types = []
for machine in flex_cloud_machine_info:
vm = VMStateModel.get_by_ip(
machine['ip'], reservation_id=params['reservation_id'])
commands = []
my_ins_type = 'Unknown'
commands.append('source ~/.bashrc')
if vm is None:
logging.error('VMStateModel.get_by_ip({0}) in None'.format(
machine['ip']))
continue
else:
my_ins_type = vm.ins_type
commands.append('export INSTANCE_TYPE={0}'.format(vm.ins_type))
if vm.ins_type not in instance_types:
instance_types.append(vm.ins_type)
ip = machine['ip']
keyfile = machine['keyfile']
username = machine['username']
success = helper.start_celery_on_vm(instance_type=my_ins_type,
ip=ip,
key_file=keyfile,
username=username,
agent_type=self.agent_type,
worker_name=ip.replace(
'.', '_'),
prepend_commands=commands)
if success == 0:
# update db with successful running vms
logging.info("celery started on host ip: {0}".format(ip))
else:
raise Exception("Fail to start celery on {0}".format(ip))
# get all intstance types and configure the celeryconfig.py locally
logging.info('For local celery setup, instance_types = {0}'.format(
instance_types))
helper.config_celery_queues(agent_type=self.agent_type,
instance_types=instance_types)
def __configure_celery(self, params, public_ips, instance_ids):
"""
Private method used for uploading the current celery configuration to each instance
that is running and ssh connectable.
Args
parameters A dictionary of parameters
public_ips A list of public ips that are going to be configed
instance_ids A list of instance_ids that are used for terminating instances and update
database if fail on configuration by some reason
"""
# Update celery config file...it should have the correct IP
# of the Queue head node, which should already be running.
# Pass it line by line so theres no weird formatting errors from
# trying to echo a multi-line file directly on the command line
key_file = os.path.join(os.path.dirname(__file__), "..", "{0}.key".format(params["keyname"]))
logging.debug("key_file = {0}".format(key_file))
if not os.path.exists(key_file):
raise Exception("ssh key_file file not found: {0}".format(key_file))
credentials = params["credentials"]
commands = []
commands.append("source /home/ubuntu/.bashrc")
commands.append("export AWS_ACCESS_KEY_ID={0}".format(str(credentials["EC2_ACCESS_KEY"])))
commands.append("export AWS_SECRET_ACCESS_KEY={0}".format(str(credentials["EC2_SECRET_KEY"])))
for ip, ins_id in zip(public_ips, instance_ids):
# helper.wait_for_ssh_connection(key_file, ip)
ins_type = VMStateModel.get_instance_type(params, ins_id)
commands.append("export INSTANCE_TYPE={0}".format(ins_type))
success = helper.start_celery_on_vm(
instance_type=ins_type,
ip=ip,
key_file=key_file,
agent_type=self.agent_type,
worker_name=ip.replace(".", "_"),
prepend_commands=commands,
)
if success == 0:
# update db with successful running vms
logging.info("celery started! ")
logging.info("host ip: {0}".format(ip))
VMStateModel.set_state(params, [ins_id], VMStateModel.STATE_RUNNING, VMStateModel.DESCRI_SUCCESS)
else:
self.agent.deregister_some_instances(params, [ins_id])
VMStateModel.set_state(
params, [ins_id], VMStateModel.STATE_FAILED, VMStateModel.DESCRI_FAIL_TO_COFIGURE_CELERY
)
raise Exception("Failure to start celery on {0}".format(ip))
# get all intstance types and configure the celeryconfig.py locally
instance_types = VMStateModel.get_running_instance_types(params)
helper.config_celery_queues(agent_type=self.agent_type, instance_types=instance_types)
def __configure_celery(self, params):
"""
Private method used for uploading the current celery configuration to each instance
that is running and ssh connectable.
Args
parameters A dictionary of parameters
"""
# Update celery config file...it should have the correct IP
# of the Queue head node, which should already be running.
# Pass it line by line so theres no weird formatting errors from
# trying to echo a multi-line file directly on the command line
logging.debug("__configure_celery() params={0}".format(params))
flex_cloud_machine_info = params[self.PARAM_FLEX_CLOUD_MACHINE_INFO]
instance_types = []
for machine in flex_cloud_machine_info:
vm = VMStateModel.get_by_ip(machine["ip"], reservation_id=params["reservation_id"])
commands = []
my_ins_type = "Unknown"
commands.append("source ~/.bashrc")
if vm is None:
logging.error("VMStateModel.get_by_ip({0}) in None".format(machine["ip"]))
continue
else:
my_ins_type = vm.ins_type
commands.append("export INSTANCE_TYPE={0}".format(vm.ins_type))
if vm.ins_type not in instance_types:
instance_types.append(vm.ins_type)
ip = machine["ip"]
keyfile = machine["keyfile"]
username = machine["username"]
success = helper.start_celery_on_vm(
instance_type=my_ins_type,
ip=ip,
key_file=keyfile,
username=username,
agent_type=self.agent_type,
worker_name=ip.replace(".", "_"),
prepend_commands=commands,
)
if success == 0:
# update db with successful running vms
logging.info("celery started on host ip: {0}".format(ip))
else:
raise Exception("Fail to start celery on {0}".format(ip))
# get all intstance types and configure the celeryconfig.py locally
logging.info("For local celery setup, instance_types = {0}".format(instance_types))
helper.config_celery_queues(agent_type=self.agent_type, instance_types=instance_types)
def __configure_celery(self, queue_head):
commands = []
commands.append('source ~/.bashrc')
commands.append('export AWS_ACCESS_KEY_ID={0}'.format(str(self.aws_credentials['AWS_ACCESS_KEY_ID'])))
commands.append('export AWS_SECRET_ACCESS_KEY={0}'.format(str(self.aws_credentials['AWS_SECRET_ACCESS_KEY'])))
for machine in self.machines:
logging.info("Starting celery on {ip}".format(ip=machine["public_ip"]))
success = helper.start_celery_on_vm(instance_type=self.INSTANCE_TYPE,
ip=machine["public_ip"],
username=machine["username"],
key_file=machine["keyfile"],
prepend_commands=commands,
agent_type=self.AGENT_TYPE)
if success != 0:
raise Exception("Failure to start celery on {0}".format(machine["public_ip"]))
# get all intstance types and configure the celeryconfig.py locally
instance_types = [self.INSTANCE_TYPE]
helper.config_celery_queues(agent_type=self.AGENT_TYPE, instance_types=instance_types)
def __configure_celery(self, params, public_ips, instance_ids):
'''
Private method used for uploading the current celery configuration to each instance
that is running and ssh connectable.
Args
parameters A dictionary of parameters
public_ips A list of public ips that are going to be configed
instance_ids A list of instance_ids that are used for terminating instances and update
database if fail on configuration by some reason
'''
# Update celery config file...it should have the correct IP
# of the Queue head node, which should already be running.
# Pass it line by line so theres no weird formatting errors from
# trying to echo a multi-line file directly on the command line
key_file = os.path.join(os.path.dirname(__file__), '..',
'{0}.key'.format(params['keyname']))
logging.debug("key_file = {0}".format(key_file))
if not os.path.exists(key_file):
raise Exception(
"ssh key_file file not found: {0}".format(key_file))
credentials = params['credentials']
commands = []
commands.append('source /home/ubuntu/.bashrc')
commands.append('export AWS_ACCESS_KEY_ID={0}'.format(
str(credentials['EC2_ACCESS_KEY'])))
commands.append('export AWS_SECRET_ACCESS_KEY={0}'.format(
str(credentials['EC2_SECRET_KEY'])))
for ip, ins_id in zip(public_ips, instance_ids):
# helper.wait_for_ssh_connection(key_file, ip)
ins_type = VMStateModel.get_instance_type(params, ins_id)
commands.append('export INSTANCE_TYPE={0}'.format(ins_type))
success = helper.start_celery_on_vm(instance_type=ins_type,
ip=ip,
key_file=key_file,
agent_type=self.agent_type,
worker_name=ip.replace(
'.', '_'),
prepend_commands=commands)
if success == 0:
# update db with successful running vms
logging.info("celery started! ")
logging.info("host ip: {0}".format(ip))
VMStateModel.set_state(params, [ins_id],
VMStateModel.STATE_RUNNING,
VMStateModel.DESCRI_SUCCESS)
else:
self.agent.deregister_some_instances(params, [ins_id])
VMStateModel.set_state(
params, [ins_id], VMStateModel.STATE_FAILED,
VMStateModel.DESCRI_FAIL_TO_COFIGURE_CELERY)
raise Exception("Failure to start celery on {0}".format(ip))
# get all intstance types and configure the celeryconfig.py locally
instance_types = VMStateModel.get_running_instance_types(params)
helper.config_celery_queues(agent_type=self.agent_type,
instance_types=instance_types)