def _run(self):
logging.debug(
'SynchronizeDB._run() thread_id={0} agent_type={1} parameters={2}'.
format(self.thread_id, self.agent_type, self.parameters))
self.is_start = False
VMStateModel.synchronize(agent=self.agent, parameters=self.parameters)
if self.update_vm_state_db():
self._start()
def _run(self):
logging.debug(
"SynchronizeDB._run() thread_id={0} agent_type={1} parameters={2}".format(
self.thread_id, self.agent_type, self.parameters
)
)
self.is_start = False
VMStateModel.synchronize(agent=self.agent, parameters=self.parameters)
if self.update_vm_state_db():
self._start()
def synchronize_db(self, params, force=False):
logging.debug('synchronize_db(force={0}) param={1}'.format(
force, params))
last_time = None
set_gap_large = False
try:
e = db.GqlQuery("SELECT * FROM VMStateSyn").get()
if e:
last_time = e.last_syn
else:
last_time = datetime.datetime.now() - datetime.timedelta(1)
except Exception as e:
logging.error(
'Error: have errors in opening db_syn file. {0}'.format(e))
return
if last_time is None:
raise Exception(
'Error: cannot read last synchronization information of db!')
else:
now = datetime.datetime.now()
delta = now - last_time
gap = delta.total_seconds()
logging.info('Time now: {0}'.format(now))
logging.info('Time last synchronization: {0}'.format(last_time))
logging.info('Time in between: {0}'.format(gap))
infrastructure = params[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
if force:
VMStateModel.synchronize(agent=agent, parameters=params)
if gap < backend_handler.SynchronizeDB.PAUSE + 1:
logging.info('Less than {0} seconds to synchronize db.'.format(
backend_handler.SynchronizeDB.PAUSE))
return
logging.info('Start synchronize db every {0} seconds.'.format(
backend_handler.SynchronizeDB.PAUSE))
from_fields = {
'op': 'start_db_syn',
'agent': pickle.dumps(agent),
'parameters': pickle.dumps(params),
}
logging.info('\n\nAdding db syn task for agent = {}'.format(
agent.AGENT_NAME))
taskqueue.add(url=InfrastructureManager.BACKEND_QUEUE_URL,
params=from_fields,
method='GET')
def synchronize_db(self, params, force=False):
logging.debug('synchronize_db(force={0}) param={1}'.format(force, params))
last_time = None
set_gap_large = False
try:
e = db.GqlQuery("SELECT * FROM VMStateSyn").get()
if e:
last_time = e.last_syn
else:
last_time = datetime.datetime.now() - datetime.timedelta(1)
except Exception as e:
logging.error('Error: have errors in opening db_syn file. {0}'.format(e))
return
if last_time is None:
raise Exception('Error: cannot read last synchronization information of db!')
else:
now = datetime.datetime.now()
delta = now - last_time
gap = delta.total_seconds()
logging.info('Time now: {0}'.format(now))
logging.info('Time last synchronization: {0}'.format(last_time))
logging.info('Time in between: {0}'.format(gap))
infrastructure = params[self.PARAM_INFRASTRUCTURE]
agent = self.agent_factory.create_agent(infrastructure)
if force:
VMStateModel.synchronize(agent = agent, parameters = params)
if gap < backend_handler.SynchronizeDB.PAUSE + 1:
logging.info('Less than {0} seconds to synchronize db.'.format(backend_handler.SynchronizeDB.PAUSE))
return
logging.info('Start synchronize db every {0} seconds.'.format(backend_handler.SynchronizeDB.PAUSE))
from_fields = {
'op': 'start_db_syn',
'agent': pickle.dumps(agent),
'parameters': pickle.dumps(params),
}
logging.info('\n\nAdding db syn task for agent = {}'.format(agent.AGENT_NAME))
taskqueue.add(url=InfrastructureManager.BACKEND_QUEUE_URL, params=from_fields, method='GET')
def prepare_vms(self, parameters):
logging.debug('prepare_vms(): parameters={0}'.format(parameters))
queue_head_machine = parameters[self.PARAM_FLEX_QUEUE_HEAD]
user_data = self.__get_user_data(parameters['user_id'])
if self.PARAM_FLEX_CLOUD_MACHINE_INFO not in parameters \
or parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO] == None \
or parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO] == []:
logging.error('Error: No {0} param!'.format(
self.PARAM_FLEX_CLOUD_MACHINE_INFO))
# Report Error
user_data.flex_cloud_status = False
user_data.flex_cloud_info_msg = 'Invalid Parameters'
user_data.put()
return
flex_cloud_machine_info = parameters[
self.PARAM_FLEX_CLOUD_MACHINE_INFO]
# Set the user message to "configuring..."
user_data.flex_cloud_status = True
user_data.flex_cloud_info_msg = 'Flex Cloud configured. Waiting for workers to become available...'
user_data.put()
# Initialize the VMstateModel db
all_accessible = True
for machine in flex_cloud_machine_info:
if self.agent.check_network_ports(machine['ip'], [22, 443]):
state = VMStateModel.STATE_ACCESSIBLE
else:
state = VMStateModel.STATE_INACCESSIBLE
all_accessible = False
vm_state = VMStateModel(state=state,
infra=self.agent_type,
ins_type=FlexConfig.INSTANCE_TYPE,
pri_ip=machine['ip'],
pub_ip=machine['ip'],
username=machine['username'],
keyfile=machine['keyfile'],
ins_id=self.agent.get_flex_instance_id(
machine['ip']),
user_id=parameters['user_id'],
res_id=self.reservation_id)
vm_state.put()
if not all_accessible:
# Report Failure
user_data.flex_cloud_status = False
user_data.flex_cloud_info_msg = 'Error: not all workers are accessible'
user_data.put()
return
if queue_head_machine == None or not helper.wait_for_ssh_connection(
queue_head_machine['keyfile'],
queue_head_machine['ip'],
username=queue_head_machine['username']):
logging.error(
'Found no viable ssh-able/running queue head machine!')
# Report Failure
user_data.flex_cloud_status = False
user_data.flex_cloud_info_msg = 'Error: Can not connect {0} (queue head) via SSH'.format(
queue_head_machine['ip'])
user_data.put()
return
if not self.__prepare_queue_head(queue_head_machine, parameters):
logging.error('Error: could not prepare queue head!')
# Report Failure
user_data.flex_cloud_status = False
user_data.flex_cloud_info_msg = 'Error preparing the queue head'
user_data.put()
return
flex_cloud_workers = []
for machine in parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO]:
if machine[self.PARAM_IS_QUEUE_HEAD] != True:
if helper.wait_for_ssh_connection(
machine['keyfile'],
machine['ip'],
username=machine['username']):
flex_cloud_workers.append(machine)
else:
# Report Failure
user_data.flex_cloud_status = False
user_data.flex_cloud_info_msg = 'Error: Can not connect to {0} via SSH'.format(
machine['ip'])
user_data.put()
return
if len(flex_cloud_workers) > 0:
logging.debug(
'Preparing workers: {0}'.format(flex_cloud_workers))
params = {
'infrastructure':
AgentTypes.FLEX,
self.PARAM_FLEX_CLOUD_MACHINE_INFO:
flex_cloud_workers,
'credentials':
parameters['credentials'],
'user_id':
parameters['user_id'],
self.PARAM_FLEX_QUEUE_HEAD:
parameters[self.PARAM_FLEX_QUEUE_HEAD],
'reservation_id':
parameters['reservation_id']
}
self.agent.prepare_instances(params)
helper.update_celery_config_with_queue_head_ip(
queue_head_ip=queue_head_machine['ip'], agent_type=self.agent_type)
self.__configure_celery(params=parameters)
# Report Success
logging.debug('Flex Cloud Deployed')
user_data.flex_cloud_status = True
user_data.flex_cloud_info_msg = 'Flex Cloud Deployed'
user_data.put()
# Force the update of the instance status
VMStateModel.synchronize(agent=self.agent, parameters=parameters)
return
def prepare_vms(self, parameters):
logging.debug("prepare_vms(): parameters={0}".format(parameters))
queue_head_machine = parameters[self.PARAM_FLEX_QUEUE_HEAD]
user_data = self.__get_user_data(parameters["user_id"])
if (
self.PARAM_FLEX_CLOUD_MACHINE_INFO not in parameters
or parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO] == None
or parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO] == []
):
logging.error("Error: No {0} param!".format(self.PARAM_FLEX_CLOUD_MACHINE_INFO))
# Report Error
user_data.flex_cloud_status = False
user_data.flex_cloud_info_msg = "Invalid Parameters"
user_data.put()
return
flex_cloud_machine_info = parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO]
# Set the user message to "configuring..."
user_data.flex_cloud_status = True
user_data.flex_cloud_info_msg = "Flex Cloud configured. Waiting for workers to become available..."
user_data.put()
# Initialize the VMstateModel db
all_accessible = True
for machine in flex_cloud_machine_info:
if self.agent.check_network_ports(machine["ip"], [22, 443]):
state = VMStateModel.STATE_ACCESSIBLE
else:
state = VMStateModel.STATE_INACCESSIBLE
all_accessible = False
vm_state = VMStateModel(
state=state,
infra=self.agent_type,
ins_type=FlexConfig.INSTANCE_TYPE,
pri_ip=machine["ip"],
pub_ip=machine["ip"],
username=machine["username"],
keyfile=machine["keyfile"],
ins_id=self.agent.get_flex_instance_id(machine["ip"]),
user_id=parameters["user_id"],
res_id=self.reservation_id,
)
vm_state.put()
if not all_accessible:
# Report Failure
user_data.flex_cloud_status = False
user_data.flex_cloud_info_msg = "Error: not all workers are accessible"
user_data.put()
return
if queue_head_machine == None or not helper.wait_for_ssh_connection(
queue_head_machine["keyfile"], queue_head_machine["ip"], username=queue_head_machine["username"]
):
logging.error("Found no viable ssh-able/running queue head machine!")
# Report Failure
user_data.flex_cloud_status = False
user_data.flex_cloud_info_msg = "Error: Can not connect {0} (queue head) via SSH".format(
queue_head_machine["ip"]
)
user_data.put()
return
if not self.__prepare_queue_head(queue_head_machine, parameters):
logging.error("Error: could not prepare queue head!")
# Report Failure
user_data.flex_cloud_status = False
user_data.flex_cloud_info_msg = "Error preparing the queue head"
user_data.put()
return
flex_cloud_workers = []
for machine in parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO]:
if machine[self.PARAM_IS_QUEUE_HEAD] != True:
if helper.wait_for_ssh_connection(machine["keyfile"], machine["ip"], username=machine["username"]):
flex_cloud_workers.append(machine)
else:
# Report Failure
user_data.flex_cloud_status = False
user_data.flex_cloud_info_msg = "Error: Can not connect to {0} via SSH".format(machine["ip"])
user_data.put()
return
if len(flex_cloud_workers) > 0:
logging.debug("Preparing workers: {0}".format(flex_cloud_workers))
params = {
"infrastructure": AgentTypes.FLEX,
self.PARAM_FLEX_CLOUD_MACHINE_INFO: flex_cloud_workers,
"credentials": parameters["credentials"],
"user_id": parameters["user_id"],
self.PARAM_FLEX_QUEUE_HEAD: parameters[self.PARAM_FLEX_QUEUE_HEAD],
"reservation_id": parameters["reservation_id"],
}
self.agent.prepare_instances(params)
helper.update_celery_config_with_queue_head_ip(
queue_head_ip=queue_head_machine["ip"], agent_type=self.agent_type
)
self.__configure_celery(params=parameters)
# Report Success
logging.debug("Flex Cloud Deployed")
user_data.flex_cloud_status = True
user_data.flex_cloud_info_msg = "Flex Cloud Deployed"
user_data.put()
# Force the update of the instance status
VMStateModel.synchronize(agent=self.agent, parameters=parameters)
return
