def periodic_update():
token = get_token()
pbclient = PBClient(token, local_config['INTERNAL_API_BASE_URL'], ssl_verify=False)
instances = pbclient.get_instances()
deprovision_list = []
update_list = []
for instance in instances:
logger.debug('checking instance for actions %s' % instance['name'])
deprovision_required = False
if instance.get('state') in [Instance.STATE_RUNNING]:
if not instance.get('lifetime_left') and instance.get('maximum_lifetime'):
deprovision_required = True
if deprovision_required:
deprovision_list.append(instance)
elif instance.get('state') not in [Instance.STATE_FAILED]:
update_list.append(instance)
if len(deprovision_list) > 10:
deprovision_list = random.sample(deprovision_list, 10)
for instance in deprovision_list:
logger.info('deprovisioning triggered for %s (reason: maximum lifetime exceeded)' % instance.get('id'))
pbclient.do_instance_patch(instance['id'], {'to_be_deleted': True})
run_update.delay(instance.get('id'))
if len(update_list) > 10:
update_list = random.sample(update_list, 10)
for instance in update_list:
run_update.delay(instance.get('id'))
def do_provision(self, token, instance_id):
self.logger.debug("do_provision %s" % instance_id)
pbclient = PBClient(token, self.config['INTERNAL_API_BASE_URL'], ssl_verify=False)
instance = pbclient.get_instance_description(instance_id)
instance_name = instance['name']
instance_user = instance['user_id']
# fetch config
blueprint_config = pbclient.get_blueprint_description(instance['blueprint_id'])
config = blueprint_config['config']
log_uploader = self.create_prov_log_uploader(token, instance_id, log_type='provisioning')
log_uploader.info("Provisioning OpenStack instance (%s)\n" % instance_id)
# fetch user public key
key_data = pbclient.get_user_key_data(instance_user).json()
if not key_data:
error = 'user\'s public key is missing'
error_body = {'state': Instance.STATE_FAILED, 'error_msg': error}
pbclient.do_instance_patch(instance_id, error_body)
self.logger.debug(error)
raise RuntimeError(error)
oss = self.get_oss()
result = oss.provision_instance(
instance_name,
config['image'],
config['flavor'],
public_key=key_data[0]['public_key'],
userdata=config.get('userdata'))
if 'error' in result:
log_uploader.warn('Provisioning failed %s' % result['error'])
return
ip = result['address_data']['public_ip']
instance_data = {
'server_id': result['server_id'],
'floating_ip': ip,
'allocated_from_pool': result['address_data']['allocated_from_pool'],
'security_group_id': result['security_group'],
'endpoints': [
{'name': 'SSH', 'access': 'ssh cloud-user@%s' % ip},
]
}
log_uploader.info("Publishing server data\n")
pbclient.do_instance_patch(
instance_id,
{'instance_data': json.dumps(instance_data), 'public_ip': ip})
log_uploader.info("Provisioning complete\n")
def do_provision(self, token, instance_id):
pbclient = PBClient(token, self.config['INTERNAL_API_BASE_URL'], ssl_verify=False)
instance = pbclient.get_instance_description(instance_id)
cluster_name = instance['name']
instance_dir = '%s/%s' % (self.config['INSTANCE_DATA_DIR'], cluster_name)
# will fail if there is already a directory for this instance
os.makedirs(instance_dir)
# fetch config for this cluster
# config = self.get_blueprint_description(token, instance['blueprint_id'])
# fetch user public key and save it
key_data = pbclient.get_user_key_data(instance['user_id']).json()
user_key_file = '%s/userkey.pub' % instance_dir
if not key_data:
error_body = {'state': Instance.STATE_FAILED, 'error_msg': 'user\'s public key is missing'}
pbclient.do_instance_patch(instance_id, error_body)
raise RuntimeError("User's public key missing")
with open(user_key_file, 'w') as kf:
kf.write(key_data[0]['public_key'])
uploader = self.create_prov_log_uploader(token, instance_id, log_type='provisioning')
self.logger.info('faking provisioning')
cmd = 'time ping -c 10 localhost'
self.run_logged_process(cmd=cmd, cwd=instance_dir, shell=True, log_uploader=uploader)
public_ip = '%s.%s.%s.%s' % (randint(1, 254), randint(1, 254), randint(1, 254), randint(1, 254))
instance_data = {
'endpoints': [
{'name': 'SSH', 'access': 'ssh cloud-user@%s' % public_ip},
{'name': 'Some Web Interface', 'access': 'http://%s/service-x' % public_ip},
]
}
pbclient.do_instance_patch(
instance_id,
{
'public_ip': public_ip, 'instance_data': json.dumps(instance_data)
}
)
def deprovision(self, token, instance_id):
self.logger.debug('starting deprovisioning')
pbclient = PBClient(token, self.config['INTERNAL_API_BASE_URL'], ssl_verify=False)
try:
pbclient.do_instance_patch(instance_id, {'state': Instance.STATE_DELETING})
self.logger.debug('calling subclass do_deprovision')
self.do_deprovision(token, instance_id)
self.logger.debug('finishing deprovisioning')
pbclient.do_instance_patch(instance_id, {'deprovisioned_at': datetime.datetime.utcnow()})
pbclient.do_instance_patch(instance_id, {'state': Instance.STATE_DELETED})
except Exception as e:
self.logger.exception('do_deprovision raised %s' % e)
pbclient.do_instance_patch(instance_id, {'state': Instance.STATE_FAILED})
raise e
def periodic_update():
token = get_token()
pbclient = PBClient(token,
local_config['INTERNAL_API_BASE_URL'],
ssl_verify=False)
instances = pbclient.get_instances()
deprovision_list = []
update_list = []
for instance in instances:
logger.debug('checking instance for actions %s' % instance['name'])
deprovision_required = False
if instance.get('state') in [Instance.STATE_RUNNING]:
if not instance.get('lifetime_left') and instance.get(
'maximum_lifetime'):
deprovision_required = True
if deprovision_required:
deprovision_list.append(instance)
elif instance.get('state') not in [Instance.STATE_FAILED]:
update_list.append(instance)
if len(deprovision_list) > 10:
deprovision_list = random.sample(deprovision_list, 10)
for instance in deprovision_list:
logger.info(
'deprovisioning triggered for %s (reason: maximum lifetime exceeded)'
% instance.get('id'))
pbclient.do_instance_patch(instance['id'], {'to_be_deleted': True})
run_update.delay(instance.get('id'))
if len(update_list) > 10:
update_list = random.sample(update_list, 10)
for instance in update_list:
run_update.delay(instance.get('id'))
def provision(self, token, instance_id):
self.logger.debug('starting provisioning')
pbclient = PBClient(token, self.config['INTERNAL_API_BASE_URL'], ssl_verify=False)
try:
pbclient.do_instance_patch(instance_id, {'state': Instance.STATE_PROVISIONING})
self.logger.debug('calling subclass do_provision')
new_state = self.do_provision(token, instance_id)
if not new_state:
new_state = Instance.STATE_RUNNING
pbclient.do_instance_patch(instance_id, {'state': new_state})
except Exception as e:
self.logger.exception('do_provision raised %s' % e)
pbclient.do_instance_patch(instance_id, {'state': Instance.STATE_FAILED})
raise e
def do_provision(self, token, instance_id):
pbclient = PBClient(token, self.config['INTERNAL_API_BASE_URL'], ssl_verify=False)
instance = pbclient.get_instance_description(instance_id)
cluster_name = instance['name']
instance_dir = '%s/%s' % (self.config['INSTANCE_DATA_DIR'], cluster_name)
# will fail if there is already a directory for this instance
os.makedirs(instance_dir)
# generate pvc config for this cluster
blueprint_config = pbclient.get_blueprint_description(instance['blueprint_id'])['config']
self.logger.debug('Blueprint config: %s' % blueprint_config)
cluster_config = self.create_cluster_config(blueprint_config, cluster_name)
with open('%s/cluster.yml' % instance_dir, 'w') as cf:
cf.write(cluster_config)
cf.write('\n')
# figure out the number of nodes from config provisioning-data
if 'number_of_nodes' in blueprint_config:
num_nodes = int(blueprint_config['number_of_nodes'])
else:
self.logger.warn('number of nodes in cluster not defined, using default: 2')
num_nodes = 2
# fetch user public key and save it
key_data = pbclient.get_user_key_data(instance['user_id']).json()
user_key_file = '%s/userkey.pub' % instance_dir
if not key_data:
pbclient.do_instance_patch(instance_id, {'state': Instance.STATE_FAILED})
raise RuntimeError("User's public key missing")
with open(user_key_file, 'w') as kf:
kf.write(key_data[0]['public_key'])
uploader = self.create_prov_log_uploader(token, instance_id, log_type='provisioning')
# generate keypair for this cluster
key_file = '%s/key.priv' % instance_dir
if not os.path.isfile(key_file):
with open(key_file, 'w') as keyfile:
args = ['nova', 'keypair-add', '%s' % cluster_name]
p = subprocess.Popen(args, cwd=instance_dir, stdout=keyfile, env=self.create_openstack_env())
p.wait()
os.chmod(key_file, stat.S_IRUSR)
# run provisioning
cmd = '/webapps/pouta_blueprints/venv/bin/python /opt/pvc/python/poutacluster.py up %d' % num_nodes
self.logger.debug('spawning "%s"' % cmd)
self.run_logged_process(cmd=cmd, cwd=instance_dir, env=self.create_openstack_env(), log_uploader=uploader)
# add user key for ssh access
cmd = '/webapps/pouta_blueprints/venv/bin/python /opt/pvc/python/poutacluster.py add_key userkey.pub'
self.logger.debug('spawning "%s"' % cmd)
self.run_logged_process(cmd=cmd, cwd=instance_dir, env=self.create_openstack_env(), log_uploader=uploader)
# get public IP
cmd = '/webapps/pouta_blueprints/venv/bin/python /opt/pvc/python/poutacluster.py info'
p = subprocess.Popen(shlex.split(cmd), cwd=instance_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
env=self.create_openstack_env())
out, err = p.communicate()
public_ip = None
for line in out.splitlines():
line = line.strip()
if line.startswith('public ip:'):
public_ip = line.split(':')[1].strip()
break
if public_ip:
instance_data = {
'endpoints': [
{'name': 'SSH', 'access': 'ssh cloud-user@%s' % public_ip},
]
}
pbclient.do_instance_patch(
instance_id,
{'public_ip': public_ip, 'instance_data': json.dumps(instance_data)}
)
# run info as the last command to show the service endpoints at the end of the log
cmd = '/webapps/pouta_blueprints/venv/bin/python /opt/pvc/python/poutacluster.py info'
self.logger.debug('spawning "%s"' % cmd)
self.run_logged_process(cmd=cmd, cwd=instance_dir, env=self.create_openstack_env(), log_uploader=uploader)
def do_provision(self, token, instance_id):
self.logger.debug("do_provision %s" % instance_id)
pbclient = PBClient(token,
self.config['INTERNAL_API_BASE_URL'],
ssl_verify=False)
instance = pbclient.get_instance_description(instance_id)
instance_name = instance['name']
instance_user = instance['user_id']
# fetch config
blueprint_config = pbclient.get_blueprint_description(
instance['blueprint_id'])
config = blueprint_config['config']
log_uploader = self.create_prov_log_uploader(token,
instance_id,
log_type='provisioning')
log_uploader.info("Provisioning OpenStack instance (%s)\n" %
instance_id)
# fetch user public key
key_data = pbclient.get_user_key_data(instance_user).json()
if not key_data:
error = 'user\'s public key is missing'
error_body = {'state': Instance.STATE_FAILED, 'error_msg': error}
pbclient.do_instance_patch(instance_id, error_body)
self.logger.debug(error)
raise RuntimeError(error)
oss = self.get_oss()
result = oss.provision_instance(instance_name,
config['image'],
config['flavor'],
public_key=key_data[0]['public_key'],
userdata=config.get('userdata'))
if 'error' in result:
log_uploader.warn('Provisioning failed %s' % result['error'])
return
ip = result['address_data']['public_ip']
instance_data = {
'server_id': result['server_id'],
'floating_ip': ip,
'allocated_from_pool':
result['address_data']['allocated_from_pool'],
'security_group_id': result['security_group'],
'endpoints': [
{
'name': 'SSH',
'access': 'ssh cloud-user@%s' % ip
},
]
}
log_uploader.info("Publishing server data\n")
pbclient.do_instance_patch(instance_id, {
'instance_data': json.dumps(instance_data),
'public_ip': ip
})
log_uploader.info("Provisioning complete\n")