def worker(q, number):
while True:
work = q.get()
member = Session.merge(work['member'])
vm = work['vm']
diagnostic = work['diagnostic']
try:
logging.info("[{}] about to run diagnostic on {}".format(number, diagnostic.host))
diagnostic.run()
logging.info("[{}] finished running diagnostic on {} (exit code: {})".format(number, vm.id, diagnostic.exitcode))
Session.add(PoolMemberDiagnostic(
vm_id=vm.id,
pool=member.pool,
start_date=diagnostic.start_date,
end_date=diagnostic.end_date,
stdout=diagnostic.stdout,
stderr=diagnostic.stderr,
exitcode=diagnostic.exitcode))
Session.commit()
logging.info("[{}] Saved diagnostic for {}".format(number, vm.id))
except Exception as e:
defect_ticket = jira.defect_for_exception(
summary_title="Lifeguard: Health Check => {})".format(e),
tb=traceback.format_exc(),
e=e)
logging.error("[{} worker experienced an error running diagnostic "
"host {}, error: {}, created defect ticket {}".format(
number, vm.name, e, defect_ticket.key))
finally:
q.task_done()
def login():
if current_user.is_authenticated:
flash('You are already logged in.')
return redirect(url_for('auth.home'))
form = LoginForm(request.form)
if request.method == 'POST' and form.validate():
username = request.form.get('username')
password = request.form.get('password')
try:
User.try_login(username, password)
except LDAPException:
flash(
'Invalid username or password. Please try again.',
'danger')
return render_template('auth/login.html', form=form)
try:
with Session.begin_nested():
user = User.query.filter_by(username=username).first()
if not user:
user = User(username)
Session.add(user)
Session.commit()
login_user(user)
flash('You have successfully logged in.', category='success')
return redirect(url_for('auth.home'))
except Exception as e:
flash('There was an error logging in: {}'.format(e), category='danger')
raise e
if form.errors:
flash(form.errors, 'danger')
return render_template('auth/login.html', form=form)
def edit_template(zone_number):
zone = Zone.query.get(zone_number)
clusters = Cluster.query.filter_by(zone=zone).all()
form = ZoneTemplateForm(request.form, obj=zone)
if request.method == 'POST':
if request.form['action'] == "cancel":
flash('Cancelled {} template update'.format(zone.name),
category="info")
return redirect(url_for('zone_bp.view', zone_number=zone.number))
elif request.form['action'] == "save":
if form.validate():
try:
zone.template = request.form['template']
zone.vars = request.form['vars']
Session.add(zone)
Session.commit()
flash(
'Successfully saved template for {}.'.format(
zone.name), 'success')
return redirect(
url_for('zone_bp.view', zone_number=zone.number))
except Exception as e:
flash('Failed to save zone template, error: {}'.format(e),
'danger')
if form.errors:
flash("Errors must be resolved before zone template can be saved",
'danger')
return render_template('zone/template.html',
form=form,
zone=zone,
clusters=clusters)
def diagnostic_worker(q, results, log):
while True:
work = q.get()
member = Session.merge(work['member'])
diagnostic = Diagnostic(user=app.config['SSH_HEALTH_CHECK_USER'],
host=member.vm_name,
ssh_identity_file=app.config['SSH_IDENTITY_FILE'],
cmd=app.config['SSH_HEALTH_CHECK_CMD'],
timeout=app.config['SSH_HEALTH_CHECK_TIMEOUT'])
try:
run_diagnostic(diagnostic)
log.msg("diagnostic against {} succeeded".format(diagnostic.host))
except Exception as e:
log.err("diagnostic error: {}".format(e))
finally:
Session.add(PoolMemberDiagnostic(
vm_id=member.vm_id,
pool=member.pool,
start_date=diagnostic.start_date,
end_date=diagnostic.end_date,
stdout=diagnostic.stdout,
stderr=diagnostic.stderr,
exitcode=diagnostic.exitcode))
Session.commit()
results.append(diagnostic)
q.task_done()
def login():
if current_user.is_authenticated:
flash('You are already logged in.')
return redirect(url_for('auth.home'))
form = LoginForm(request.form)
if request.method == 'POST' and form.validate():
username = request.form.get('username')
password = request.form.get('password')
try:
User.try_login(username, password)
except LDAPException:
flash('Invalid username or password. Please try again.', 'danger')
return render_template('auth/login.html', form=form)
try:
with Session.begin_nested():
user = User.query.filter_by(username=username).first()
if not user:
user = User(username)
Session.add(user)
Session.commit()
login_user(user)
flash('You have successfully logged in.', category='success')
return redirect(url_for('auth.home'))
except Exception as e:
flash('There was an error logging in: {}'.format(e),
category='danger')
raise e
if form.errors:
flash(form.errors, 'danger')
return render_template('auth/login.html', form=form)
def delete(zone_number):
zone = Zone.query.get(zone_number)
clusters = Cluster.query.filter_by(zone=zone).all()
form = ActionForm(request.form, zone=zone)
if request.method == 'POST' and form.validate():
try:
if request.form['action'] == 'Cancel':
flash('Delete {} action cancelled'.format(zone.name),
category='info')
return redirect(
url_for('zone_bp.view', zone_number=zone.number))
elif request.form['action'] == 'Confirm':
Session.delete(zone)
Session.commit()
flash('{} has been deleted'.format(zone.name),
category='success')
return redirect(url_for('zone_bp.list'))
except Exception as e:
flash('There was an error deleting zone {}'.format(zone.name),
category='success')
return redirect(url_for('zone_bp.list'))
return render_template('zone/delete.html',
form=form,
zone=zone,
clusters=clusters)
def diagnostic_worker(q, results, log):
while True:
work = q.get()
member = Session.merge(work['member'])
diagnostic = Diagnostic(
user=app.config['SSH_HEALTH_CHECK_USER'],
host=member.vm_name,
ssh_identity_file=app.config['SSH_IDENTITY_FILE'],
cmd=app.config['SSH_HEALTH_CHECK_CMD'],
timeout=app.config['SSH_HEALTH_CHECK_TIMEOUT'])
try:
run_diagnostic(diagnostic)
log.msg("diagnostic against {} succeeded".format(diagnostic.host))
except Exception as e:
log.err("diagnostic error: {}".format(e))
finally:
Session.add(
PoolMemberDiagnostic(vm_id=member.vm_id,
pool=member.pool,
start_date=diagnostic.start_date,
end_date=diagnostic.end_date,
stdout=diagnostic.stdout,
stderr=diagnostic.stderr,
exitcode=diagnostic.exitcode))
Session.commit()
results.append(diagnostic)
q.task_done()
def edit_template(zone_number, cluster_id):
zone = Zone.query.get(zone_number)
cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first()
pools = VirtualMachinePool.query.filter_by(cluster_id=cluster.id, zone_number=cluster.zone_number).all()
form = ClusterTemplateForm(request.form, obj=cluster)
if request.method == 'POST':
if request.form['action'] == "cancel":
flash('Cancelled {} cluster template update'.format(cluster.name), category="info")
return redirect(url_for('cluster_bp.view', zone_number=zone.number, cluster_id=cluster.id))
elif request.form['action'] == "save":
try:
cluster.template = request.form['template']
cluster.vars = request.form['vars']
Session.add(cluster)
Session.commit()
flash('Successfully saved cluster template for {} (ID={}).'
.format(cluster.name, cluster.id), 'success')
return redirect(url_for('cluster_bp.view', zone_number=zone.number, cluster_id=cluster.id))
except Exception as e:
flash('Failed to save cluster template, error: {}'.format(e), 'danger')
if form.errors:
flash("Errors must be resolved before cluster template can be saved", 'danger')
return render_template('cluster/template.html',
form=form,
cluster=cluster,
pools=pools)
def delete(pool_id):
pool = None
vms_by_id = {}
form = ActionForm()
try:
pool = VirtualMachinePool.query.get(pool_id)
except Exception as e:
flash("There was an error fetching pool_id={}: {}".format(pool_id, e), category='danger')
if request.method == 'POST' and form.validate():
try:
if request.form['action'] == 'cancel':
flash('Delete {} action cancelled'.format(pool.name), category='info')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.form['action'] == 'confirm':
redirect_url = url_for('cluster_bp.view', zone_number=pool.cluster.zone.number, cluster_id=pool.cluster.id)
members = pool.get_memberships()
for member in members:
Session.delete(member)
for ticket in pool.get_tickets():
logging.info("deleted ticket {}".format(ticket.ticket_key))
Session.delete(ticket)
Session.delete(pool)
Session.commit()
flash('Deleted pool {} with {} members'.format(pool.name, len(members)), category='success')
return redirect(url_for('cluster_bp.view', zone_number=pool.cluster.zone.number, cluster_id=pool.cluster.id))
except Exception as e:
# raise e
flash('There was an error deleting pool {}: {}'.format(pool.name, e), category='danger')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
return render_template('vpool/delete.html',
form=form,
pool=pool,
vms_by_id=vms_by_id)
def shrink(pool_id):
pool = form_shrink_vm_ids = None
form = ActionForm()
try:
pool = VirtualMachinePool.query.get(pool_id)
if request.method == 'POST' and request.form['action'] == 'cancel':
flash('Shrink {} cancelled'.format(pool.name), category='info')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.method == 'POST' and request.form['action'] == 'shrink':
form_shrink_vm_ids = [
int(id) for id in request.form.getlist('shrink_vm_ids')
]
members = pool.get_memberships()
shrink_members = pool.get_members_to_shrink(members,
form_shrink_vm_ids)
if shrink_members is None or len(shrink_members) == 0:
raise Exception(
"Cannot determine any members to shutdown for shrinking")
except Exception as e:
flash("There was an error determining memberships for shrinking: {}".
format(e),
category='danger')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
if request.method == 'POST' and form.validate(
) and request.form['action'] == 'shrink':
title = 'Plan Shrink => Pool {} ({} members to {})'.format(
pool.name, len(members), pool.cardinality)
description = "Pool shrink triggered that will shutdown {} VM(s): \n\n*{}".format(
len(shrink_members),
"\n*".join([m.vm.name for m in shrink_members]))
task = Task(name=title,
description="{}\n{}".format(title, description),
username=current_user.username)
Session.add(task)
Session.commit()
task_thread = TaskThread(task_id=task.id,
run_function=plan_shrink,
pool=pool,
shrink_members=shrink_members)
task_thread.start()
flash(
Markup("Started background task {}: {}".format(
task.name, task.link())))
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
return render_template('vpool/shrink.html',
form=form,
pool=pool,
shrink_members=shrink_members)
def create_new_pool(zone_number, cluster_id):
try:
zone = Zone.query.get(zone_number)
cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first()
form = pool = None
pool = VirtualMachinePool(zone_number=cluster.zone.number,
cluster=cluster)
pools = VirtualMachinePool.query.filter_by(
cluster_id=cluster.id, zone_number=cluster.zone_number).all()
form = PoolEditForm(request.form, obj=pool)
except Exception as e:
flash(
"There was an error fetching objects required for creating pool: {}"
.format(e))
return redirect(
url_for('cluster_bp.view',
zone_number=zone.number,
cluster_id=cluster.id))
if request.method == 'POST':
if request.form['action'] == "cancel":
flash('Cancelled {} pool template update'.format(pool.name),
category="info")
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.form['action'] == "save":
try:
cardinality_pattern = re.compile("\d+")
pool.name = request.form['name']
pool.template = request.form['template']
pool.vars = request.form['vars']
if not cardinality_pattern.fullmatch(
request.form['cardinality']):
raise Exception("Cardinality {} not numeric".format(
request.form['cardinality']))
pool.cardinality = request.form['cardinality']
Session.add(pool)
Session.commit()
flash(
'Successfully saved pool template for {} (ID={}).'.format(
pool.name, pool.id), 'success')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
except Exception as e:
flash('Failed to save pool error: {}'.format(e), 'danger')
if form.errors:
flash("Errors must be resolved before pool can be saved", 'danger')
return render_template('vpool/create.html',
form=form,
pool=pool,
pools=pools)
def pending_elasticity_tickets(self):
tickets = []
for t in Session.query(PoolTicket).filter_by(pool=self).all():
if not t.done and t.action_id in [PoolTicketActions.expand.value, PoolTicketActions.shrink.value]:
tickets.append(t)
print("found ticket: {}".format(t.done))
return tickets
def remove_done(pool_id):
form = ActionForm()
pool = one_proxy = members = None
try:
pool = VirtualMachinePool.query.get(pool_id)
members = pool.get_memberships()
except Exception as e:
flash("There was an error finshed VMs: {}".format(e), category='danger')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
if request.method == 'POST' and form.validate():
try:
if request.form['action'] == 'cancel':
flash('Cleanup of {} cancelled'.format(pool.name), category='info')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.form['action'] == 'confirm':
vm_ids_to_delete = [int(id) for id in request.form.getlist('done_vm_ids')]
delete_members = []
Session()
for m in members:
if m.vm.id in vm_ids_to_delete:
delete_members.append(m)
delete_ticket = jira.instance.create_issue(
project=app.config['JIRA_PROJECT'],
summary='[auto-{}] Pool Cleanup: {} (deleting {} done VMs)'.format(
current_user.username, pool.name, len(vm_ids_to_delete)),
description="Pool cleanup triggered that will delete {} VM(s): \n\n*{}".format(
len(vm_ids_to_delete),
"\n*".join(['ID {}: {} ({})'.format(m.vm.id, m.vm.name, m.vm.ip_address) for m in delete_members])),
customfield_13842=jira.get_datetime_now(),
issuetype={'name': 'Task'})
one_proxy = OneProxy(pool.cluster.zone.xmlrpc_uri, pool.cluster.zone.session_string, verify_certs=False)
for m in delete_members:
one_proxy.action_vm(m.remove_cmd(), m.vm.id)
Session.delete(m)
Session.commit()
flash('Deleted {} done VMs to cleanup pool {}'.format(len(delete_members), pool.name))
jira.resolve(delete_ticket)
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
except Exception as e:
flash("Error performing cleanup of pool {}: {}".format(pool.name, e), category='danger')
jira.defect_for_exception("Error during cleanup of pool {}".format(pool.name), e)
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
return render_template('vpool/remove_done.html',
form=form,
pool=pool,
members=members)
def expand(pool_id):
pool = members = expansion_names = form_expansion_names = None
form = ActionForm()
try:
pool = VirtualMachinePool.query.get(pool_id)
if request.method == 'POST' and request.form['action'] == 'cancel':
flash('Expansion of {} cancelled'.format(pool.name),
category='info')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.method == 'POST' and request.form['action'] == 'expand':
form_expansion_names = request.form.getlist('expansion_names')
members = pool.get_memberships()
expansion_names = pool.get_expansion_names(members,
form_expansion_names)
except Exception as e:
flash(
"There was an error determining new names required for expansion: {}"
.format(e),
category='danger')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
if request.method == 'POST' and form.validate(
) and request.form['action'] == 'expand':
title = 'Plan Change => Pool Expansion: {} ({} members to {})'.format(
pool.name, len(members), pool.cardinality)
description = "Pool expansion triggered that will instantiate {} new VM(s): \n\n*{}".format(
len(expansion_names), "\n*".join(expansion_names))
task = Task(name=title,
description="{}\n{}".format(title, description),
username=current_user.username)
Session.add(task)
Session.commit()
task_thread = TaskThread(task_id=task.id,
run_function=plan_expansion,
pool=pool,
expansion_names=expansion_names)
task_thread.start()
flash(
Markup("Started background task {}: {}".format(
task.name, task.link())))
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
return render_template('vpool/expand.html',
form=form,
members=members,
pool=pool,
expansion_names=expansion_names)
def create_expansion_ticket(name, pool, members, expansion_names):
logging.info("[{}] {} requires expansion and an existing change ticket doesn't exist".format(name, pool.name))
title = 'Plan Change => Pool Expansion: {} ({} members to {})'.format(pool.name, len(members), pool.cardinality)
description = "Pool expansion triggered that will instantiate {} new VM(s): \n\n*{}".format(
len(expansion_names),
"\n*".join(expansion_names))
task = Task(
name=title,
description="{}\n{}".format(title, description),
username="******")
Session.add(task)
Session.commit()
task_thread = TaskThread(task=task,
run_function=plan_expansion,
pool=Session.merge(pool),
expansion_names=expansion_names)
threads.append(task_thread)
logging.info("[{}] launched background task {} to {}".format(name, task.id, title))
def create_update_ticket(name, pool, members, update_members):
logging.info("[{}] {} requires updating and an existing change ticket doesn't exist".format(name, pool.name))
title = 'Plan Update => Pool {} ({}/{} members need updates)'.format(pool.name, len(members), pool.cardinality)
description = "Pool update triggered that will update {} VM(s): \n\n*{}".format(
len(update_members),
"\n*".join([m.vm.name for m in update_members]))
task = Task(
name=title,
description="{}\n{}".format(title, description),
username="******")
Session.add(task)
Session.commit()
task_thread = TaskThread(task=task,
run_function=plan_update,
pool=Session.merge(pool),
update_members=update_members)
threads.append(task_thread)
logging.info("[{}] launched background task {} to {}".format(name, task.id, title))
def pending_elasticity_tickets(self):
tickets = []
for t in Session.query(PoolTicket).filter_by(pool=self).all():
if not t.done and t.action_id in [
PoolTicketActions.expand.value,
PoolTicketActions.shrink.value
]:
tickets.append(t)
print("found ticket: {}".format(t.done))
return tickets
def __init__(self, task, run_function, log=None, **kwargs):
if None in [task, run_function]:
raise Exception("Required parameter(s) is None: task={}, run_function={}".format(task, run_function))
if log is not None:
self.log = log
else:
self.log = DumbLog()
self.task = None
self.task = Session.merge(task)
self.run_function = types.MethodType(run_function, self)
super().__init__(target=self.run_task, daemon=True, kwargs=kwargs)
def shrink(pool_id):
pool = form_shrink_vm_ids = None
form = ActionForm()
try:
pool = VirtualMachinePool.query.get(pool_id)
if request.method == 'POST' and request.form['action'] == 'cancel':
flash('Shrink {} cancelled'.format(pool.name), category='info')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.method == 'POST' and request.form['action'] == 'shrink':
form_shrink_vm_ids = [int(id) for id in request.form.getlist('shrink_vm_ids')]
members = pool.get_memberships()
shrink_members = pool.get_members_to_shrink(members, form_shrink_vm_ids)
if shrink_members is None or len(shrink_members) == 0:
raise Exception("Cannot determine any members to shutdown for shrinking")
except Exception as e:
flash("There was an error determining memberships for shrinking: {}".format(e), category='danger')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
if request.method == 'POST' and form.validate() and request.form['action'] == 'shrink':
title = 'Plan Shrink => Pool {} ({} members to {})'.format(pool.name, len(members), pool.cardinality)
description = "Pool shrink triggered that will shutdown {} VM(s): \n\n*{}".format(
len(shrink_members),
"\n*".join([m.vm.name for m in shrink_members]))
task = Task(
name=title,
description="{}\n{}".format(title, description),
username=current_user.username)
Session.add(task)
Session.commit()
task_thread = TaskThread(task_id=task.id,
run_function=plan_shrink,
pool=pool,
shrink_members=shrink_members)
task_thread.start()
flash(Markup("Started background task {}: {}".format(task.name, task.link())))
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
return render_template('vpool/shrink.html',
form=form,
pool=pool,
shrink_members=shrink_members)
def update(pool_id):
form = ActionForm()
pool = VirtualMachinePool.query.get(pool_id)
form_update_ids = None
if request.method == 'POST' and request.form['action'] == 'cancel':
flash('Update of {} cancelled'.format(pool.name), category='info')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.method == 'POST' and request.form['action'] == 'update':
form_update_ids = request.form.getlist('update_ids')
members = pool.get_memberships()
update_ids = pool.get_update_ids(members, form_update_ids)
if request.method == 'POST' and form.validate(
) and request.form['action'] == 'update':
id_to_template = {}
for m in members:
if m.vm.id in update_ids:
id_to_template[m.vm.id] = m.current_template()
title = "Plan Change => Update {} member(s) in pool {}".format(
len(update_ids), pool.name)
description = "The attached templates will replace the VMs identified in their filename (based on VM ID)"
task = Task(name=title,
description="{}\n{}".format(title, description),
username=current_user.username)
Session.add(task)
Session.commit()
task_thread = TaskThread(task_id=task.id,
run_function=plan_update,
pool=pool,
id_to_template=id_to_template)
task_thread.start()
flash(
Markup("Started background task {}: {}".format(title,
task.link())))
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
return render_template('vpool/update.html',
form=form,
pool=pool,
members=members,
update_ids=update_ids)
def __init__(self, task, run_function, log=None, **kwargs):
if None in [task, run_function]:
raise Exception(
"Required parameter(s) is None: task={}, run_function={}".
format(task, run_function))
if log is not None:
self.log = log
else:
self.log = DumbLog()
self.task = None
self.task = Session.merge(task)
self.run_function = types.MethodType(run_function, self)
super().__init__(target=self.run_task, daemon=True, kwargs=kwargs)
def expand(pool_id):
pool = members = expansion_names = form_expansion_names = None
form = ActionForm()
try:
pool = VirtualMachinePool.query.get(pool_id)
if request.method == 'POST' and request.form['action'] == 'cancel':
flash('Expansion of {} cancelled'.format(pool.name), category='info')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.method == 'POST' and request.form['action'] == 'expand':
form_expansion_names = request.form.getlist('expansion_names')
members = pool.get_memberships()
expansion_names = pool.get_expansion_names(members, form_expansion_names)
except Exception as e:
flash("There was an error determining new names required for expansion: {}".format(e), category='danger')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
if request.method == 'POST' and form.validate() and request.form['action'] == 'expand':
title = 'Plan Change => Pool Expansion: {} ({} members to {})'.format(pool.name, len(members), pool.cardinality)
description = "Pool expansion triggered that will instantiate {} new VM(s): \n\n*{}".format(
len(expansion_names),
"\n*".join(expansion_names))
task = Task(
name=title,
description="{}\n{}".format(title, description),
username=current_user.username)
Session.add(task)
Session.commit()
task_thread = TaskThread(task_id=task.id,
run_function=plan_expansion,
pool=pool,
expansion_names=expansion_names)
task_thread.start()
flash(Markup("Started background task {}: {}".format(task.name, task.link())))
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
return render_template('vpool/expand.html',
form=form,
members=members,
pool=pool,
expansion_names=expansion_names)
def discover(zone_number):
zone = Zone.query.get(zone_number)
clusters = Cluster.query.filter_by(zone=zone).all()
one_proxy = OneProxy(zone.xmlrpc_uri,
zone.session_string,
verify_certs=False)
one_clusters = one_proxy.get_clusters()
for one_cluster in one_clusters:
existing_cluster = Cluster.query.filter_by(zone_number=zone.number,
id=one_cluster.id).first()
if existing_cluster is None:
discovered_cluster = Cluster(id=one_cluster.id,
zone=zone,
name=one_cluster.name)
Session.add(discovered_cluster)
Session.commit()
flash('Newly discovered ONE cluster: {} (ID={}) in zone {}'.format(
one_cluster.id, one_cluster.name, zone.name),
category='success')
return render_template('zone/discover.html',
zone=zone,
one_clusters=one_clusters,
clusters=clusters)
def edit_template(zone_number, cluster_id):
zone = Zone.query.get(zone_number)
cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first()
pools = VirtualMachinePool.query.filter_by(
cluster_id=cluster.id, zone_number=cluster.zone_number).all()
form = ClusterTemplateForm(request.form, obj=cluster)
if request.method == 'POST':
if request.form['action'] == "cancel":
flash('Cancelled {} cluster template update'.format(cluster.name),
category="info")
return redirect(
url_for('cluster_bp.view',
zone_number=zone.number,
cluster_id=cluster.id))
elif request.form['action'] == "save":
try:
cluster.template = request.form['template']
cluster.vars = request.form['vars']
Session.add(cluster)
Session.commit()
flash(
'Successfully saved cluster template for {} (ID={}).'.
format(cluster.name, cluster.id), 'success')
return redirect(
url_for('cluster_bp.view',
zone_number=zone.number,
cluster_id=cluster.id))
except Exception as e:
flash('Failed to save cluster template, error: {}'.format(e),
'danger')
if form.errors:
flash("Errors must be resolved before cluster template can be saved",
'danger')
return render_template('cluster/template.html',
form=form,
cluster=cluster,
pools=pools)
def create_new_pool(zone_number, cluster_id):
try:
zone = Zone.query.get(zone_number)
cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first()
form = pool = None
pool = VirtualMachinePool(zone_number=cluster.zone.number, cluster=cluster)
pools = VirtualMachinePool.query.filter_by(cluster_id=cluster.id, zone_number=cluster.zone_number).all()
form = PoolEditForm(request.form, obj=pool)
except Exception as e:
flash("There was an error fetching objects required for creating pool: {}".format(e))
return redirect(url_for('cluster_bp.view', zone_number=zone.number, cluster_id=cluster.id))
if request.method == 'POST':
if request.form['action'] == "cancel":
flash('Cancelled {} pool template update'.format(pool.name), category="info")
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.form['action'] == "save":
try:
cardinality_pattern = re.compile("\d+")
pool.name = request.form['name']
pool.template = request.form['template']
pool.vars = request.form['vars']
if not cardinality_pattern.fullmatch(request.form['cardinality']):
raise Exception("Cardinality {} not numeric".format(request.form['cardinality']))
pool.cardinality = request.form['cardinality']
Session.add(pool)
Session.commit()
flash('Successfully saved pool template for {} (ID={}).'
.format(pool.name, pool.id), 'success')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
except Exception as e:
flash('Failed to save pool error: {}'.format(e), 'danger')
if form.errors:
flash("Errors must be resolved before pool can be saved", 'danger')
return render_template('vpool/create.html',
form=form,
pool=pool,
pools=pools)
def update(pool_id):
form = ActionForm()
pool = VirtualMachinePool.query.get(pool_id)
form_update_ids = None
if request.method == 'POST' and request.form['action'] == 'cancel':
flash('Update of {} cancelled'.format(pool.name), category='info')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.method == 'POST' and request.form['action'] == 'update':
form_update_ids = request.form.getlist('update_ids')
members = pool.get_memberships()
update_ids = pool.get_update_ids(members, form_update_ids)
if request.method == 'POST' and form.validate() and request.form['action'] == 'update':
id_to_template = {}
for m in members:
if m.vm.id in update_ids:
id_to_template[m.vm.id] = m.current_template()
title = "Plan Change => Update {} member(s) in pool {}".format(len(update_ids), pool.name)
description = "The attached templates will replace the VMs identified in their filename (based on VM ID)"
task = Task(name=title,
description="{}\n{}".format(title, description),
username=current_user.username)
Session.add(task)
Session.commit()
task_thread = TaskThread(task_id=task.id,
run_function=plan_update,
pool=pool,
id_to_template=id_to_template)
task_thread.start()
flash(Markup("Started background task {}: {}".format(title, task.link())))
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
return render_template('vpool/update.html',
form=form,
pool=pool,
members=members,
update_ids=update_ids)
def manage(zone_number):
zone = Zone()
zones = Zone.query.order_by(Zone.number.desc()).all()
template = 'zone/add.html'
clusters = None
if zone_number is not None:
zone = Zone.query.get(zone_number)
clusters = Cluster.query.filter_by(zone=zone)
template = 'zone/edit.html'
form = ZoneForm(request.form, obj=zone)
if request.method == 'POST':
if request.form['action'] == "cancel":
if zone_number is not None:
flash('{} edit cancelled'.format(zone.name), category="info")
return redirect(
url_for('zone_bp.view', zone_number=zone_number))
flash('Create zone cancelled', category="info")
return redirect(url_for('zone_bp.list'))
elif request.form['action'] == "save":
if form.validate():
try:
form.populate_obj(zone)
Session.add(zone)
Session.commit()
flash('Successfully saved {}.'.format(zone.name),
'success')
return redirect(url_for('zone_bp.list'))
except Exception as e:
flash('Failed to save zone, error: {}'.format(e), 'danger')
if form.errors:
flash("Errors must be resolved before zone can be saved", 'danger')
return render_template(template,
form=form,
zone=zone,
zones=zones,
clusters=clusters)
def all_pools_and_members():
"""
Gets all pools efficiently by caching VMs per zone
:return: A array of pool, member_array tuples
"""
a = []
zone_vm_cache = {}
for pool in Session.query(VirtualMachinePool).all():
if not pool.cluster.zone.name in zone_vm_cache:
logging.info("VM cache for zone {} doesn't exist...".format(pool.cluster.zone.name))
one_proxy = OneProxy(pool.cluster.zone.xmlrpc_uri, pool.cluster.zone.session_string, verify_certs=False)
zone_vm_cache[pool.cluster.zone.name] = {vm.id: vm for vm in one_proxy.get_vms(INCLUDING_DONE)}
logging.info("VM cache for zone {} populated with {} entries".format(
pool.cluster.zone.name, len(zone_vm_cache[pool.cluster.zone.name])))
a.append((pool, pool.get_memberships(vm_cache=zone_vm_cache[pool.cluster.zone.name])))
return a
def delete(pool_id):
pool = None
vms_by_id = {}
form = ActionForm()
try:
pool = VirtualMachinePool.query.get(pool_id)
except Exception as e:
flash("There was an error fetching pool_id={}: {}".format(pool_id, e),
category='danger')
if request.method == 'POST' and form.validate():
try:
if request.form['action'] == 'cancel':
flash('Delete {} action cancelled'.format(pool.name),
category='info')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.form['action'] == 'confirm':
redirect_url = url_for('cluster_bp.view',
zone_number=pool.cluster.zone.number,
cluster_id=pool.cluster.id)
members = pool.get_memberships()
for member in members:
Session.delete(member)
for ticket in pool.get_tickets():
logging.info("deleted ticket {}".format(ticket.ticket_key))
Session.delete(ticket)
Session.delete(pool)
Session.commit()
flash('Deleted pool {} with {} members'.format(
pool.name, len(members)),
category='success')
return redirect(
url_for('cluster_bp.view',
zone_number=pool.cluster.zone.number,
cluster_id=pool.cluster.id))
except Exception as e:
# raise e
flash('There was an error deleting pool {}: {}'.format(
pool.name, e),
category='danger')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
return render_template('vpool/delete.html',
form=form,
pool=pool,
vms_by_id=vms_by_id)
def worker(name, q, cowboy_mode, plan, implement):
while True:
work = q.get()
members = work['members']
pool = Session.merge(work['pool'])
if cowboy_mode:
name = "{}_{}".format('cowboy', name)
logging.info("[{}] called in cowbow mode. All change management "
"protocol around status/schedule will be ingored".format(name))
try:
logging.info("[{}] assigned tickets for pool {}".format(name, pool.name))
if plan:
existing_ticket = pool.pending_ticket()
expansion_names = pool.get_expansion_names(members)
shrink_members = pool.get_members_to_shrink(members)
update_members = pool.get_update_members(members)
if existing_ticket is not None:
logging.info("[{}] existing expand ticket {} already created for {}".format(
name, existing_ticket.ticket_key, pool.name))
elif expansion_names:
create_expansion_ticket(name, pool, members, expansion_names)
elif shrink_members:
create_shrink_ticket(name, pool, members, shrink_members)
elif update_members:
create_update_ticket(name, pool, members, update_members)
else:
logging.info("[{}] skipping ticket planning".format(name))
if implement:
execute_tickets(name, pool, cowboy_mode)
else:
logging.info("[{}] skipping ticket implementation".format(name))
logging.info("[{}] finished working on tickets for pool {}".format(name, pool.name))
except Exception as e:
defect_ticket = jira.defect_for_exception(
username="******",
summary_title="Lifeguard: Health Check => {})".format(e),
tb=traceback.format_exc(),
e=e)
logging.error("[{}] experienced an error pool {}, error: {}, created defect ticket {}".format(
name, pool.name, e, defect_ticket.key))
finally:
q.task_done()
def edit(pool_id):
form = pool = one_proxy = members = None
try:
pool = VirtualMachinePool.query.get(pool_id)
form = PoolEditForm(request.form, obj=pool)
members = pool.get_memberships()
except Exception as e:
flash(
"There was an error fetching objects required for editing pool {}: {}"
.format(pool.name, e))
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
if request.method == 'POST':
if request.form['action'] == "cancel":
flash('Cancelled {} pool template update'.format(pool.name),
category="info")
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.form['action'] == "save":
try:
cardinality_pattern = re.compile("\d+")
pool.name = request.form['name']
pool.template = request.form['template']
pool.vars = request.form['vars']
if not cardinality_pattern.fullmatch(
request.form['cardinality']):
raise Exception("Cardinality {} not numeric".format(
request.form['cardinality']))
pool.cardinality = request.form['cardinality']
Session.add(pool)
for m in members:
if m.template == 'not-yet-compiled':
m.template = ''
Session.add(m)
Session.commit()
flash(
'Successfully saved pool template for {} (ID={}).'.format(
pool.name, pool.id), 'success')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
except Exception as e:
flash('Failed to save pool template, error: {}'.format(e),
'danger')
if form.errors:
flash("Errors must be resolved before pool template can be saved",
'danger')
return render_template('vpool/edit.html',
form=form,
members=members,
pool=pool)
def edit(pool_id):
form = pool = one_proxy = members = None
try:
pool = VirtualMachinePool.query.get(pool_id)
form = PoolEditForm(request.form, obj=pool)
members = pool.get_memberships()
except Exception as e:
flash("There was an error fetching objects required for editing pool {}: {}".format(
pool.name, e))
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
if request.method == 'POST':
if request.form['action'] == "cancel":
flash('Cancelled {} pool template update'.format(pool.name), category="info")
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.form['action'] == "save":
try:
cardinality_pattern = re.compile("\d+")
pool.name = request.form['name']
pool.template = request.form['template']
pool.vars = request.form['vars']
if not cardinality_pattern.fullmatch(request.form['cardinality']):
raise Exception("Cardinality {} not numeric".format(request.form['cardinality']))
pool.cardinality = request.form['cardinality']
Session.add(pool)
for m in members:
if m.template == 'not-yet-compiled':
m.template = ''
Session.add(m)
Session.commit()
flash('Successfully saved pool template for {} (ID={}).'
.format(pool.name, pool.id), 'success')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
except Exception as e:
flash('Failed to save pool template, error: {}'.format(e), 'danger')
if form.errors:
flash("Errors must be resolved before pool template can be saved", 'danger')
return render_template('vpool/edit.html',
form=form,
members=members,
pool=pool)
def execute_tickets(name, pool, cowboy_mode=False):
for t in pool.change_tickets:
try:
if t.done == False:
logging.info("[{}] fetching {}".format(name, t.ticket_key))
crq = jira.instance.issue(t.ticket_key)
if not cowboy_mode and JiraApi.expired(crq):
logging.info("[{}] {} has expired".format(name, crq.key))
jira.cancel_crq_and_tasks(crq, "change has expired")
t.done = True
Session.add(t)
logging.info("[{}] marked {} ticket {} as done for pool {} as crq is expired".format(
name, t.action_name(), t.ticket_key, pool.name))
elif cowboy_mode or JiraApi.in_window(crq):
logging.info("[{}] change {} is in window".format(name, crq.key))
if cowboy_mode or jira.is_ready(crq):
title = "{} pool {} under ticket {}".format(t.action_name(), pool.name, crq.key)
task = Task(
name=title,
description=title,
username="******")
Session.add(task)
Session.commit()
task_thread = TaskThread(task=task,
run_function=get_runnable_from_action_id(t.action_id),
pool=pool,
pool_ticket=t,
issue=crq,
cowboy_mode=cowboy_mode)
threads.append(task_thread)
logging.info("[{}] launched background task {} to {}".format(name, task.id, title))
else:
logging.error("[{}] {} is in window and either it or one or "
"more sub tasks are not ready".format(name, crq.key))
else:
logging.info("[{}] {} change {} for pool {} not yet in window".format(
name, t.action_name(), crq.key, pool.name))
except Exception as e:
logging.error("[{}] error executing {}: {}".format(name, t.ticket_key, e))
def pending_ticket(self): return Session.query(PoolTicket).filter_by(pool=self, done=False).first()
def assign_to_pool(zone_number, cluster_id):
vms = []
id_to_vm = {}
selected_vm_ids = {}
pools = None
zone = None
cluster = None
memberships = {}
try:
Session()
zone = Zone.query.get(zone_number)
cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first()
one_proxy = OneProxy(zone.xmlrpc_uri,
zone.session_string,
verify_certs=False)
for membership in PoolMembership.query.join(
VirtualMachinePool).filter_by(cluster=cluster).all():
memberships[membership.vm_id] = membership
for vm in one_proxy.get_vms():
if vm.disk_cluster.id == cluster.id:
vms.append(vm)
id_to_vm[vm.id] = vm
pools = VirtualMachinePool.get_all(cluster)
except Exception as e:
# raise e
flash("Error fetching VMs in zone number {}: {}".format(
zone.number, e),
category='danger')
form = ActionForm()
active_tab = 'create_new_pool'
# Form submission handling begins
if form.validate_on_submit():
# Determine which tab needs to be active based on the action
if request.form['action'] is not None:
print('something')
active_tab = {
'create new pool': 'create_new_pool',
'add to pool': 'add_to_existing_pool'
}[request.form['action']]
# Get a list of clusters of all selected VMs--pools cannot span clusters
selected_clusters = {}
for id in request.form.getlist('chk_vm_id'):
selected_vm_ids[int(id)] = id
selected_clusters[id_to_vm[int(id)].disk_cluster.id] = True
# Error checking begins
proceed = True
if len(selected_vm_ids) == 0:
flash("No virtual machines were selected!", category='danger')
proceed = False
elif len(selected_clusters) != 1:
flash("Selected VMs must all be in the same cluster",
category='danger')
proceed = False
# Handle the appropriate action if able to proceed
if proceed and request.form['action'] == 'add to pool':
if (request.form['pool_id']
) is None or request.form['pool_id'] == '':
flash('No pool selected', category='danger')
else:
pool = VirtualMachinePool.query.get(request.form['pool_id'])
for vm_id in selected_vm_ids.keys():
Session.add(
PoolMembership(pool=pool,
vm_name=id_to_vm[vm_id].name,
vm_id=vm_id,
date_added=datetime.utcnow()))
Session.commit()
flash(Markup(
'Successfully added {} members to pool <a href="{}">{}</a>'
.format(
len(selected_vm_ids),
url_for('vpool_bp.view', pool_id=pool.id),
pool.name,
)),
category='success')
return redirect(
url_for('vpool_bp.assign_to_pool',
zone_number=zone.number,
cluster_id=cluster.id))
if proceed and request.form['action'] == 'create new pool':
try:
if request.form['new_pool_name'] is None or request.form[
'new_pool_name'] == '':
raise Exception('Pool name cannot be blank')
pool = VirtualMachinePool(name=request.form['new_pool_name'],
cluster=cluster,
zone_number=zone.number,
cardinality=len(selected_vm_ids))
Session.add(pool)
Session.flush()
for vm_id in selected_vm_ids.keys():
membership = PoolMembership(pool=pool,
vm_name=id_to_vm[vm_id].name,
vm_id=vm_id,
date_added=datetime.utcnow())
memberships[vm_id] = membership
Session.add(membership)
Session.flush()
Session.commit()
flash(Markup(
'Successfully created <a href="{}">{}</a> with {} pool members'
.format(url_for('vpool_bp.view', pool_id=pool.id),
pool.name, len(selected_vm_ids))),
category='success')
except Exception as e:
Session.rollback()
flash('Error creating your new pool: {}'.format(e),
category='danger')
return render_template('vpool/assign_to_pool.html',
form=form,
zone=zone,
cluster=cluster,
vms=vms,
memberships=memberships,
selected_vm_ids=selected_vm_ids,
pools=pools,
active_tab_name=active_tab)
def update(self, pool, pool_ticket, issue, cowboy_mode=False):
pool = Session.merge(pool)
pool_ticket = Session.merge(pool_ticket)
self.task = Session.merge(self.task)
one_proxy = OneProxy(pool.cluster.zone.xmlrpc_uri,
pool.cluster.zone.session_string,
verify_certs=False)
try:
c_start = JiraApi.get_now()
jira.start_crq(issue, log=self.log, cowboy_mode=cowboy_mode)
for t in issue.fields.subtasks:
t_start = JiraApi.get_now()
t2 = jira.instance.issue(t.key)
jira.start_task(t2, log=self.log, cowboy_mode=cowboy_mode)
updated_members = []
for a in t2.fields.attachment:
pool_id, vm_id = a.filename.split('.', 2)[:2]
template = a.get().decode(encoding="utf-8", errors="strict")
member = PoolMembership.query.filter_by(pool=pool,
vm_id=vm_id).first()
vm_name = member.vm_name
member.retire()
Session.delete(member)
Session.commit()
new_id = one_proxy.create_vm(template=template)
new_member = PoolMembership(pool=pool,
vm_name=vm_name,
vm_id=new_id,
template=template,
date_added=datetime.utcnow())
Session.add(new_member)
Session.commit()
self.log.msg(
"Instantiated new VM ID {} and added as member of pool {}".
format(new_member.vm_id, pool.name))
updated_members.append(new_member)
self.log.msg(
"waiting for 120 seconds before running post change diagnostics"
)
time.sleep(120)
run_diagnostics_on_pool(pool, self.log)
jira.complete_task(t,
start_time=t_start,
log=self.log,
cowboy_mode=cowboy_mode)
jira.complete_crq(issue,
start_time=c_start,
log=self.log,
cowboy_mode=cowboy_mode)
except Exception as e:
self.log.err("Error occured: {}".format(e))
jira.cancel_crq_and_tasks(
issue,
comment="an exception occured running this change: {}".format(e))
raise e
finally:
pool_ticket.done = True
Session.merge(pool_ticket)
Session.commit()
def update(self, pool, pool_ticket, issue, cowboy_mode=False):
pool = Session.merge(pool)
pool_ticket = Session.merge(pool_ticket)
self.task = Session.merge(self.task)
one_proxy = OneProxy(pool.cluster.zone.xmlrpc_uri, pool.cluster.zone.session_string, verify_certs=False)
try:
c_start = JiraApi.get_now()
jira.start_crq(issue, log=self.log, cowboy_mode=cowboy_mode)
for t in issue.fields.subtasks:
t_start = JiraApi.get_now()
t2 = jira.instance.issue(t.key)
jira.start_task(t2, log=self.log, cowboy_mode=cowboy_mode)
updated_members = []
for a in t2.fields.attachment:
pool_id, vm_id = a.filename.split('.', 2)[:2]
template = a.get().decode(encoding="utf-8", errors="strict")
member = PoolMembership.query.filter_by(pool=pool, vm_id=vm_id).first()
vm_name = member.vm_name
member.retire()
Session.delete(member)
Session.commit()
new_id = one_proxy.create_vm(template=template)
new_member = PoolMembership(pool=pool, vm_name=vm_name, vm_id=new_id, template=template, date_added=datetime.utcnow())
Session.add(new_member)
Session.commit()
self.log.msg("Instantiated new VM ID {} and added as member of pool {}".format(new_member.vm_id, pool.name))
updated_members.append(new_member)
self.log.msg("waiting for 120 seconds before running post change diagnostics")
time.sleep(120)
run_diagnostics_on_pool(pool, self.log)
jira.complete_task(t, start_time=t_start, log=self.log, cowboy_mode=cowboy_mode)
jira.complete_crq(issue, start_time=c_start, log=self.log, cowboy_mode=cowboy_mode)
except Exception as e:
self.log.err("Error occured: {}".format(e))
jira.cancel_crq_and_tasks(issue, comment="an exception occured running this change: {}".format(e))
raise e
finally:
pool_ticket.done = True
Session.merge(pool_ticket)
Session.commit()
def plan_update(self, pool, update_members):
task = crq = None
try:
pool = Session.merge(pool)
start, end = jira.next_immediate_window_dates()
logging = jira.instance.issue('SVC-1020')
crq = jira.instance.create_issue(
project=app.config['JIRA_CRQ_PROJECT'],
issuetype={'name': 'Change Request'},
assignee={'name': app.config['JIRA_USERNAME']},
summary="[IMPLEMENT] {}".format(self.task.name),
description=self.task.description,
customfield_14530=start,
customfield_14531=end,
customfield_19031={'value': 'Maintenance'},
customfield_15152=[{
'value': 'Global'
}],
customfield_19430={'value': 'No conflict with any restrictions'},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_17679="Pool update required")
self.log.msg("Created change request: {}".format(crq.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNING'])
self.log.msg("Transitioned {} to planning".format(crq.key))
jira.instance.create_issue_link('Relate', crq, logging)
self.log.msg("Related {} to LOGGING service {}".format(
crq.key, logging.key))
batch_size = floor(
len(update_members) / app.config['BATCH_SIZE_PERCENT'])
batch_size = 1 if batch_size == 0 else batch_size
num_batches = min(len(update_members),
app.config['BATCH_SIZE_PERCENT'])
self.log.msg(
"updating {} hosts in pool {} requires {} tasks with no more than {} hosts per task"
.format(len(update_members), pool.name, num_batches, batch_size))
batch_num = 0
while len(update_members):
batch_num += 1
task = jira.instance.create_issue(
issuetype={'name': 'MOP Task'},
assignee={'name': app.config['JIRA_USERNAME']},
project=app.config['JIRA_CRQ_PROJECT'],
summary='[TASK {}/{} (Update {}%)] {}'.format(
batch_num, num_batches, app.config['BATCH_SIZE_PERCENT'],
self.task.name),
parent={'key': crq.key},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_15150={'value': 'No'})
self.log.msg("Created task: {}".format(task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_PLANNING'])
self.log.msg("Transitioned {} to planning".format(task.key))
env = Environment(loader=ObjectLoader())
for num in range(0, min(len(update_members), batch_size)):
m = update_members.pop()
filename = '{}.{}.template'.format(m.pool.id, m.vm_id)
attachment_content = io.StringIO(m.current_template())
jira.instance.add_attachment(issue=task,
filename=filename,
attachment=attachment_content)
self.log.msg("Attached template for {} to task {}".format(
filename, task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_WRITTEN'])
self.log.msg("Transitioned task {} to written".format(task.key))
jira.approver_instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_APPROVED'])
self.log.msg("Approved task {}".format(task.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNED_CHANGE'])
self.log.msg("Transitioned change request {} to approved".format(
task.key))
jira.approver_instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_APPROVED'])
self.log.msg("Transitioned change request {} to approved".format(
crq.key))
self.log.msg("Task ID {}".format(self.task.id))
db_ticket = PoolTicket(pool=pool,
action_id=PoolTicketActions.update.value,
ticket_key=crq.key,
task=Session.merge(self.task))
Session.add(db_ticket)
Session.commit()
except Exception as e:
Session.rollback()
if crq is not None:
jira.cancel_crq_and_tasks(
crq, comment="failure creating change tickets")
raise e
def assign_to_pool(zone_number, cluster_id):
vms = []
id_to_vm = {}
selected_vm_ids = {}
pools = None
zone = None
cluster = None
memberships = {}
try:
Session()
zone = Zone.query.get(zone_number)
cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first()
one_proxy = OneProxy(zone.xmlrpc_uri, zone.session_string, verify_certs=False)
for membership in PoolMembership.query.join(VirtualMachinePool).filter_by(cluster=cluster).all():
memberships[membership.vm_id] = membership
for vm in one_proxy.get_vms():
if vm.disk_cluster.id == cluster.id:
vms.append(vm)
id_to_vm[vm.id] = vm
pools = VirtualMachinePool.get_all(cluster)
except Exception as e:
# raise e
flash("Error fetching VMs in zone number {}: {}"
.format(zone.number, e), category='danger')
form = ActionForm()
active_tab = 'create_new_pool'
# Form submission handling begins
if form.validate_on_submit():
# Determine which tab needs to be active based on the action
if request.form['action'] is not None:
print('something')
active_tab = {
'create new pool': 'create_new_pool',
'add to pool': 'add_to_existing_pool'}[request.form['action']]
# Get a list of clusters of all selected VMs--pools cannot span clusters
selected_clusters = {}
for id in request.form.getlist('chk_vm_id'):
selected_vm_ids[int(id)] = id
selected_clusters[id_to_vm[int(id)].disk_cluster.id] = True
# Error checking begins
proceed = True
if len(selected_vm_ids) == 0:
flash("No virtual machines were selected!", category='danger')
proceed = False
elif len(selected_clusters) != 1:
flash("Selected VMs must all be in the same cluster", category='danger')
proceed = False
# Handle the appropriate action if able to proceed
if proceed and request.form['action'] == 'add to pool':
if (request.form['pool_id']) is None or request.form['pool_id'] == '':
flash('No pool selected', category='danger')
else:
pool = VirtualMachinePool.query.get(request.form['pool_id'])
for vm_id in selected_vm_ids.keys():
Session.add(PoolMembership(pool=pool, vm_name=id_to_vm[vm_id].name, vm_id=vm_id, date_added=datetime.utcnow()))
Session.commit()
flash(Markup('Successfully added {} members to pool <a href="{}">{}</a>'.format(
len(selected_vm_ids),
url_for('vpool_bp.view', pool_id=pool.id),
pool.name, )), category='success')
return redirect(url_for('vpool_bp.assign_to_pool', zone_number=zone.number, cluster_id=cluster.id))
if proceed and request.form['action'] == 'create new pool':
try:
if request.form['new_pool_name'] is None or request.form['new_pool_name'] == '':
raise Exception('Pool name cannot be blank')
pool = VirtualMachinePool(
name=request.form['new_pool_name'],
cluster=cluster,
zone_number=zone.number,
cardinality=len(selected_vm_ids))
Session.add(pool)
Session.flush()
for vm_id in selected_vm_ids.keys():
membership = PoolMembership(pool=pool, vm_name=id_to_vm[vm_id].name, vm_id=vm_id, date_added=datetime.utcnow())
memberships[vm_id] = membership
Session.add(membership)
Session.flush()
Session.commit()
flash(Markup('Successfully created <a href="{}">{}</a> with {} pool members'.format(
url_for('vpool_bp.view', pool_id=pool.id),
pool.name, len(selected_vm_ids))), category='success')
except Exception as e:
Session.rollback()
flash('Error creating your new pool: {}'.format(e), category='danger')
return render_template(
'vpool/assign_to_pool.html',
form=form,
zone=zone,
cluster=cluster,
vms=vms,
memberships=memberships,
selected_vm_ids=selected_vm_ids,
pools=pools,
active_tab_name=active_tab
)
def pending_ticket(self):
return Session.query(PoolTicket).filter_by(pool=self,
done=False).first()
def get_all(cluster):
return Session.query(VirtualMachinePool).filter_by(cluster=cluster)
def get_peer_pools(self):
return Session.query(VirtualMachinePool).filter_by(
cluster=self.cluster)
def get_all(zone):
return Session().query(PoolMembership).join(
PoolMembership.pool, aliased=True).filter_by(zone=zone)
def get_all(cluster): return Session.query(VirtualMachinePool).filter_by(cluster=cluster)
def remove_done(pool_id):
form = ActionForm()
pool = one_proxy = members = None
try:
pool = VirtualMachinePool.query.get(pool_id)
members = pool.get_memberships()
except Exception as e:
flash("There was an error finshed VMs: {}".format(e),
category='danger')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
if request.method == 'POST' and form.validate():
try:
if request.form['action'] == 'cancel':
flash('Cleanup of {} cancelled'.format(pool.name),
category='info')
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
elif request.form['action'] == 'confirm':
vm_ids_to_delete = [
int(id) for id in request.form.getlist('done_vm_ids')
]
delete_members = []
Session()
for m in members:
if m.vm.id in vm_ids_to_delete:
delete_members.append(m)
delete_ticket = jira.instance.create_issue(
project=app.config['JIRA_PROJECT'],
summary='[auto-{}] Pool Cleanup: {} (deleting {} done VMs)'
.format(current_user.username, pool.name,
len(vm_ids_to_delete)),
description=
"Pool cleanup triggered that will delete {} VM(s): \n\n*{}"
.format(
len(vm_ids_to_delete), "\n*".join([
'ID {}: {} ({})'.format(m.vm.id, m.vm.name,
m.vm.ip_address)
for m in delete_members
])),
customfield_13842=jira.get_datetime_now(),
issuetype={'name': 'Task'})
one_proxy = OneProxy(pool.cluster.zone.xmlrpc_uri,
pool.cluster.zone.session_string,
verify_certs=False)
for m in delete_members:
one_proxy.action_vm(m.remove_cmd(), m.vm.id)
Session.delete(m)
Session.commit()
flash('Deleted {} done VMs to cleanup pool {}'.format(
len(delete_members), pool.name))
jira.resolve(delete_ticket)
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
except Exception as e:
flash("Error performing cleanup of pool {}: {}".format(
pool.name, e),
category='danger')
jira.defect_for_exception(
"Error during cleanup of pool {}".format(pool.name), e)
return redirect(url_for('vpool_bp.view', pool_id=pool.id))
return render_template('vpool/remove_done.html',
form=form,
pool=pool,
members=members)
def get_peer_pools(self): return Session.query(VirtualMachinePool).filter_by(cluster=self.cluster)
def run_task(self, **kwargs):
Session.merge(self.task)
self.task.start_time = datetime.utcnow()
self.task.ident = threading.get_ident()
self.task.status = TaskStatus.running.value
Session.merge(self.task)
Session.commit()
try:
self.run_function(**kwargs)
self.task.log = self.log.messages
self.task.end_time = datetime.utcnow()
self.task.status = TaskStatus.finished.value
self.task.result = TaskResult.success.value
self.task = Session.merge(self.task)
Session.commit()
except Exception as e:
self.task.log = self.log.messages
self.task.tb = traceback.format_exc()
self.task.end_time = datetime.utcnow()
self.task.status = TaskStatus.finished.value
self.task.result = TaskResult.fail.value
self.task = Session.merge(self.task)
Session.commit()
defect = jira.defect_for_exception(
"Background Task Error: {}".format(
self.task.name),
e, tb=traceback.format_exc(),
username=self.task.username)
self.task.defect_ticket = defect.key
self.task = Session.merge(self.task)
Session.commit()
finally:
Session.remove()
def run_task(self, **kwargs):
Session.merge(self.task)
self.task.start_time = datetime.utcnow()
self.task.ident = threading.get_ident()
self.task.status = TaskStatus.running.value
Session.merge(self.task)
Session.commit()
try:
self.run_function(**kwargs)
self.task.log = self.log.messages
self.task.end_time = datetime.utcnow()
self.task.status = TaskStatus.finished.value
self.task.result = TaskResult.success.value
self.task = Session.merge(self.task)
Session.commit()
except Exception as e:
self.task.log = self.log.messages
self.task.tb = traceback.format_exc()
self.task.end_time = datetime.utcnow()
self.task.status = TaskStatus.finished.value
self.task.result = TaskResult.fail.value
self.task = Session.merge(self.task)
Session.commit()
defect = jira.defect_for_exception(
"Background Task Error: {}".format(self.task.name),
e,
tb=traceback.format_exc(),
username=self.task.username)
self.task.defect_ticket = defect.key
self.task = Session.merge(self.task)
Session.commit()
finally:
Session.remove()
def plan_expansion(self, pool, expansion_names):
task = crq = None
try:
pool = Session.merge(pool)
start, end = jira.next_immediate_window_dates()
logging = jira.instance.issue('SVC-1020')
crq = jira.instance.create_issue(
project=app.config['JIRA_CRQ_PROJECT'],
issuetype={'name': 'Change Request'},
assignee={'name': app.config['JIRA_USERNAME']},
summary='[TEST IMPLEMENT] {}'.format(self.task.name),
description=self.task.description,
customfield_14530=start,
customfield_14531=end,
customfield_19031={'value': 'Maintenance'},
customfield_15152=[{
'value': 'Global'
}],
customfield_19430={'value': 'No conflict with any restrictions'},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_17679="Pool expansion required")
self.log.msg("Created change request: {}".format(crq.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNING'])
self.log.msg("Transitioned {} to planning".format(crq.key))
jira.instance.create_issue_link('Relate', crq, logging)
self.log.msg("Related {} to LOGGING service {}".format(
crq.key, logging.key))
task = jira.instance.create_issue(
issuetype={'name': 'MOP Task'},
assignee={'name': app.config['JIRA_USERNAME']},
project=app.config['JIRA_CRQ_PROJECT'],
description=
"Instanitate the attached templates in the zone associated "
"to the pool identified in the filename <pool_id>.<hostname>",
summary='[TEST IMPLEMENTATION TASK] {}'.format(self.task.name),
parent={'key': crq.key},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_15150={'value': 'No'})
self.log.msg("Created task: {}".format(task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_PLANNING'])
self.log.msg("Transitioned {} to planning".format(task.key))
env = Environment(loader=ObjectLoader())
for hostname in expansion_names:
vars = VarParser.parse_kv_strings_to_dict(
pool.cluster.zone.vars, pool.cluster.vars, pool.vars,
'hostname={}'.format(hostname))
vm_template = env.from_string(pool.template).render(pool=pool,
vars=vars)
attachment_content = io.StringIO(vm_template)
jira.instance.add_attachment(issue=task,
filename='{}.{}.template'.format(
pool.id, hostname),
attachment=attachment_content)
self.log.msg("Attached template for {} to task {}".format(
hostname, task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_WRITTEN'])
self.log.msg("Transitioned task {} to written".format(task.key))
jira.approver_instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_APPROVED'])
self.log.msg("Approved task {}".format(task.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNED_CHANGE'])
self.log.msg("Transitioned task {} to approved".format(task.key))
jira.approver_instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_APPROVED'])
self.log.msg("Transitioned change request {} to approved".format(
crq.key))
self.log.msg("Task ID {}".format(self.task.id))
db_ticket = PoolTicket(pool=Session.merge(pool),
action_id=PoolTicketActions.expand.value,
ticket_key=crq.key,
task=Session.merge(self.task))
Session.add(db_ticket)
Session.commit()
except Exception as e:
Session.rollback()
if crq is not None:
jira.cancel_crq_and_tasks(
crq, comment="failure creating change tickets")
raise e
def shutdown_session(exception=None): Session.remove()
def shrink(self, pool, pool_ticket, issue, cowboy_mode=False):
pool = Session.merge(pool)
pool_ticket = Session.merge(pool_ticket)
self.task = Session.merge(self.task)
try:
c_start = JiraApi.get_now()
jira.start_crq(issue, log=self.log, cowboy_mode=cowboy_mode)
for t in issue.fields.subtasks:
t_start = JiraApi.get_now()
t2 = jira.instance.issue(t.key)
jira.start_task(t2, log=self.log, cowboy_mode=cowboy_mode)
for a in t2.fields.attachment:
pool_id, vm_id = a.filename.split('.', 2)[:2]
member = PoolMembership.query.filter_by(pool=pool,
vm_id=vm_id).first()
member.retire()
Session.delete(member)
Session.commit()
self.log.msg(
"Retired VM {} and removed it as member of pool {}".format(
member.vm_id, pool.name))
jira.complete_task(t,
start_time=t_start,
log=self.log,
cowboy_mode=cowboy_mode)
self.log.msg(
"waiting for 120 seconds before running post task diagnostics")
time.sleep(120)
run_diagnostics_on_pool(pool, self.log)
jira.complete_crq(issue,
start_time=c_start,
log=self.log,
cowboy_mode=cowboy_mode)
except Exception as e:
self.log.err("Error occured: {}".format(e))
jira.cancel_crq_and_tasks(
issue,
comment="an exception occured running this change: {}".format(e))
raise e
finally:
pool_ticket.done = True
Session.merge(pool_ticket)
Session.commit()
def plan_shrink(self, pool, shrink_members):
task = crq = None
try:
pool = Session.merge(pool)
start, end = jira.next_immediate_window_dates()
logging = jira.instance.issue('SVC-1020')
crq = jira.instance.create_issue(
project=app.config['JIRA_CRQ_PROJECT'],
issuetype={'name': 'Change Request'},
assignee={'name': app.config['JIRA_USERNAME']},
summary="[IMPLEMENT] {}".format(self.task.name),
description=self.task.description,
customfield_14530=start,
customfield_14531=end,
customfield_19031={'value': 'Maintenance'},
customfield_15152=[{
'value': 'Global'
}],
customfield_19430={'value': 'No conflict with any restrictions'},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_17679="Pool shrink required")
self.log.msg("Created change request: {}".format(crq.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNING'])
self.log.msg("Transitioned {} to planning".format(crq.key))
jira.instance.create_issue_link('Relate', crq, logging)
self.log.msg("Related {} to LOGGING service {}".format(
crq.key, logging.key))
task = jira.instance.create_issue(
issuetype={'name': 'MOP Task'},
assignee={'name': app.config['JIRA_USERNAME']},
project=app.config['JIRA_CRQ_PROJECT'],
summary='[IMPLEMENTATION TASK] {}'.format(self.task.name),
parent={'key': crq.key},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_15150={'value': 'No'})
self.log.msg("Created task: {}".format(task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_PLANNING'])
self.log.msg("Transitioned {} to planning".format(task.key))
for m in [Session.merge(m) for m in shrink_members]:
filename = '{}.{}.template'.format(pool.id, m.vm_id)
attachment_content = io.StringIO(m.template)
jira.instance.add_attachment(issue=task,
filename=filename,
attachment=attachment_content)
self.log.msg("Attached member {} to shrink to task {}".format(
filename, task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_WRITTEN'])
self.log.msg("Transitioned task {} to written".format(task.key))
jira.approver_instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_APPROVED'])
self.log.msg("Approved task {}".format(task.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNED_CHANGE'])
self.log.msg("Transitioned task {} to approved".format(task.key))
jira.approver_instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_APPROVED'])
self.log.msg("Transitioned change request {} to approved".format(
crq.key))
self.log.msg("Task ID {}".format(self.task.id))
db_ticket = PoolTicket(pool=pool,
action_id=PoolTicketActions.shrink.value,
ticket_key=crq.key,
task=Session.merge(self.task))
Session.add(db_ticket)
Session.commit()
except Exception as e:
if crq is not None:
jira.cancel_crq_and_tasks(
crq, comment="failure creating change tickets")
raise e
def shrink(self, pool, pool_ticket, issue, cowboy_mode=False):
pool = Session.merge(pool)
pool_ticket = Session.merge(pool_ticket)
self.task = Session.merge(self.task)
try:
c_start = JiraApi.get_now()
jira.start_crq(issue, log=self.log, cowboy_mode=cowboy_mode)
for t in issue.fields.subtasks:
t_start = JiraApi.get_now()
t2 = jira.instance.issue(t.key)
jira.start_task(t2, log=self.log, cowboy_mode=cowboy_mode)
for a in t2.fields.attachment:
pool_id, vm_id = a.filename.split('.', 2)[:2]
member = PoolMembership.query.filter_by(pool=pool, vm_id=vm_id).first()
member.retire()
Session.delete(member)
Session.commit()
self.log.msg("Retired VM {} and removed it as member of pool {}".format(member.vm_id, pool.name))
jira.complete_task(t, start_time=t_start, log=self.log, cowboy_mode=cowboy_mode)
self.log.msg("waiting for 120 seconds before running post task diagnostics")
time.sleep(120)
run_diagnostics_on_pool(pool, self.log)
jira.complete_crq(issue, start_time=c_start, log=self.log, cowboy_mode=cowboy_mode)
except Exception as e:
self.log.err("Error occured: {}".format(e))
jira.cancel_crq_and_tasks(issue, comment="an exception occured running this change: {}".format(e))
raise e
finally:
pool_ticket.done = True
Session.merge(pool_ticket)
Session.commit()