def edit_template(zone_number): zone = Zone.query.get(zone_number) clusters = Cluster.query.filter_by(zone=zone).all() form = ZoneTemplateForm(request.form, obj=zone) if request.method == 'POST': if request.form['action'] == "cancel": flash('Cancelled {} template update'.format(zone.name), category="info") return redirect(url_for('zone_bp.view', zone_number=zone.number)) elif request.form['action'] == "save": if form.validate(): try: zone.template = request.form['template'] zone.vars = request.form['vars'] Session.add(zone) Session.commit() flash( 'Successfully saved template for {}.'.format( zone.name), 'success') return redirect( url_for('zone_bp.view', zone_number=zone.number)) except Exception as e: flash('Failed to save zone template, error: {}'.format(e), 'danger') if form.errors: flash("Errors must be resolved before zone template can be saved", 'danger') return render_template('zone/template.html', form=form, zone=zone, clusters=clusters)
def worker(q, number): while True: work = q.get() member = Session.merge(work['member']) vm = work['vm'] diagnostic = work['diagnostic'] try: logging.info("[{}] about to run diagnostic on {}".format(number, diagnostic.host)) diagnostic.run() logging.info("[{}] finished running diagnostic on {} (exit code: {})".format(number, vm.id, diagnostic.exitcode)) Session.add(PoolMemberDiagnostic( vm_id=vm.id, pool=member.pool, start_date=diagnostic.start_date, end_date=diagnostic.end_date, stdout=diagnostic.stdout, stderr=diagnostic.stderr, exitcode=diagnostic.exitcode)) Session.commit() logging.info("[{}] Saved diagnostic for {}".format(number, vm.id)) except Exception as e: defect_ticket = jira.defect_for_exception( summary_title="Lifeguard: Health Check => {})".format(e), tb=traceback.format_exc(), e=e) logging.error("[{} worker experienced an error running diagnostic " "host {}, error: {}, created defect ticket {}".format( number, vm.name, e, defect_ticket.key)) finally: q.task_done()
def diagnostic_worker(q, results, log): while True: work = q.get() member = Session.merge(work['member']) diagnostic = Diagnostic(user=app.config['SSH_HEALTH_CHECK_USER'], host=member.vm_name, ssh_identity_file=app.config['SSH_IDENTITY_FILE'], cmd=app.config['SSH_HEALTH_CHECK_CMD'], timeout=app.config['SSH_HEALTH_CHECK_TIMEOUT']) try: run_diagnostic(diagnostic) log.msg("diagnostic against {} succeeded".format(diagnostic.host)) except Exception as e: log.err("diagnostic error: {}".format(e)) finally: Session.add(PoolMemberDiagnostic( vm_id=member.vm_id, pool=member.pool, start_date=diagnostic.start_date, end_date=diagnostic.end_date, stdout=diagnostic.stdout, stderr=diagnostic.stderr, exitcode=diagnostic.exitcode)) Session.commit() results.append(diagnostic) q.task_done()
def diagnostic_worker(q, results, log): while True: work = q.get() member = Session.merge(work['member']) diagnostic = Diagnostic( user=app.config['SSH_HEALTH_CHECK_USER'], host=member.vm_name, ssh_identity_file=app.config['SSH_IDENTITY_FILE'], cmd=app.config['SSH_HEALTH_CHECK_CMD'], timeout=app.config['SSH_HEALTH_CHECK_TIMEOUT']) try: run_diagnostic(diagnostic) log.msg("diagnostic against {} succeeded".format(diagnostic.host)) except Exception as e: log.err("diagnostic error: {}".format(e)) finally: Session.add( PoolMemberDiagnostic(vm_id=member.vm_id, pool=member.pool, start_date=diagnostic.start_date, end_date=diagnostic.end_date, stdout=diagnostic.stdout, stderr=diagnostic.stderr, exitcode=diagnostic.exitcode)) Session.commit() results.append(diagnostic) q.task_done()
def login(): if current_user.is_authenticated: flash('You are already logged in.') return redirect(url_for('auth.home')) form = LoginForm(request.form) if request.method == 'POST' and form.validate(): username = request.form.get('username') password = request.form.get('password') try: User.try_login(username, password) except LDAPException: flash('Invalid username or password. Please try again.', 'danger') return render_template('auth/login.html', form=form) try: with Session.begin_nested(): user = User.query.filter_by(username=username).first() if not user: user = User(username) Session.add(user) Session.commit() login_user(user) flash('You have successfully logged in.', category='success') return redirect(url_for('auth.home')) except Exception as e: flash('There was an error logging in: {}'.format(e), category='danger') raise e if form.errors: flash(form.errors, 'danger') return render_template('auth/login.html', form=form)
def edit_template(zone_number, cluster_id): zone = Zone.query.get(zone_number) cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first() pools = VirtualMachinePool.query.filter_by(cluster_id=cluster.id, zone_number=cluster.zone_number).all() form = ClusterTemplateForm(request.form, obj=cluster) if request.method == 'POST': if request.form['action'] == "cancel": flash('Cancelled {} cluster template update'.format(cluster.name), category="info") return redirect(url_for('cluster_bp.view', zone_number=zone.number, cluster_id=cluster.id)) elif request.form['action'] == "save": try: cluster.template = request.form['template'] cluster.vars = request.form['vars'] Session.add(cluster) Session.commit() flash('Successfully saved cluster template for {} (ID={}).' .format(cluster.name, cluster.id), 'success') return redirect(url_for('cluster_bp.view', zone_number=zone.number, cluster_id=cluster.id)) except Exception as e: flash('Failed to save cluster template, error: {}'.format(e), 'danger') if form.errors: flash("Errors must be resolved before cluster template can be saved", 'danger') return render_template('cluster/template.html', form=form, cluster=cluster, pools=pools)
def login(): if current_user.is_authenticated: flash('You are already logged in.') return redirect(url_for('auth.home')) form = LoginForm(request.form) if request.method == 'POST' and form.validate(): username = request.form.get('username') password = request.form.get('password') try: User.try_login(username, password) except LDAPException: flash( 'Invalid username or password. Please try again.', 'danger') return render_template('auth/login.html', form=form) try: with Session.begin_nested(): user = User.query.filter_by(username=username).first() if not user: user = User(username) Session.add(user) Session.commit() login_user(user) flash('You have successfully logged in.', category='success') return redirect(url_for('auth.home')) except Exception as e: flash('There was an error logging in: {}'.format(e), category='danger') raise e if form.errors: flash(form.errors, 'danger') return render_template('auth/login.html', form=form)
def update(self, pool, pool_ticket, issue, cowboy_mode=False): pool = Session.merge(pool) pool_ticket = Session.merge(pool_ticket) self.task = Session.merge(self.task) one_proxy = OneProxy(pool.cluster.zone.xmlrpc_uri, pool.cluster.zone.session_string, verify_certs=False) try: c_start = JiraApi.get_now() jira.start_crq(issue, log=self.log, cowboy_mode=cowboy_mode) for t in issue.fields.subtasks: t_start = JiraApi.get_now() t2 = jira.instance.issue(t.key) jira.start_task(t2, log=self.log, cowboy_mode=cowboy_mode) updated_members = [] for a in t2.fields.attachment: pool_id, vm_id = a.filename.split('.', 2)[:2] template = a.get().decode(encoding="utf-8", errors="strict") member = PoolMembership.query.filter_by(pool=pool, vm_id=vm_id).first() vm_name = member.vm_name member.retire() Session.delete(member) Session.commit() new_id = one_proxy.create_vm(template=template) new_member = PoolMembership(pool=pool, vm_name=vm_name, vm_id=new_id, template=template, date_added=datetime.utcnow()) Session.add(new_member) Session.commit() self.log.msg( "Instantiated new VM ID {} and added as member of pool {}". format(new_member.vm_id, pool.name)) updated_members.append(new_member) self.log.msg( "waiting for 120 seconds before running post change diagnostics" ) time.sleep(120) run_diagnostics_on_pool(pool, self.log) jira.complete_task(t, start_time=t_start, log=self.log, cowboy_mode=cowboy_mode) jira.complete_crq(issue, start_time=c_start, log=self.log, cowboy_mode=cowboy_mode) except Exception as e: self.log.err("Error occured: {}".format(e)) jira.cancel_crq_and_tasks( issue, comment="an exception occured running this change: {}".format(e)) raise e finally: pool_ticket.done = True Session.merge(pool_ticket) Session.commit()
def shrink(pool_id): pool = form_shrink_vm_ids = None form = ActionForm() try: pool = VirtualMachinePool.query.get(pool_id) if request.method == 'POST' and request.form['action'] == 'cancel': flash('Shrink {} cancelled'.format(pool.name), category='info') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) elif request.method == 'POST' and request.form['action'] == 'shrink': form_shrink_vm_ids = [ int(id) for id in request.form.getlist('shrink_vm_ids') ] members = pool.get_memberships() shrink_members = pool.get_members_to_shrink(members, form_shrink_vm_ids) if shrink_members is None or len(shrink_members) == 0: raise Exception( "Cannot determine any members to shutdown for shrinking") except Exception as e: flash("There was an error determining memberships for shrinking: {}". format(e), category='danger') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) if request.method == 'POST' and form.validate( ) and request.form['action'] == 'shrink': title = 'Plan Shrink => Pool {} ({} members to {})'.format( pool.name, len(members), pool.cardinality) description = "Pool shrink triggered that will shutdown {} VM(s): \n\n*{}".format( len(shrink_members), "\n*".join([m.vm.name for m in shrink_members])) task = Task(name=title, description="{}\n{}".format(title, description), username=current_user.username) Session.add(task) Session.commit() task_thread = TaskThread(task_id=task.id, run_function=plan_shrink, pool=pool, shrink_members=shrink_members) task_thread.start() flash( Markup("Started background task {}: {}".format( task.name, task.link()))) return redirect(url_for('vpool_bp.view', pool_id=pool.id)) return render_template('vpool/shrink.html', form=form, pool=pool, shrink_members=shrink_members)
def create_new_pool(zone_number, cluster_id): try: zone = Zone.query.get(zone_number) cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first() form = pool = None pool = VirtualMachinePool(zone_number=cluster.zone.number, cluster=cluster) pools = VirtualMachinePool.query.filter_by( cluster_id=cluster.id, zone_number=cluster.zone_number).all() form = PoolEditForm(request.form, obj=pool) except Exception as e: flash( "There was an error fetching objects required for creating pool: {}" .format(e)) return redirect( url_for('cluster_bp.view', zone_number=zone.number, cluster_id=cluster.id)) if request.method == 'POST': if request.form['action'] == "cancel": flash('Cancelled {} pool template update'.format(pool.name), category="info") return redirect(url_for('vpool_bp.view', pool_id=pool.id)) elif request.form['action'] == "save": try: cardinality_pattern = re.compile("\d+") pool.name = request.form['name'] pool.template = request.form['template'] pool.vars = request.form['vars'] if not cardinality_pattern.fullmatch( request.form['cardinality']): raise Exception("Cardinality {} not numeric".format( request.form['cardinality'])) pool.cardinality = request.form['cardinality'] Session.add(pool) Session.commit() flash( 'Successfully saved pool template for {} (ID={}).'.format( pool.name, pool.id), 'success') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) except Exception as e: flash('Failed to save pool error: {}'.format(e), 'danger') if form.errors: flash("Errors must be resolved before pool can be saved", 'danger') return render_template('vpool/create.html', form=form, pool=pool, pools=pools)
def edit(pool_id): form = pool = one_proxy = members = None try: pool = VirtualMachinePool.query.get(pool_id) form = PoolEditForm(request.form, obj=pool) members = pool.get_memberships() except Exception as e: flash( "There was an error fetching objects required for editing pool {}: {}" .format(pool.name, e)) return redirect(url_for('vpool_bp.view', pool_id=pool.id)) if request.method == 'POST': if request.form['action'] == "cancel": flash('Cancelled {} pool template update'.format(pool.name), category="info") return redirect(url_for('vpool_bp.view', pool_id=pool.id)) elif request.form['action'] == "save": try: cardinality_pattern = re.compile("\d+") pool.name = request.form['name'] pool.template = request.form['template'] pool.vars = request.form['vars'] if not cardinality_pattern.fullmatch( request.form['cardinality']): raise Exception("Cardinality {} not numeric".format( request.form['cardinality'])) pool.cardinality = request.form['cardinality'] Session.add(pool) for m in members: if m.template == 'not-yet-compiled': m.template = '' Session.add(m) Session.commit() flash( 'Successfully saved pool template for {} (ID={}).'.format( pool.name, pool.id), 'success') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) except Exception as e: flash('Failed to save pool template, error: {}'.format(e), 'danger') if form.errors: flash("Errors must be resolved before pool template can be saved", 'danger') return render_template('vpool/edit.html', form=form, members=members, pool=pool)
def expand(pool_id): pool = members = expansion_names = form_expansion_names = None form = ActionForm() try: pool = VirtualMachinePool.query.get(pool_id) if request.method == 'POST' and request.form['action'] == 'cancel': flash('Expansion of {} cancelled'.format(pool.name), category='info') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) elif request.method == 'POST' and request.form['action'] == 'expand': form_expansion_names = request.form.getlist('expansion_names') members = pool.get_memberships() expansion_names = pool.get_expansion_names(members, form_expansion_names) except Exception as e: flash( "There was an error determining new names required for expansion: {}" .format(e), category='danger') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) if request.method == 'POST' and form.validate( ) and request.form['action'] == 'expand': title = 'Plan Change => Pool Expansion: {} ({} members to {})'.format( pool.name, len(members), pool.cardinality) description = "Pool expansion triggered that will instantiate {} new VM(s): \n\n*{}".format( len(expansion_names), "\n*".join(expansion_names)) task = Task(name=title, description="{}\n{}".format(title, description), username=current_user.username) Session.add(task) Session.commit() task_thread = TaskThread(task_id=task.id, run_function=plan_expansion, pool=pool, expansion_names=expansion_names) task_thread.start() flash( Markup("Started background task {}: {}".format( task.name, task.link()))) return redirect(url_for('vpool_bp.view', pool_id=pool.id)) return render_template('vpool/expand.html', form=form, members=members, pool=pool, expansion_names=expansion_names)
def create_update_ticket(name, pool, members, update_members): logging.info("[{}] {} requires updating and an existing change ticket doesn't exist".format(name, pool.name)) title = 'Plan Update => Pool {} ({}/{} members need updates)'.format(pool.name, len(members), pool.cardinality) description = "Pool update triggered that will update {} VM(s): \n\n*{}".format( len(update_members), "\n*".join([m.vm.name for m in update_members])) task = Task( name=title, description="{}\n{}".format(title, description), username="******") Session.add(task) Session.commit() task_thread = TaskThread(task=task, run_function=plan_update, pool=Session.merge(pool), update_members=update_members) threads.append(task_thread) logging.info("[{}] launched background task {} to {}".format(name, task.id, title))
def create_expansion_ticket(name, pool, members, expansion_names): logging.info("[{}] {} requires expansion and an existing change ticket doesn't exist".format(name, pool.name)) title = 'Plan Change => Pool Expansion: {} ({} members to {})'.format(pool.name, len(members), pool.cardinality) description = "Pool expansion triggered that will instantiate {} new VM(s): \n\n*{}".format( len(expansion_names), "\n*".join(expansion_names)) task = Task( name=title, description="{}\n{}".format(title, description), username="******") Session.add(task) Session.commit() task_thread = TaskThread(task=task, run_function=plan_expansion, pool=Session.merge(pool), expansion_names=expansion_names) threads.append(task_thread) logging.info("[{}] launched background task {} to {}".format(name, task.id, title))
def shrink(pool_id): pool = form_shrink_vm_ids = None form = ActionForm() try: pool = VirtualMachinePool.query.get(pool_id) if request.method == 'POST' and request.form['action'] == 'cancel': flash('Shrink {} cancelled'.format(pool.name), category='info') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) elif request.method == 'POST' and request.form['action'] == 'shrink': form_shrink_vm_ids = [int(id) for id in request.form.getlist('shrink_vm_ids')] members = pool.get_memberships() shrink_members = pool.get_members_to_shrink(members, form_shrink_vm_ids) if shrink_members is None or len(shrink_members) == 0: raise Exception("Cannot determine any members to shutdown for shrinking") except Exception as e: flash("There was an error determining memberships for shrinking: {}".format(e), category='danger') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) if request.method == 'POST' and form.validate() and request.form['action'] == 'shrink': title = 'Plan Shrink => Pool {} ({} members to {})'.format(pool.name, len(members), pool.cardinality) description = "Pool shrink triggered that will shutdown {} VM(s): \n\n*{}".format( len(shrink_members), "\n*".join([m.vm.name for m in shrink_members])) task = Task( name=title, description="{}\n{}".format(title, description), username=current_user.username) Session.add(task) Session.commit() task_thread = TaskThread(task_id=task.id, run_function=plan_shrink, pool=pool, shrink_members=shrink_members) task_thread.start() flash(Markup("Started background task {}: {}".format(task.name, task.link()))) return redirect(url_for('vpool_bp.view', pool_id=pool.id)) return render_template('vpool/shrink.html', form=form, pool=pool, shrink_members=shrink_members)
def update(pool_id): form = ActionForm() pool = VirtualMachinePool.query.get(pool_id) form_update_ids = None if request.method == 'POST' and request.form['action'] == 'cancel': flash('Update of {} cancelled'.format(pool.name), category='info') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) elif request.method == 'POST' and request.form['action'] == 'update': form_update_ids = request.form.getlist('update_ids') members = pool.get_memberships() update_ids = pool.get_update_ids(members, form_update_ids) if request.method == 'POST' and form.validate( ) and request.form['action'] == 'update': id_to_template = {} for m in members: if m.vm.id in update_ids: id_to_template[m.vm.id] = m.current_template() title = "Plan Change => Update {} member(s) in pool {}".format( len(update_ids), pool.name) description = "The attached templates will replace the VMs identified in their filename (based on VM ID)" task = Task(name=title, description="{}\n{}".format(title, description), username=current_user.username) Session.add(task) Session.commit() task_thread = TaskThread(task_id=task.id, run_function=plan_update, pool=pool, id_to_template=id_to_template) task_thread.start() flash( Markup("Started background task {}: {}".format(title, task.link()))) return redirect(url_for('vpool_bp.view', pool_id=pool.id)) return render_template('vpool/update.html', form=form, pool=pool, members=members, update_ids=update_ids)
def edit(pool_id): form = pool = one_proxy = members = None try: pool = VirtualMachinePool.query.get(pool_id) form = PoolEditForm(request.form, obj=pool) members = pool.get_memberships() except Exception as e: flash("There was an error fetching objects required for editing pool {}: {}".format( pool.name, e)) return redirect(url_for('vpool_bp.view', pool_id=pool.id)) if request.method == 'POST': if request.form['action'] == "cancel": flash('Cancelled {} pool template update'.format(pool.name), category="info") return redirect(url_for('vpool_bp.view', pool_id=pool.id)) elif request.form['action'] == "save": try: cardinality_pattern = re.compile("\d+") pool.name = request.form['name'] pool.template = request.form['template'] pool.vars = request.form['vars'] if not cardinality_pattern.fullmatch(request.form['cardinality']): raise Exception("Cardinality {} not numeric".format(request.form['cardinality'])) pool.cardinality = request.form['cardinality'] Session.add(pool) for m in members: if m.template == 'not-yet-compiled': m.template = '' Session.add(m) Session.commit() flash('Successfully saved pool template for {} (ID={}).' .format(pool.name, pool.id), 'success') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) except Exception as e: flash('Failed to save pool template, error: {}'.format(e), 'danger') if form.errors: flash("Errors must be resolved before pool template can be saved", 'danger') return render_template('vpool/edit.html', form=form, members=members, pool=pool)
def update(self, pool, pool_ticket, issue, cowboy_mode=False): pool = Session.merge(pool) pool_ticket = Session.merge(pool_ticket) self.task = Session.merge(self.task) one_proxy = OneProxy(pool.cluster.zone.xmlrpc_uri, pool.cluster.zone.session_string, verify_certs=False) try: c_start = JiraApi.get_now() jira.start_crq(issue, log=self.log, cowboy_mode=cowboy_mode) for t in issue.fields.subtasks: t_start = JiraApi.get_now() t2 = jira.instance.issue(t.key) jira.start_task(t2, log=self.log, cowboy_mode=cowboy_mode) updated_members = [] for a in t2.fields.attachment: pool_id, vm_id = a.filename.split('.', 2)[:2] template = a.get().decode(encoding="utf-8", errors="strict") member = PoolMembership.query.filter_by(pool=pool, vm_id=vm_id).first() vm_name = member.vm_name member.retire() Session.delete(member) Session.commit() new_id = one_proxy.create_vm(template=template) new_member = PoolMembership(pool=pool, vm_name=vm_name, vm_id=new_id, template=template, date_added=datetime.utcnow()) Session.add(new_member) Session.commit() self.log.msg("Instantiated new VM ID {} and added as member of pool {}".format(new_member.vm_id, pool.name)) updated_members.append(new_member) self.log.msg("waiting for 120 seconds before running post change diagnostics") time.sleep(120) run_diagnostics_on_pool(pool, self.log) jira.complete_task(t, start_time=t_start, log=self.log, cowboy_mode=cowboy_mode) jira.complete_crq(issue, start_time=c_start, log=self.log, cowboy_mode=cowboy_mode) except Exception as e: self.log.err("Error occured: {}".format(e)) jira.cancel_crq_and_tasks(issue, comment="an exception occured running this change: {}".format(e)) raise e finally: pool_ticket.done = True Session.merge(pool_ticket) Session.commit()
def execute_tickets(name, pool, cowboy_mode=False): for t in pool.change_tickets: try: if t.done == False: logging.info("[{}] fetching {}".format(name, t.ticket_key)) crq = jira.instance.issue(t.ticket_key) if not cowboy_mode and JiraApi.expired(crq): logging.info("[{}] {} has expired".format(name, crq.key)) jira.cancel_crq_and_tasks(crq, "change has expired") t.done = True Session.add(t) logging.info("[{}] marked {} ticket {} as done for pool {} as crq is expired".format( name, t.action_name(), t.ticket_key, pool.name)) elif cowboy_mode or JiraApi.in_window(crq): logging.info("[{}] change {} is in window".format(name, crq.key)) if cowboy_mode or jira.is_ready(crq): title = "{} pool {} under ticket {}".format(t.action_name(), pool.name, crq.key) task = Task( name=title, description=title, username="******") Session.add(task) Session.commit() task_thread = TaskThread(task=task, run_function=get_runnable_from_action_id(t.action_id), pool=pool, pool_ticket=t, issue=crq, cowboy_mode=cowboy_mode) threads.append(task_thread) logging.info("[{}] launched background task {} to {}".format(name, task.id, title)) else: logging.error("[{}] {} is in window and either it or one or " "more sub tasks are not ready".format(name, crq.key)) else: logging.info("[{}] {} change {} for pool {} not yet in window".format( name, t.action_name(), crq.key, pool.name)) except Exception as e: logging.error("[{}] error executing {}: {}".format(name, t.ticket_key, e))
def expand(pool_id): pool = members = expansion_names = form_expansion_names = None form = ActionForm() try: pool = VirtualMachinePool.query.get(pool_id) if request.method == 'POST' and request.form['action'] == 'cancel': flash('Expansion of {} cancelled'.format(pool.name), category='info') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) elif request.method == 'POST' and request.form['action'] == 'expand': form_expansion_names = request.form.getlist('expansion_names') members = pool.get_memberships() expansion_names = pool.get_expansion_names(members, form_expansion_names) except Exception as e: flash("There was an error determining new names required for expansion: {}".format(e), category='danger') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) if request.method == 'POST' and form.validate() and request.form['action'] == 'expand': title = 'Plan Change => Pool Expansion: {} ({} members to {})'.format(pool.name, len(members), pool.cardinality) description = "Pool expansion triggered that will instantiate {} new VM(s): \n\n*{}".format( len(expansion_names), "\n*".join(expansion_names)) task = Task( name=title, description="{}\n{}".format(title, description), username=current_user.username) Session.add(task) Session.commit() task_thread = TaskThread(task_id=task.id, run_function=plan_expansion, pool=pool, expansion_names=expansion_names) task_thread.start() flash(Markup("Started background task {}: {}".format(task.name, task.link()))) return redirect(url_for('vpool_bp.view', pool_id=pool.id)) return render_template('vpool/expand.html', form=form, members=members, pool=pool, expansion_names=expansion_names)
def discover(zone_number): zone = Zone.query.get(zone_number) clusters = Cluster.query.filter_by(zone=zone).all() one_proxy = OneProxy(zone.xmlrpc_uri, zone.session_string, verify_certs=False) one_clusters = one_proxy.get_clusters() for one_cluster in one_clusters: existing_cluster = Cluster.query.filter_by(zone_number=zone.number, id=one_cluster.id).first() if existing_cluster is None: discovered_cluster = Cluster(id=one_cluster.id, zone=zone, name=one_cluster.name) Session.add(discovered_cluster) Session.commit() flash('Newly discovered ONE cluster: {} (ID={}) in zone {}'.format( one_cluster.id, one_cluster.name, zone.name), category='success') return render_template('zone/discover.html', zone=zone, one_clusters=one_clusters, clusters=clusters)
def edit_template(zone_number, cluster_id): zone = Zone.query.get(zone_number) cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first() pools = VirtualMachinePool.query.filter_by( cluster_id=cluster.id, zone_number=cluster.zone_number).all() form = ClusterTemplateForm(request.form, obj=cluster) if request.method == 'POST': if request.form['action'] == "cancel": flash('Cancelled {} cluster template update'.format(cluster.name), category="info") return redirect( url_for('cluster_bp.view', zone_number=zone.number, cluster_id=cluster.id)) elif request.form['action'] == "save": try: cluster.template = request.form['template'] cluster.vars = request.form['vars'] Session.add(cluster) Session.commit() flash( 'Successfully saved cluster template for {} (ID={}).'. format(cluster.name, cluster.id), 'success') return redirect( url_for('cluster_bp.view', zone_number=zone.number, cluster_id=cluster.id)) except Exception as e: flash('Failed to save cluster template, error: {}'.format(e), 'danger') if form.errors: flash("Errors must be resolved before cluster template can be saved", 'danger') return render_template('cluster/template.html', form=form, cluster=cluster, pools=pools)
def create_new_pool(zone_number, cluster_id): try: zone = Zone.query.get(zone_number) cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first() form = pool = None pool = VirtualMachinePool(zone_number=cluster.zone.number, cluster=cluster) pools = VirtualMachinePool.query.filter_by(cluster_id=cluster.id, zone_number=cluster.zone_number).all() form = PoolEditForm(request.form, obj=pool) except Exception as e: flash("There was an error fetching objects required for creating pool: {}".format(e)) return redirect(url_for('cluster_bp.view', zone_number=zone.number, cluster_id=cluster.id)) if request.method == 'POST': if request.form['action'] == "cancel": flash('Cancelled {} pool template update'.format(pool.name), category="info") return redirect(url_for('vpool_bp.view', pool_id=pool.id)) elif request.form['action'] == "save": try: cardinality_pattern = re.compile("\d+") pool.name = request.form['name'] pool.template = request.form['template'] pool.vars = request.form['vars'] if not cardinality_pattern.fullmatch(request.form['cardinality']): raise Exception("Cardinality {} not numeric".format(request.form['cardinality'])) pool.cardinality = request.form['cardinality'] Session.add(pool) Session.commit() flash('Successfully saved pool template for {} (ID={}).' .format(pool.name, pool.id), 'success') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) except Exception as e: flash('Failed to save pool error: {}'.format(e), 'danger') if form.errors: flash("Errors must be resolved before pool can be saved", 'danger') return render_template('vpool/create.html', form=form, pool=pool, pools=pools)
def update(pool_id): form = ActionForm() pool = VirtualMachinePool.query.get(pool_id) form_update_ids = None if request.method == 'POST' and request.form['action'] == 'cancel': flash('Update of {} cancelled'.format(pool.name), category='info') return redirect(url_for('vpool_bp.view', pool_id=pool.id)) elif request.method == 'POST' and request.form['action'] == 'update': form_update_ids = request.form.getlist('update_ids') members = pool.get_memberships() update_ids = pool.get_update_ids(members, form_update_ids) if request.method == 'POST' and form.validate() and request.form['action'] == 'update': id_to_template = {} for m in members: if m.vm.id in update_ids: id_to_template[m.vm.id] = m.current_template() title = "Plan Change => Update {} member(s) in pool {}".format(len(update_ids), pool.name) description = "The attached templates will replace the VMs identified in their filename (based on VM ID)" task = Task(name=title, description="{}\n{}".format(title, description), username=current_user.username) Session.add(task) Session.commit() task_thread = TaskThread(task_id=task.id, run_function=plan_update, pool=pool, id_to_template=id_to_template) task_thread.start() flash(Markup("Started background task {}: {}".format(title, task.link()))) return redirect(url_for('vpool_bp.view', pool_id=pool.id)) return render_template('vpool/update.html', form=form, pool=pool, members=members, update_ids=update_ids)
def manage(zone_number): zone = Zone() zones = Zone.query.order_by(Zone.number.desc()).all() template = 'zone/add.html' clusters = None if zone_number is not None: zone = Zone.query.get(zone_number) clusters = Cluster.query.filter_by(zone=zone) template = 'zone/edit.html' form = ZoneForm(request.form, obj=zone) if request.method == 'POST': if request.form['action'] == "cancel": if zone_number is not None: flash('{} edit cancelled'.format(zone.name), category="info") return redirect( url_for('zone_bp.view', zone_number=zone_number)) flash('Create zone cancelled', category="info") return redirect(url_for('zone_bp.list')) elif request.form['action'] == "save": if form.validate(): try: form.populate_obj(zone) Session.add(zone) Session.commit() flash('Successfully saved {}.'.format(zone.name), 'success') return redirect(url_for('zone_bp.list')) except Exception as e: flash('Failed to save zone, error: {}'.format(e), 'danger') if form.errors: flash("Errors must be resolved before zone can be saved", 'danger') return render_template(template, form=form, zone=zone, zones=zones, clusters=clusters)
def assign_to_pool(zone_number, cluster_id): vms = [] id_to_vm = {} selected_vm_ids = {} pools = None zone = None cluster = None memberships = {} try: Session() zone = Zone.query.get(zone_number) cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first() one_proxy = OneProxy(zone.xmlrpc_uri, zone.session_string, verify_certs=False) for membership in PoolMembership.query.join( VirtualMachinePool).filter_by(cluster=cluster).all(): memberships[membership.vm_id] = membership for vm in one_proxy.get_vms(): if vm.disk_cluster.id == cluster.id: vms.append(vm) id_to_vm[vm.id] = vm pools = VirtualMachinePool.get_all(cluster) except Exception as e: # raise e flash("Error fetching VMs in zone number {}: {}".format( zone.number, e), category='danger') form = ActionForm() active_tab = 'create_new_pool' # Form submission handling begins if form.validate_on_submit(): # Determine which tab needs to be active based on the action if request.form['action'] is not None: print('something') active_tab = { 'create new pool': 'create_new_pool', 'add to pool': 'add_to_existing_pool' }[request.form['action']] # Get a list of clusters of all selected VMs--pools cannot span clusters selected_clusters = {} for id in request.form.getlist('chk_vm_id'): selected_vm_ids[int(id)] = id selected_clusters[id_to_vm[int(id)].disk_cluster.id] = True # Error checking begins proceed = True if len(selected_vm_ids) == 0: flash("No virtual machines were selected!", category='danger') proceed = False elif len(selected_clusters) != 1: flash("Selected VMs must all be in the same cluster", category='danger') proceed = False # Handle the appropriate action if able to proceed if proceed and request.form['action'] == 'add to pool': if (request.form['pool_id'] ) is None or request.form['pool_id'] == '': flash('No pool selected', category='danger') else: pool = VirtualMachinePool.query.get(request.form['pool_id']) for vm_id in selected_vm_ids.keys(): Session.add( PoolMembership(pool=pool, vm_name=id_to_vm[vm_id].name, vm_id=vm_id, date_added=datetime.utcnow())) Session.commit() flash(Markup( 'Successfully added {} members to pool <a href="{}">{}</a>' .format( len(selected_vm_ids), url_for('vpool_bp.view', pool_id=pool.id), pool.name, )), category='success') return redirect( url_for('vpool_bp.assign_to_pool', zone_number=zone.number, cluster_id=cluster.id)) if proceed and request.form['action'] == 'create new pool': try: if request.form['new_pool_name'] is None or request.form[ 'new_pool_name'] == '': raise Exception('Pool name cannot be blank') pool = VirtualMachinePool(name=request.form['new_pool_name'], cluster=cluster, zone_number=zone.number, cardinality=len(selected_vm_ids)) Session.add(pool) Session.flush() for vm_id in selected_vm_ids.keys(): membership = PoolMembership(pool=pool, vm_name=id_to_vm[vm_id].name, vm_id=vm_id, date_added=datetime.utcnow()) memberships[vm_id] = membership Session.add(membership) Session.flush() Session.commit() flash(Markup( 'Successfully created <a href="{}">{}</a> with {} pool members' .format(url_for('vpool_bp.view', pool_id=pool.id), pool.name, len(selected_vm_ids))), category='success') except Exception as e: Session.rollback() flash('Error creating your new pool: {}'.format(e), category='danger') return render_template('vpool/assign_to_pool.html', form=form, zone=zone, cluster=cluster, vms=vms, memberships=memberships, selected_vm_ids=selected_vm_ids, pools=pools, active_tab_name=active_tab)
def assign_to_pool(zone_number, cluster_id): vms = [] id_to_vm = {} selected_vm_ids = {} pools = None zone = None cluster = None memberships = {} try: Session() zone = Zone.query.get(zone_number) cluster = Cluster.query.filter_by(zone=zone, id=cluster_id).first() one_proxy = OneProxy(zone.xmlrpc_uri, zone.session_string, verify_certs=False) for membership in PoolMembership.query.join(VirtualMachinePool).filter_by(cluster=cluster).all(): memberships[membership.vm_id] = membership for vm in one_proxy.get_vms(): if vm.disk_cluster.id == cluster.id: vms.append(vm) id_to_vm[vm.id] = vm pools = VirtualMachinePool.get_all(cluster) except Exception as e: # raise e flash("Error fetching VMs in zone number {}: {}" .format(zone.number, e), category='danger') form = ActionForm() active_tab = 'create_new_pool' # Form submission handling begins if form.validate_on_submit(): # Determine which tab needs to be active based on the action if request.form['action'] is not None: print('something') active_tab = { 'create new pool': 'create_new_pool', 'add to pool': 'add_to_existing_pool'}[request.form['action']] # Get a list of clusters of all selected VMs--pools cannot span clusters selected_clusters = {} for id in request.form.getlist('chk_vm_id'): selected_vm_ids[int(id)] = id selected_clusters[id_to_vm[int(id)].disk_cluster.id] = True # Error checking begins proceed = True if len(selected_vm_ids) == 0: flash("No virtual machines were selected!", category='danger') proceed = False elif len(selected_clusters) != 1: flash("Selected VMs must all be in the same cluster", category='danger') proceed = False # Handle the appropriate action if able to proceed if proceed and request.form['action'] == 'add to pool': if (request.form['pool_id']) is None or request.form['pool_id'] == '': flash('No pool selected', category='danger') else: pool = VirtualMachinePool.query.get(request.form['pool_id']) for vm_id in selected_vm_ids.keys(): Session.add(PoolMembership(pool=pool, vm_name=id_to_vm[vm_id].name, vm_id=vm_id, date_added=datetime.utcnow())) Session.commit() flash(Markup('Successfully added {} members to pool <a href="{}">{}</a>'.format( len(selected_vm_ids), url_for('vpool_bp.view', pool_id=pool.id), pool.name, )), category='success') return redirect(url_for('vpool_bp.assign_to_pool', zone_number=zone.number, cluster_id=cluster.id)) if proceed and request.form['action'] == 'create new pool': try: if request.form['new_pool_name'] is None or request.form['new_pool_name'] == '': raise Exception('Pool name cannot be blank') pool = VirtualMachinePool( name=request.form['new_pool_name'], cluster=cluster, zone_number=zone.number, cardinality=len(selected_vm_ids)) Session.add(pool) Session.flush() for vm_id in selected_vm_ids.keys(): membership = PoolMembership(pool=pool, vm_name=id_to_vm[vm_id].name, vm_id=vm_id, date_added=datetime.utcnow()) memberships[vm_id] = membership Session.add(membership) Session.flush() Session.commit() flash(Markup('Successfully created <a href="{}">{}</a> with {} pool members'.format( url_for('vpool_bp.view', pool_id=pool.id), pool.name, len(selected_vm_ids))), category='success') except Exception as e: Session.rollback() flash('Error creating your new pool: {}'.format(e), category='danger') return render_template( 'vpool/assign_to_pool.html', form=form, zone=zone, cluster=cluster, vms=vms, memberships=memberships, selected_vm_ids=selected_vm_ids, pools=pools, active_tab_name=active_tab )
def plan_shrink(self, pool, shrink_members): task = crq = None try: pool = Session.merge(pool) start, end = jira.next_immediate_window_dates() logging = jira.instance.issue('SVC-1020') crq = jira.instance.create_issue( project=app.config['JIRA_CRQ_PROJECT'], issuetype={'name': 'Change Request'}, assignee={'name': app.config['JIRA_USERNAME']}, summary="[IMPLEMENT] {}".format(self.task.name), description=self.task.description, customfield_14530=start, customfield_14531=end, customfield_19031={'value': 'Maintenance'}, customfield_15152=[{ 'value': 'Global' }], customfield_19430={'value': 'No conflict with any restrictions'}, customfield_14135={ 'value': 'IPG', 'child': { 'value': 'IPG Big Data' } }, customfield_17679="Pool shrink required") self.log.msg("Created change request: {}".format(crq.key)) jira.instance.transition_issue( crq, app.config['JIRA_TRANSITION_CRQ_PLANNING']) self.log.msg("Transitioned {} to planning".format(crq.key)) jira.instance.create_issue_link('Relate', crq, logging) self.log.msg("Related {} to LOGGING service {}".format( crq.key, logging.key)) task = jira.instance.create_issue( issuetype={'name': 'MOP Task'}, assignee={'name': app.config['JIRA_USERNAME']}, project=app.config['JIRA_CRQ_PROJECT'], summary='[IMPLEMENTATION TASK] {}'.format(self.task.name), parent={'key': crq.key}, customfield_14135={ 'value': 'IPG', 'child': { 'value': 'IPG Big Data' } }, customfield_15150={'value': 'No'}) self.log.msg("Created task: {}".format(task.key)) jira.instance.transition_issue( task, app.config['JIRA_TRANSITION_TASK_PLANNING']) self.log.msg("Transitioned {} to planning".format(task.key)) for m in [Session.merge(m) for m in shrink_members]: filename = '{}.{}.template'.format(pool.id, m.vm_id) attachment_content = io.StringIO(m.template) jira.instance.add_attachment(issue=task, filename=filename, attachment=attachment_content) self.log.msg("Attached member {} to shrink to task {}".format( filename, task.key)) jira.instance.transition_issue( task, app.config['JIRA_TRANSITION_TASK_WRITTEN']) self.log.msg("Transitioned task {} to written".format(task.key)) jira.approver_instance.transition_issue( task, app.config['JIRA_TRANSITION_TASK_APPROVED']) self.log.msg("Approved task {}".format(task.key)) jira.instance.transition_issue( crq, app.config['JIRA_TRANSITION_CRQ_PLANNED_CHANGE']) self.log.msg("Transitioned task {} to approved".format(task.key)) jira.approver_instance.transition_issue( crq, app.config['JIRA_TRANSITION_CRQ_APPROVED']) self.log.msg("Transitioned change request {} to approved".format( crq.key)) self.log.msg("Task ID {}".format(self.task.id)) db_ticket = PoolTicket(pool=pool, action_id=PoolTicketActions.shrink.value, ticket_key=crq.key, task=Session.merge(self.task)) Session.add(db_ticket) Session.commit() except Exception as e: if crq is not None: jira.cancel_crq_and_tasks( crq, comment="failure creating change tickets") raise e
def plan_expansion(self, pool, expansion_names): task = crq = None try: pool = Session.merge(pool) start, end = jira.next_immediate_window_dates() logging = jira.instance.issue('SVC-1020') crq = jira.instance.create_issue( project=app.config['JIRA_CRQ_PROJECT'], issuetype={'name': 'Change Request'}, assignee={'name': app.config['JIRA_USERNAME']}, summary='[TEST IMPLEMENT] {}'.format(self.task.name), description=self.task.description, customfield_14530=start, customfield_14531=end, customfield_19031={'value': 'Maintenance'}, customfield_15152=[{ 'value': 'Global' }], customfield_19430={'value': 'No conflict with any restrictions'}, customfield_14135={ 'value': 'IPG', 'child': { 'value': 'IPG Big Data' } }, customfield_17679="Pool expansion required") self.log.msg("Created change request: {}".format(crq.key)) jira.instance.transition_issue( crq, app.config['JIRA_TRANSITION_CRQ_PLANNING']) self.log.msg("Transitioned {} to planning".format(crq.key)) jira.instance.create_issue_link('Relate', crq, logging) self.log.msg("Related {} to LOGGING service {}".format( crq.key, logging.key)) task = jira.instance.create_issue( issuetype={'name': 'MOP Task'}, assignee={'name': app.config['JIRA_USERNAME']}, project=app.config['JIRA_CRQ_PROJECT'], description= "Instanitate the attached templates in the zone associated " "to the pool identified in the filename <pool_id>.<hostname>", summary='[TEST IMPLEMENTATION TASK] {}'.format(self.task.name), parent={'key': crq.key}, customfield_14135={ 'value': 'IPG', 'child': { 'value': 'IPG Big Data' } }, customfield_15150={'value': 'No'}) self.log.msg("Created task: {}".format(task.key)) jira.instance.transition_issue( task, app.config['JIRA_TRANSITION_TASK_PLANNING']) self.log.msg("Transitioned {} to planning".format(task.key)) env = Environment(loader=ObjectLoader()) for hostname in expansion_names: vars = VarParser.parse_kv_strings_to_dict( pool.cluster.zone.vars, pool.cluster.vars, pool.vars, 'hostname={}'.format(hostname)) vm_template = env.from_string(pool.template).render(pool=pool, vars=vars) attachment_content = io.StringIO(vm_template) jira.instance.add_attachment(issue=task, filename='{}.{}.template'.format( pool.id, hostname), attachment=attachment_content) self.log.msg("Attached template for {} to task {}".format( hostname, task.key)) jira.instance.transition_issue( task, app.config['JIRA_TRANSITION_TASK_WRITTEN']) self.log.msg("Transitioned task {} to written".format(task.key)) jira.approver_instance.transition_issue( task, app.config['JIRA_TRANSITION_TASK_APPROVED']) self.log.msg("Approved task {}".format(task.key)) jira.instance.transition_issue( crq, app.config['JIRA_TRANSITION_CRQ_PLANNED_CHANGE']) self.log.msg("Transitioned task {} to approved".format(task.key)) jira.approver_instance.transition_issue( crq, app.config['JIRA_TRANSITION_CRQ_APPROVED']) self.log.msg("Transitioned change request {} to approved".format( crq.key)) self.log.msg("Task ID {}".format(self.task.id)) db_ticket = PoolTicket(pool=Session.merge(pool), action_id=PoolTicketActions.expand.value, ticket_key=crq.key, task=Session.merge(self.task)) Session.add(db_ticket) Session.commit() except Exception as e: Session.rollback() if crq is not None: jira.cancel_crq_and_tasks( crq, comment="failure creating change tickets") raise e
def plan_update(self, pool, update_members): task = crq = None try: pool = Session.merge(pool) start, end = jira.next_immediate_window_dates() logging = jira.instance.issue('SVC-1020') crq = jira.instance.create_issue( project=app.config['JIRA_CRQ_PROJECT'], issuetype={'name': 'Change Request'}, assignee={'name': app.config['JIRA_USERNAME']}, summary="[IMPLEMENT] {}".format(self.task.name), description=self.task.description, customfield_14530=start, customfield_14531=end, customfield_19031={'value': 'Maintenance'}, customfield_15152=[{ 'value': 'Global' }], customfield_19430={'value': 'No conflict with any restrictions'}, customfield_14135={ 'value': 'IPG', 'child': { 'value': 'IPG Big Data' } }, customfield_17679="Pool update required") self.log.msg("Created change request: {}".format(crq.key)) jira.instance.transition_issue( crq, app.config['JIRA_TRANSITION_CRQ_PLANNING']) self.log.msg("Transitioned {} to planning".format(crq.key)) jira.instance.create_issue_link('Relate', crq, logging) self.log.msg("Related {} to LOGGING service {}".format( crq.key, logging.key)) batch_size = floor( len(update_members) / app.config['BATCH_SIZE_PERCENT']) batch_size = 1 if batch_size == 0 else batch_size num_batches = min(len(update_members), app.config['BATCH_SIZE_PERCENT']) self.log.msg( "updating {} hosts in pool {} requires {} tasks with no more than {} hosts per task" .format(len(update_members), pool.name, num_batches, batch_size)) batch_num = 0 while len(update_members): batch_num += 1 task = jira.instance.create_issue( issuetype={'name': 'MOP Task'}, assignee={'name': app.config['JIRA_USERNAME']}, project=app.config['JIRA_CRQ_PROJECT'], summary='[TASK {}/{} (Update {}%)] {}'.format( batch_num, num_batches, app.config['BATCH_SIZE_PERCENT'], self.task.name), parent={'key': crq.key}, customfield_14135={ 'value': 'IPG', 'child': { 'value': 'IPG Big Data' } }, customfield_15150={'value': 'No'}) self.log.msg("Created task: {}".format(task.key)) jira.instance.transition_issue( task, app.config['JIRA_TRANSITION_TASK_PLANNING']) self.log.msg("Transitioned {} to planning".format(task.key)) env = Environment(loader=ObjectLoader()) for num in range(0, min(len(update_members), batch_size)): m = update_members.pop() filename = '{}.{}.template'.format(m.pool.id, m.vm_id) attachment_content = io.StringIO(m.current_template()) jira.instance.add_attachment(issue=task, filename=filename, attachment=attachment_content) self.log.msg("Attached template for {} to task {}".format( filename, task.key)) jira.instance.transition_issue( task, app.config['JIRA_TRANSITION_TASK_WRITTEN']) self.log.msg("Transitioned task {} to written".format(task.key)) jira.approver_instance.transition_issue( task, app.config['JIRA_TRANSITION_TASK_APPROVED']) self.log.msg("Approved task {}".format(task.key)) jira.instance.transition_issue( crq, app.config['JIRA_TRANSITION_CRQ_PLANNED_CHANGE']) self.log.msg("Transitioned change request {} to approved".format( task.key)) jira.approver_instance.transition_issue( crq, app.config['JIRA_TRANSITION_CRQ_APPROVED']) self.log.msg("Transitioned change request {} to approved".format( crq.key)) self.log.msg("Task ID {}".format(self.task.id)) db_ticket = PoolTicket(pool=pool, action_id=PoolTicketActions.update.value, ticket_key=crq.key, task=Session.merge(self.task)) Session.add(db_ticket) Session.commit() except Exception as e: Session.rollback() if crq is not None: jira.cancel_crq_and_tasks( crq, comment="failure creating change tickets") raise e