def shrink(self, pool, pool_ticket, issue, cowboy_mode=False):
pool = Session.merge(pool)
pool_ticket = Session.merge(pool_ticket)
self.task = Session.merge(self.task)
try:
c_start = JiraApi.get_now()
jira.start_crq(issue, log=self.log, cowboy_mode=cowboy_mode)
for t in issue.fields.subtasks:
t_start = JiraApi.get_now()
t2 = jira.instance.issue(t.key)
jira.start_task(t2, log=self.log, cowboy_mode=cowboy_mode)
for a in t2.fields.attachment:
pool_id, vm_id = a.filename.split('.', 2)[:2]
member = PoolMembership.query.filter_by(pool=pool, vm_id=vm_id).first()
member.retire()
Session.delete(member)
Session.commit()
self.log.msg("Retired VM {} and removed it as member of pool {}".format(member.vm_id, pool.name))
jira.complete_task(t, start_time=t_start, log=self.log, cowboy_mode=cowboy_mode)
self.log.msg("waiting for 120 seconds before running post task diagnostics")
time.sleep(120)
run_diagnostics_on_pool(pool, self.log)
jira.complete_crq(issue, start_time=c_start, log=self.log, cowboy_mode=cowboy_mode)
except Exception as e:
self.log.err("Error occured: {}".format(e))
jira.cancel_crq_and_tasks(issue, comment="an exception occured running this change: {}".format(e))
raise e
finally:
pool_ticket.done = True
Session.merge(pool_ticket)
Session.commit()
def expand(self, pool, pool_ticket, issue, cowboy_mode=False):
new_vm_ids = []
pool = Session.merge(pool)
pool_ticket = Session.merge(pool_ticket)
self.task = Session.merge(self.task)
one_proxy = OneProxy(pool.cluster.zone.xmlrpc_uri, pool.cluster.zone.session_string, verify_certs=False)
try:
c_start = JiraApi.get_now()
jira.start_crq(issue, log=self.log, cowboy_mode=cowboy_mode)
for t in issue.fields.subtasks:
t_start = JiraApi.get_now()
t2 = jira.instance.issue(t.key)
jira.start_task(t2, log=self.log, cowboy_mode=cowboy_mode)
for a in t2.fields.attachment:
pool_id, vm_name = a.filename.split('.', 2)[:2]
template = a.get().decode(encoding="utf-8", errors="strict")
vm_id = one_proxy.create_vm(template=template)
new_vm_ids.append(vm_id)
m = PoolMembership(pool=pool, vm_name=vm_name, vm_id=vm_id, template=a.get(), date_added=datetime.utcnow())
Session.merge(m)
Session.commit()
self.log.msg("created new vm: {}".format(vm_name))
jira.complete_task(t, start_time=t_start, log=self.log, cowboy_mode=cowboy_mode)
self.log.msg("waiting for 120 seconds before running post task diagnostics")
time.sleep(120)
run_diagnostics_on_pool(pool, self.log)
jira.complete_crq(issue, start_time=c_start, log=self.log, cowboy_mode=cowboy_mode)
except Exception as e:
self.log.err("Error occured: {}".format(e))
jira.cancel_crq_and_tasks(issue, comment="an exception occured running this change: {}".format(e))
raise e
finally:
pool_ticket.done = True
Session.merge(pool_ticket)
Session.commit()
def update(self, pool, pool_ticket, issue, cowboy_mode=False):
pool = Session.merge(pool)
pool_ticket = Session.merge(pool_ticket)
self.task = Session.merge(self.task)
one_proxy = OneProxy(pool.cluster.zone.xmlrpc_uri,
pool.cluster.zone.session_string,
verify_certs=False)
try:
c_start = JiraApi.get_now()
jira.start_crq(issue, log=self.log, cowboy_mode=cowboy_mode)
for t in issue.fields.subtasks:
t_start = JiraApi.get_now()
t2 = jira.instance.issue(t.key)
jira.start_task(t2, log=self.log, cowboy_mode=cowboy_mode)
updated_members = []
for a in t2.fields.attachment:
pool_id, vm_id = a.filename.split('.', 2)[:2]
template = a.get().decode(encoding="utf-8", errors="strict")
member = PoolMembership.query.filter_by(pool=pool,
vm_id=vm_id).first()
vm_name = member.vm_name
member.retire()
Session.delete(member)
Session.commit()
new_id = one_proxy.create_vm(template=template)
new_member = PoolMembership(pool=pool,
vm_name=vm_name,
vm_id=new_id,
template=template,
date_added=datetime.utcnow())
Session.add(new_member)
Session.commit()
self.log.msg(
"Instantiated new VM ID {} and added as member of pool {}".
format(new_member.vm_id, pool.name))
updated_members.append(new_member)
self.log.msg(
"waiting for 120 seconds before running post change diagnostics"
)
time.sleep(120)
run_diagnostics_on_pool(pool, self.log)
jira.complete_task(t,
start_time=t_start,
log=self.log,
cowboy_mode=cowboy_mode)
jira.complete_crq(issue,
start_time=c_start,
log=self.log,
cowboy_mode=cowboy_mode)
except Exception as e:
self.log.err("Error occured: {}".format(e))
jira.cancel_crq_and_tasks(
issue,
comment="an exception occured running this change: {}".format(e))
raise e
finally:
pool_ticket.done = True
Session.merge(pool_ticket)
Session.commit()
def shrink(self, pool, pool_ticket, issue, cowboy_mode=False):
pool = Session.merge(pool)
pool_ticket = Session.merge(pool_ticket)
self.task = Session.merge(self.task)
try:
c_start = JiraApi.get_now()
jira.start_crq(issue, log=self.log, cowboy_mode=cowboy_mode)
for t in issue.fields.subtasks:
t_start = JiraApi.get_now()
t2 = jira.instance.issue(t.key)
jira.start_task(t2, log=self.log, cowboy_mode=cowboy_mode)
for a in t2.fields.attachment:
pool_id, vm_id = a.filename.split('.', 2)[:2]
member = PoolMembership.query.filter_by(pool=pool,
vm_id=vm_id).first()
member.retire()
Session.delete(member)
Session.commit()
self.log.msg(
"Retired VM {} and removed it as member of pool {}".format(
member.vm_id, pool.name))
jira.complete_task(t,
start_time=t_start,
log=self.log,
cowboy_mode=cowboy_mode)
self.log.msg(
"waiting for 120 seconds before running post task diagnostics")
time.sleep(120)
run_diagnostics_on_pool(pool, self.log)
jira.complete_crq(issue,
start_time=c_start,
log=self.log,
cowboy_mode=cowboy_mode)
except Exception as e:
self.log.err("Error occured: {}".format(e))
jira.cancel_crq_and_tasks(
issue,
comment="an exception occured running this change: {}".format(e))
raise e
finally:
pool_ticket.done = True
Session.merge(pool_ticket)
Session.commit()
def execute_tickets(name, pool, cowboy_mode=False):
for t in pool.change_tickets:
try:
if t.done == False:
logging.info("[{}] fetching {}".format(name, t.ticket_key))
crq = jira.instance.issue(t.ticket_key)
if not cowboy_mode and JiraApi.expired(crq):
logging.info("[{}] {} has expired".format(name, crq.key))
jira.cancel_crq_and_tasks(crq, "change has expired")
t.done = True
Session.add(t)
logging.info("[{}] marked {} ticket {} as done for pool {} as crq is expired".format(
name, t.action_name(), t.ticket_key, pool.name))
elif cowboy_mode or JiraApi.in_window(crq):
logging.info("[{}] change {} is in window".format(name, crq.key))
if cowboy_mode or jira.is_ready(crq):
title = "{} pool {} under ticket {}".format(t.action_name(), pool.name, crq.key)
task = Task(
name=title,
description=title,
username="******")
Session.add(task)
Session.commit()
task_thread = TaskThread(task=task,
run_function=get_runnable_from_action_id(t.action_id),
pool=pool,
pool_ticket=t,
issue=crq,
cowboy_mode=cowboy_mode)
threads.append(task_thread)
logging.info("[{}] launched background task {} to {}".format(name, task.id, title))
else:
logging.error("[{}] {} is in window and either it or one or "
"more sub tasks are not ready".format(name, crq.key))
else:
logging.info("[{}] {} change {} for pool {} not yet in window".format(
name, t.action_name(), crq.key, pool.name))
except Exception as e:
logging.error("[{}] error executing {}: {}".format(name, t.ticket_key, e))
def plan_expansion(self, pool, expansion_names):
task = crq = None
try:
pool = Session.merge(pool)
start, end = jira.next_immediate_window_dates()
logging = jira.instance.issue('SVC-1020')
crq = jira.instance.create_issue(
project=app.config['JIRA_CRQ_PROJECT'],
issuetype={'name': 'Change Request'},
assignee={'name': app.config['JIRA_USERNAME']},
summary='[TEST IMPLEMENT] {}'.format(self.task.name),
description=self.task.description,
customfield_14530=start,
customfield_14531=end,
customfield_19031={'value': 'Maintenance'},
customfield_15152=[{
'value': 'Global'
}],
customfield_19430={'value': 'No conflict with any restrictions'},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_17679="Pool expansion required")
self.log.msg("Created change request: {}".format(crq.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNING'])
self.log.msg("Transitioned {} to planning".format(crq.key))
jira.instance.create_issue_link('Relate', crq, logging)
self.log.msg("Related {} to LOGGING service {}".format(
crq.key, logging.key))
task = jira.instance.create_issue(
issuetype={'name': 'MOP Task'},
assignee={'name': app.config['JIRA_USERNAME']},
project=app.config['JIRA_CRQ_PROJECT'],
description=
"Instanitate the attached templates in the zone associated "
"to the pool identified in the filename <pool_id>.<hostname>",
summary='[TEST IMPLEMENTATION TASK] {}'.format(self.task.name),
parent={'key': crq.key},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_15150={'value': 'No'})
self.log.msg("Created task: {}".format(task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_PLANNING'])
self.log.msg("Transitioned {} to planning".format(task.key))
env = Environment(loader=ObjectLoader())
for hostname in expansion_names:
vars = VarParser.parse_kv_strings_to_dict(
pool.cluster.zone.vars, pool.cluster.vars, pool.vars,
'hostname={}'.format(hostname))
vm_template = env.from_string(pool.template).render(pool=pool,
vars=vars)
attachment_content = io.StringIO(vm_template)
jira.instance.add_attachment(issue=task,
filename='{}.{}.template'.format(
pool.id, hostname),
attachment=attachment_content)
self.log.msg("Attached template for {} to task {}".format(
hostname, task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_WRITTEN'])
self.log.msg("Transitioned task {} to written".format(task.key))
jira.approver_instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_APPROVED'])
self.log.msg("Approved task {}".format(task.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNED_CHANGE'])
self.log.msg("Transitioned task {} to approved".format(task.key))
jira.approver_instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_APPROVED'])
self.log.msg("Transitioned change request {} to approved".format(
crq.key))
self.log.msg("Task ID {}".format(self.task.id))
db_ticket = PoolTicket(pool=Session.merge(pool),
action_id=PoolTicketActions.expand.value,
ticket_key=crq.key,
task=Session.merge(self.task))
Session.add(db_ticket)
Session.commit()
except Exception as e:
Session.rollback()
if crq is not None:
jira.cancel_crq_and_tasks(
crq, comment="failure creating change tickets")
raise e
def plan_shrink(self, pool, shrink_members):
task = crq = None
try:
pool = Session.merge(pool)
start, end = jira.next_immediate_window_dates()
logging = jira.instance.issue('SVC-1020')
crq = jira.instance.create_issue(
project=app.config['JIRA_CRQ_PROJECT'],
issuetype={'name': 'Change Request'},
assignee={'name': app.config['JIRA_USERNAME']},
summary="[IMPLEMENT] {}".format(self.task.name),
description=self.task.description,
customfield_14530=start,
customfield_14531=end,
customfield_19031={'value': 'Maintenance'},
customfield_15152=[{
'value': 'Global'
}],
customfield_19430={'value': 'No conflict with any restrictions'},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_17679="Pool shrink required")
self.log.msg("Created change request: {}".format(crq.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNING'])
self.log.msg("Transitioned {} to planning".format(crq.key))
jira.instance.create_issue_link('Relate', crq, logging)
self.log.msg("Related {} to LOGGING service {}".format(
crq.key, logging.key))
task = jira.instance.create_issue(
issuetype={'name': 'MOP Task'},
assignee={'name': app.config['JIRA_USERNAME']},
project=app.config['JIRA_CRQ_PROJECT'],
summary='[IMPLEMENTATION TASK] {}'.format(self.task.name),
parent={'key': crq.key},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_15150={'value': 'No'})
self.log.msg("Created task: {}".format(task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_PLANNING'])
self.log.msg("Transitioned {} to planning".format(task.key))
for m in [Session.merge(m) for m in shrink_members]:
filename = '{}.{}.template'.format(pool.id, m.vm_id)
attachment_content = io.StringIO(m.template)
jira.instance.add_attachment(issue=task,
filename=filename,
attachment=attachment_content)
self.log.msg("Attached member {} to shrink to task {}".format(
filename, task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_WRITTEN'])
self.log.msg("Transitioned task {} to written".format(task.key))
jira.approver_instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_APPROVED'])
self.log.msg("Approved task {}".format(task.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNED_CHANGE'])
self.log.msg("Transitioned task {} to approved".format(task.key))
jira.approver_instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_APPROVED'])
self.log.msg("Transitioned change request {} to approved".format(
crq.key))
self.log.msg("Task ID {}".format(self.task.id))
db_ticket = PoolTicket(pool=pool,
action_id=PoolTicketActions.shrink.value,
ticket_key=crq.key,
task=Session.merge(self.task))
Session.add(db_ticket)
Session.commit()
except Exception as e:
if crq is not None:
jira.cancel_crq_and_tasks(
crq, comment="failure creating change tickets")
raise e
def plan_update(self, pool, update_members):
task = crq = None
try:
pool = Session.merge(pool)
start, end = jira.next_immediate_window_dates()
logging = jira.instance.issue('SVC-1020')
crq = jira.instance.create_issue(
project=app.config['JIRA_CRQ_PROJECT'],
issuetype={'name': 'Change Request'},
assignee={'name': app.config['JIRA_USERNAME']},
summary="[IMPLEMENT] {}".format(self.task.name),
description=self.task.description,
customfield_14530=start,
customfield_14531=end,
customfield_19031={'value': 'Maintenance'},
customfield_15152=[{
'value': 'Global'
}],
customfield_19430={'value': 'No conflict with any restrictions'},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_17679="Pool update required")
self.log.msg("Created change request: {}".format(crq.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNING'])
self.log.msg("Transitioned {} to planning".format(crq.key))
jira.instance.create_issue_link('Relate', crq, logging)
self.log.msg("Related {} to LOGGING service {}".format(
crq.key, logging.key))
batch_size = floor(
len(update_members) / app.config['BATCH_SIZE_PERCENT'])
batch_size = 1 if batch_size == 0 else batch_size
num_batches = min(len(update_members),
app.config['BATCH_SIZE_PERCENT'])
self.log.msg(
"updating {} hosts in pool {} requires {} tasks with no more than {} hosts per task"
.format(len(update_members), pool.name, num_batches, batch_size))
batch_num = 0
while len(update_members):
batch_num += 1
task = jira.instance.create_issue(
issuetype={'name': 'MOP Task'},
assignee={'name': app.config['JIRA_USERNAME']},
project=app.config['JIRA_CRQ_PROJECT'],
summary='[TASK {}/{} (Update {}%)] {}'.format(
batch_num, num_batches, app.config['BATCH_SIZE_PERCENT'],
self.task.name),
parent={'key': crq.key},
customfield_14135={
'value': 'IPG',
'child': {
'value': 'IPG Big Data'
}
},
customfield_15150={'value': 'No'})
self.log.msg("Created task: {}".format(task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_PLANNING'])
self.log.msg("Transitioned {} to planning".format(task.key))
env = Environment(loader=ObjectLoader())
for num in range(0, min(len(update_members), batch_size)):
m = update_members.pop()
filename = '{}.{}.template'.format(m.pool.id, m.vm_id)
attachment_content = io.StringIO(m.current_template())
jira.instance.add_attachment(issue=task,
filename=filename,
attachment=attachment_content)
self.log.msg("Attached template for {} to task {}".format(
filename, task.key))
jira.instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_WRITTEN'])
self.log.msg("Transitioned task {} to written".format(task.key))
jira.approver_instance.transition_issue(
task, app.config['JIRA_TRANSITION_TASK_APPROVED'])
self.log.msg("Approved task {}".format(task.key))
jira.instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_PLANNED_CHANGE'])
self.log.msg("Transitioned change request {} to approved".format(
task.key))
jira.approver_instance.transition_issue(
crq, app.config['JIRA_TRANSITION_CRQ_APPROVED'])
self.log.msg("Transitioned change request {} to approved".format(
crq.key))
self.log.msg("Task ID {}".format(self.task.id))
db_ticket = PoolTicket(pool=pool,
action_id=PoolTicketActions.update.value,
ticket_key=crq.key,
task=Session.merge(self.task))
Session.add(db_ticket)
Session.commit()
except Exception as e:
Session.rollback()
if crq is not None:
jira.cancel_crq_and_tasks(
crq, comment="failure creating change tickets")
raise e