def vm_utilization_rrd(host_ip, m_type=None):
"""
Handles periodic collection of VM and Host utilization data and updates of
respective RRD file.
"""
logger.info("ENTERING RRD UPDATION/CREATION........on host: %s" % host_ip)
try:
rrd_logger.debug("Starting RRD Processing for Host: %s" % host_ip)
rrd_logger.debug(host_ip)
if is_pingable(host_ip):
update_rrd(host_ip, m_type)
else:
rrd_logger.error("UNABLE TO UPDATE RRDs for host : %s" % host_ip)
except Exception as e:
rrd_logger.debug("ERROR OCCURED: %s" % e)
finally:
rrd_logger.debug("Completing RRD Processing for Host: %s" % host_ip)
logger.debug("EXITING RRD UPDATION/CREATION........on host: %s" %
host_ip)
def vm_utilization_rrd(host_ip,m_type=None):
"""
Handles periodic collection of VM and Host utilization data and updation of respective RRD file."""
logger.info("ENTERING RRD UPDATION/CREATION........on host: %s" % host_ip)
try:
rrd_logger.debug("Starting RRD Processing for Host: %s" % host_ip)
rrd_logger.debug(host_ip)
if is_pingable(host_ip):
if m_type is None:
update_rrd(host_ip)
else:
update_rrd(host_ip,m_type)
else:
rrd_logger.error("UNABLE TO UPDATE RRDs for host : %s" % host_ip)
except Exception as e:
rrd_logger.debug("ERROR OCCURED: %s" % e)
finally:
rrd_logger.debug("Completing RRD Processing for Host: %s" % host_ip)
logger.debug("EXITING RRD UPDATION/CREATION........on host: %s" % host_ip)
def shutdown_baadal():
logger.info('Starting Baadal Shutdown')
update_constant('baadal_status', BAADAL_STATUS_DOWN_IN_PROGRESS)
current.db.commit()
vms = current.db(
current.db.vm_data.status.belongs(
current.VM_STATUS_RUNNING, current.VM_STATUS_SUSPENDED)).select()
# pool = ThreadPool(THREAD_POOL_COUNT)
for vm_detail in vms:
snapshot_and_destroy(vm_detail.id, vm_detail.vm_identity,
vm_detail.status)
current.db.vm_event_log.insert(vm_id=vm_detail.id,
attribute='VM Status',
requester_id=-1,
old_value=vm_detail.status,
new_value='System Shutdown')
current.db.commit()
# pool.add_task(snapshot_and_suspend, vm_detail.id, vm_detail.vm_identity)
# pool.wait_completion()
update_constant('baadal_status', BAADAL_STATUS_DOWN)
current.db.commit()
def bootup_host(host_id_list):
logger.info('Starting Host Bootup')
# pool = ThreadPool(THREAD_POOL_COUNT)
vms = current.db(~current.db.vm_data.status.belongs(current.VM_STATUS_UNKNOWN, current.VM_STATUS_IN_QUEUE) &
current.db.vm_data.host_id.belongs(host_id_list)).select()
for vm_detail in vms:
revert_and_resume(vm_detail.id, vm_detail.vm_identity)
def process_loadbalancer():
logger.info("ENTERING PROCESS LOADBALANCER VM ........")
try:
(host_list, vm_list) = find_host_and_guest_list()
loadbalance_vm(host_list, vm_list)
except:
log_exception()
pass
finally:
logger.debug("EXITING PROCESS LOADBALANCER VM......")
def process_loadbalancer():
logger.info("ENTERING PROCESS LOADBALANCER VM ........")
try:
(host_list,vm_list)=find_host_and_guest_list()
loadbalance_vm(host_list,vm_list)
except:
log_exception()
pass
finally:
logger.debug("EXITING PROCESS LOADBALANCER VM......")
def process_container_queue(task_event_id):
"""
Invoked when scheduler runs task of type 'Container_Task'
For every task, function calls the corresponding handler
and updates the database on the basis of the response
"""
logger.info("\n ENTERING Container_Task........")
task_event_data = db.task_queue_event[task_event_id]
task_queue_data = db.task_queue[task_event_data.task_id]
container_data = db.container_data[
task_event_data.cont_id] if task_event_data.cont_id != None else None
try:
#Update attention_time for task in the event table
task_event_data.update_record(attention_time=get_datetime(),
status=TASK_QUEUE_STATUS_PROCESSING)
#Call the corresponding function from vm_helper
logger.debug("Starting Container_Task processing...")
ret = task[task_queue_data.task_type](task_queue_data.parameters)
logger.debug("Completed Container_Task processing...")
#On return, update the status and end time in task event table
task_event_data.update_record(status=ret[0],
message=ret[1],
end_time=get_datetime())
if ret[0] == TASK_QUEUE_STATUS_FAILED:
logger.debug("Container_Task FAILED")
logger.debug("Container_Task Error Message: %s" % ret[1])
task_queue_data.update_record(status=TASK_QUEUE_STATUS_FAILED)
elif ret[0] == TASK_QUEUE_STATUS_SUCCESS:
# Create log event for the task
logger.debug("Container_Task SUCCESSFUL")
if container_data:
_log_vm_event(container_data, task_queue_data)
# For successful task, delete the task from queue
if db.task_queue[task_queue_data.id]:
del db.task_queue[task_queue_data.id]
if 'request_id' in task_queue_data.parameters:
del db.request_queue[task_queue_data.parameters['request_id']]
if task_event_data.task_type not in (VM_TASK_MIGRATE_HOST,
VM_TASK_MIGRATE_DS):
_send_cont_task_complete_mail(task_event_data)
except:
msg = log_exception()
task_event_data.update_record(status=TASK_QUEUE_STATUS_FAILED,
message=msg)
finally:
db.commit()
logger.info("EXITING Container_Task........\n")
def bootup_host(host_id_list):
logger.info('Starting Host Bootup')
# pool = ThreadPool(THREAD_POOL_COUNT)
vms = current.db(
~current.db.vm_data.status.belongs(current.VM_STATUS_UNKNOWN,
current.VM_STATUS_IN_QUEUE)
& current.db.vm_data.host_id.belongs(host_id_list)).select()
for vm_detail in vms:
revert_and_resume(vm_detail.id, vm_detail.vm_identity)
def send_object_task_complete_mail(task_event, object_name):
vm_users = []
vm_id = task_event.parameters['vm_id'] if 'vm_id' in task_event.parameters else None
if vm_id:
for user in db(db.user_object_map.ob_id == vm_id).select(db.user_object_map.user_id):
vm_users.append(user['user_id'])
else:
vm_users.append(task_event.requester_id)
logger.info("\n %s" %vm_users)
send_email_to_vm_user(task_event.task_type, object_name, task_event.start_time, vm_users)
def process_unusedvm():
logger.info("ENTERING PROCESS UNUSED VM ........")
try:
process_shutdown_unusedvm()
process_purge_shutdownvm()
except:
log_exception()
pass
finally:
db.commit()
logger.debug("EXITING PROCESS UNUSED VM......")
def check_vnc_access():
"""
Clears all timed out VNC Mappings
Invoked when scheduler runs task of type 'vnc_access'"""
logger.info("ENTERNING CLEAR ALL TIMEDOUT VNC MAPPINGS")
try:
clear_all_timedout_vnc_mappings()
except:
log_exception()
pass
finally:
logger.debug("EXITING CLEAR ALL TIMEDOUT VNC MAPPINGS........")
def host_sanity_check():
"""
Handles periodic Host sanity check
Invoked when scheduler runs task of type 'host_sanity'"""
logger.info("ENTERNING HOST SANITY CHECK........")
try:
host_status_sanity_check()
except:
log_exception()
pass
finally:
logger.debug("EXITING HOST SANITY CHECK........")
def vm_sanity_check():
"""
Handles periodic VM sanity check
Invoked when scheduler runs task of type 'vm_sanity'"""
logger.info("ENTERNING VM SANITY CHECK........")
try:
check_vm_sanity()
except:
log_exception()
pass
finally:
logger.debug("EXITING VM SANITY CHECK........")
def host_sanity_check():
"""
Handles periodic Host sanity check
Invoked when scheduler runs task of type 'host_sanity'"""
logger.info("ENTERNING HOST SANITY CHECK........")
try:
host_status_sanity_check()
except:
log_exception()
pass
finally:
logger.debug("EXITING HOST SANITY CHECK........")
def check_vnc_access():
"""
Clears all timed out VNC Mappings
Invoked when scheduler runs task of type 'vnc_access'"""
logger.info("ENTERNING CLEAR ALL TIMEDOUT VNC MAPPINGS")
try:
clear_all_timedout_vnc_mappings()
except:
log_exception()
pass
finally:
logger.debug("EXITING CLEAR ALL TIMEDOUT VNC MAPPINGS........")
def vm_sanity_check():
"""
Handles periodic VM sanity check
Invoked when scheduler runs task of type 'vm_sanity'"""
logger.info("ENTERNING VM SANITY CHECK........")
try:
check_vm_sanity()
except:
log_exception()
pass
finally:
logger.debug("EXITING VM SANITY CHECK........")
def process_purge_shutdownvm():
logger.info("ENTERING PURGE SHUTDOWN VM ........")
vmShutDownDays = config.get("GENERAL_CONF", "shutdown_vm_days")
try:
# Fetch all the VM's which are locked and whose delete warning date is not null.
for vm_data in db(db.vm_data.locked == True and db.vm_data.delete_warning_date!=None).select(db.vm_data.ALL):
daysDiff=0
daysDiff=(get_datetime()-vm_data.delete_warning_date).days
if(daysDiff >=0 ):
for vm_details in db(db.vm_event_log.vm_id==vm_data.id).select(db.vm_event_log.ALL,orderby = ~db.vm_event_log.id,limitby=(0,1)):
daysDiff=(get_datetime()-vm_details.timestamp).days
if(vm_details.new_value == "Shutdown" and int(daysDiff)>=int(vmShutDownDays)):
logger.info("Need to delete the VM ID:"+str(vm_data.id))
add_vm_task_to_queue(vm_data.id,VM_TASK_DELETE)
# make an entry in task queue so that scheduler can pick up and delete the VM.
else:
logger.info("No need to delete the VM ID:"+str(vm_data.id)+" as it is in use now. ")
db(db.vm_data.id == vm_details.vm_id).update(locked='F',delete_warning_date=None)
else:
logger.info("No need to process shutdown VM :"+str(vm_data.id))
except:
log_exception()
pass
finally:
db.commit()
logger.debug("EXITING PURGE SHUTDOWN VM ........")
def process_unusedvm():
"""
Purge/shutdown the unused VM's
"""
logger.info("ENTERING PROCESS UNUSED VM ........")
try:
process_shutdown_unusedvm()
process_purge_shutdownvm()
except:
log_exception()
pass
finally:
db.commit()
logger.debug("EXITING PROCESS UNUSED VM......")
def process_task_queue(task_event_id):
"""
Invoked when scheduler runs task of type 'vm_task'
For every task, function calls the corresponding handler
and updates the database on the basis of the response
"""
logger.info("\n ENTERING VM_TASK........")
task_event_data = db.task_queue_event[task_event_id]
task_queue_data = db.task_queue[task_event_data.task_id]
vm_data = db.vm_data[task_event_data.vm_id] if task_event_data.vm_id != None else None
try:
#Update attention_time for task in the event table
task_event_data.update_record(attention_time=get_datetime(), status=TASK_QUEUE_STATUS_PROCESSING)
#Call the corresponding function from vm_helper
logger.debug("Starting VM_TASK processing...")
ret = task[task_queue_data.task_type](task_queue_data.parameters)
logger.debug("Completed VM_TASK processing...")
#On return, update the status and end time in task event table
task_event_data.update_record(status=ret[0], message=ret[1], end_time=get_datetime())
if ret[0] == TASK_QUEUE_STATUS_FAILED:
logger.debug("VM_TASK FAILED")
logger.debug("VM_TASK Error Message: %s" % ret[1])
task_queue_data.update_record(status=TASK_QUEUE_STATUS_FAILED)
elif ret[0] == TASK_QUEUE_STATUS_SUCCESS:
# Create log event for the task
logger.debug("VM_TASK SUCCESSFUL")
if vm_data:
_log_vm_event(vm_data, task_queue_data)
# For successful task, delete the task from queue
if db.task_queue[task_queue_data.id]:
del db.task_queue[task_queue_data.id]
if 'request_id' in task_queue_data.parameters:
del db.request_queue[task_queue_data.parameters['request_id']]
if task_event_data.task_type not in (VM_TASK_MIGRATE_HOST, VM_TASK_MIGRATE_DS):
_send_task_complete_mail(task_event_data)
except:
msg = log_exception()
task_event_data.update_record(status=TASK_QUEUE_STATUS_FAILED, message=msg)
finally:
db.commit()
logger.info("EXITING VM_TASK........\n")
def process_vmdaily_checks():
"""
Function will check for the shutdown VM's and sends email to the user"""
logger.info("Entering VM's Daily Checks........")
try:
process_sendwarning_unusedvm()
process_sendwarning_shutdownvm()
except:
log_exception()
pass
finally:
db.commit()
logger.debug("EXITING VM DAILY CHECKS........")
def bootup_baadal():
logger.info('Starting Baadal Bootup')
update_constant('baadal_status', BAADAL_STATUS_UP_IN_PROGRESS)
current.db.commit()
# pool = ThreadPool(THREAD_POOL_COUNT)
vms = current.db(~current.db.vm_data.status.belongs(current.VM_STATUS_UNKNOWN, current.VM_STATUS_IN_QUEUE)).select()
for vm_detail in vms:
revert_and_resume(vm_detail.id, vm_detail.vm_identity)
# pool.add_task(revert_and_delete_snapshot, vm_detail.id, vm_detail.vm_identity, sys_snapshot.id, sys_snapshot.snapshot_name)
# pool.wait_completion()
update_constant('baadal_status', BAADAL_STATUS_UP)
current.db.commit()
def shutdown_host(host_id_list):
logger.info('Starting Host Shutdown')
vms = current.db((current.db.vm_data.status.belongs(current.VM_STATUS_RUNNING, current.VM_STATUS_SUSPENDED)) &
current.db.vm_data.host_id.belongs(host_id_list)).select()
# pool = ThreadPool(THREAD_POOL_COUNT)
for vm_detail in vms:
snapshot_and_destroy(vm_detail.id, vm_detail.vm_identity, vm_detail.status)
current.db.vm_event_log.insert(vm_id = vm_detail.id,
attribute = 'VM Status',
requester_id = -1,
old_value = vm_detail.status,
new_value = 'System Shutdown')
current.db.commit()
def process_vmdaily_checks():
"""
Check for the shutdown VM's and unused VM's and sends warning email to the user
"""
logger.info("Entering VM's Daily Checks........")
try:
process_sendwarning_unusedvm()
process_sendwarning_shutdownvm()
except:
log_exception()
pass
finally:
db.commit()
logger.debug("EXITING VM DAILY CHECKS........")
def shutdown_host(host_id_list):
logger.info('Starting Host Shutdown')
vms = current.db(
(current.db.vm_data.status.belongs(current.VM_STATUS_RUNNING,
current.VM_STATUS_SUSPENDED))
& current.db.vm_data.host_id.belongs(host_id_list)).select()
# pool = ThreadPool(THREAD_POOL_COUNT)
for vm_detail in vms:
snapshot_and_destroy(vm_detail.id, vm_detail.vm_identity,
vm_detail.status)
current.db.vm_event_log.insert(vm_id=vm_detail.id,
attribute='VM Status',
requester_id=-1,
old_value=vm_detail.status,
new_value='System Shutdown')
current.db.commit()
def bootup_baadal():
logger.info('Starting Baadal Bootup')
update_constant('baadal_status', BAADAL_STATUS_UP_IN_PROGRESS)
current.db.commit()
# pool = ThreadPool(THREAD_POOL_COUNT)
vms = current.db(~current.db.vm_data.status.belongs(
current.VM_STATUS_UNKNOWN, current.VM_STATUS_IN_QUEUE)).select()
for vm_detail in vms:
revert_and_resume(vm_detail.id, vm_detail.vm_identity)
# pool.add_task(revert_and_delete_snapshot, vm_detail.id, vm_detail.vm_identity, sys_snapshot.id, sys_snapshot.snapshot_name)
# pool.wait_completion()
update_constant('baadal_status', BAADAL_STATUS_UP)
current.db.commit()
def process_unusedvm_purge():
logger.info("ENTERING PURGE UNUSED VM ........")
try:
# Fetch all the VM's which are locked and whose delete warning date=today.
for vm_data in db(db.vm_data.locked == True).select(db.vm_data.ALL):
for vm_details in db(db.vm_event_log.vm_id==vm_data.id).select(db.vm_event_log.ALL,orderby = ~db.vm_event_log.id,limitby=(0,1)):
daysDiff=(get_datetime()-vm_data.delete_warning_date).days
if(vm_details.new_value == "Shutdown" and daysDiff >= 0):
logger.info("Need to delete the VM ID:"+str(vm_data.id))
add_vm_task_to_queue(vm_data.id,VM_TASK_DELETE)
# make an entry in task queue so that scheduler can pick up and delete the VM.
except:
log_exception()
pass
finally:
db.commit()
logger.debug("EXITING PURGE UNUSED VM ........")
def shutdown_baadal():
logger.info('Starting Baadal Shutdown')
update_constant('baadal_status', BAADAL_STATUS_DOWN_IN_PROGRESS)
current.db.commit()
vms = current.db(current.db.vm_data.status.belongs(current.VM_STATUS_RUNNING, current.VM_STATUS_SUSPENDED)).select()
# pool = ThreadPool(THREAD_POOL_COUNT)
for vm_detail in vms:
snapshot_and_destroy(vm_detail.id, vm_detail.vm_identity, vm_detail.status)
current.db.vm_event_log.insert(vm_id = vm_detail.id,
attribute = 'VM Status',
requester_id = -1,
old_value = vm_detail.status,
new_value = 'System Shutdown')
current.db.commit()
# pool.add_task(snapshot_and_suspend, vm_detail.id, vm_detail.vm_identity)
# pool.wait_completion()
update_constant('baadal_status', BAADAL_STATUS_DOWN)
current.db.commit()
def process_shutdown_unusedvm():
logger.info("ENTERING SHUTDOWN UNUSED VM ........")
try:
# Fetch all the VM's which are locked and whose shutdown_warning_date=today.
vmCPUThreshold = config.get("GENERAL_CONF", "cpu_threshold_limit")
vmreadThreshold = config.get("GENERAL_CONF", "nwRead_threshold_limit")
vmwriteThreshold = config.get("GENERAL_CONF", "nwWrite_threshold_limit")
thresholdcontext = dict(CPUThreshold=vmCPUThreshold,
ReadThreshold=vmreadThreshold,
WriteThreshold=vmwriteThreshold)
for vmData in db(db.vm_data.shutdown_warning_date!=None).select(db.vm_data.ALL):
daysDiff=(get_datetime()-vmData.shutdown_warning_date).days
if(daysDiff >= 0):
'''Again compare the data for last 20 days from rrd logs '''
retVal=compare_rrd_data_with_threshold(vmData.vm_identity,thresholdcontext)
logger.info(" DaysDiff are "+str(daysDiff)+" return value is "+str(retVal))
if(retVal == True):
logger.info("Need to shutdown the VM ID:"+str(vmData.id))
add_vm_task_to_queue(vmData.id,VM_TASK_DESTROY)
# make an entry in task queue so that scheduler can pick up and shutdown the VM.
else:
logger.info("No Need to shutdown the VM ID:"+str(vmData.id)+" as VM is in use now. ")
#update db to clean the shutdown warning date
db(db.vm_data.id == vmData.id).update(shutdown_warning_date=None)
else:
logger.info("No need to process purge for the VM:"+str(vmData.id))
except:
log_exception()
pass
finally:
db.commit()
logger.debug("EXITING SHUTDOWN UNUSED VM ........")
def process_sendwarning_unusedvm():
logger.info("Entering send warning to unused VM........")
try:
''' performing daily checks for network usage '''
vmCPUThreshold = config.get("GENERAL_CONF", "cpu_threshold_limit")
vmreadThreshold = config.get("GENERAL_CONF", "nwRead_threshold_limit")
vmwriteThreshold = config.get("GENERAL_CONF", "nwWrite_threshold_limit")
thresholdcontext = dict(CPUThreshold=vmCPUThreshold,
ReadThreshold=vmreadThreshold,
WriteThreshold=vmwriteThreshold)
logger.info("checking network usage with threshold values as CPUThreshold is:"+str(thresholdcontext['CPUThreshold'])+" WriteThreshold is :"+str(thresholdcontext['WriteThreshold'])+" ReadThreshold is :"+ str(thresholdcontext['ReadThreshold']))
vms = db(db.vm_data.status.belongs(VM_STATUS_RUNNING, VM_STATUS_SUSPENDED) & (db.vm_data.shutdown_warning_date == None) & (db.vm_data.start_time < (get_datetime() - timedelta(days=20)))).select()
'''check vm should have been created 20days back'''
for vm in vms:
logger.info("comparing threshold for the vm "+ str(vm.vm_identity))
send_email=0
retVal=compare_rrd_data_with_threshold(vm.vm_identity,thresholdcontext)
if(retVal == True):
vm_users = []
vm_name = ""
for user in db((db.user_vm_map.vm_id == vm.id) & (db.user_vm_map.vm_id == db.vm_data.id) & (db.vm_data.shutdown_warning_date == None )).select(db.user_vm_map.user_id,db.vm_data.vm_name):
send_email=1
vm_users.append(user.user_vm_map.user_id)
vm_name=user.vm_data.vm_name
if (send_email == 1):
vm_shutdown_time=send_email_vm_warning(VM_TASK_WARNING_SHUTDOWN,vm_users,vm_name,'')
logger.debug("Mail sent for vm_name:"+str(vm_name)+"|shutdown time returned from the function:"+ str(vm_shutdown_time))
db(db.vm_data.id == vm.id).update(shutdown_warning_date=vm_shutdown_time)
db.commit()
else:
logger.debug("Warning Email to use the VM has already been sent to VM_ID:"+str(vm.id))
else:
logger.info("VM:"+str(vm.id)+" is in use.. no need to send shutdown warning mail ...")
except:
log_exception()
pass
finally:
db.commit()
logger.debug("EXITING send warning to unused VM........")
def process_object_task(task_event_id):
"""Invoked when scheduler runs task of type 'object_task'
For every task, function calls the corresponding handler
and updates the database on the basis of the response """
logger.info("\n ENTERING OBJECT_TASK ........")
task_event_data = db.task_queue_event[task_event_id]
task_queue_data = db.task_queue[task_event_data.task_id]
object_data = db.object_store_data[task_event_data.parameters[
'vm_id']] if task_event_data.parameters['vm_id'] != None else None
try:
#Update attention_time for task in the event table
task_event_data.update_record(attention_time=get_datetime(),
status=TASK_QUEUE_STATUS_PROCESSING)
#Call the corresponding function from vm_helper
logger.debug("Starting OBJECT_TASK processing...")
ret = create_object_store(task_event_id, object_data)
logger.debug("Completed OBJECT_TASK processing...")
#On return, update the status and end time in task event table
task_event_data.update_record(status=ret[0],
message=ret[1],
end_time=get_datetime())
if ret[0] == TASK_QUEUE_STATUS_FAILED:
logger.debug("OBJECT_TASK FAILED")
logger.debug("OBJECT_TASK Error Message: %s" % ret[1])
task_queue_data.update_record(status=TASK_QUEUE_STATUS_FAILED)
elif ret[0] == TASK_QUEUE_STATUS_SUCCESS:
# Create log event for the task
logger.debug("OBJECT_TASK SUCCESSFUL")
if object_data:
logger.info("\n object_data: %s" % object_data)
# For successful task, delete the task from queue
if db.task_queue[task_queue_data.id]:
del db.task_queue[task_queue_data.id]
if 'request_id' in task_queue_data.parameters:
del db.request_queue[task_queue_data.parameters['request_id']]
_send_object_task_complete_mail(task_event_data,
object_data['object_store_name'])
except:
msg = log_exception()
task_event_data.update_record(status=TASK_QUEUE_STATUS_FAILED,
message=msg)
finally:
db.commit()
logger.info("EXITING OBJECT_TASK........\n")
def process_sendwarning_shutdownvm():
logger.info("Entering Process send warning mail to shutdown vm........")
try:
vmShutDownDays = config.get("GENERAL_CONF", "shutdown_vm_days")
send_email=0
for vm_id in db().select(db.vm_event_log.vm_id, distinct=True):
for vm_details in db(db.vm_event_log.vm_id==vm_id['vm_id']).select(db.vm_event_log.ALL,orderby = ~db.vm_event_log.id,limitby=(0,1)):
daysDiff=(get_datetime()-vm_details.timestamp).days
vm_shutdown_time=vm_details.timestamp
logger.info("VM details are VM_ID:" + str(vm_details['vm_id'])+ "|ID:"+str(vm_details['id'])+"|new_values is:"+str(vm_details['new_value'])+"|daysDiff:" + str(daysDiff)+"|vmShutDownDays:"+vmShutDownDays+"|vm_shutdown_time :"+str(vm_shutdown_time))
if (vm_details.new_value == "Shutdown" and int(daysDiff)>=int(vmShutDownDays)):
vm_users = []
vm_name = ""
for user in db((db.user_vm_map.vm_id == vm_details.vm_id) & (db.user_vm_map.vm_id == db.vm_data.id) & (db.vm_data.locked != True) & (db.vm_data.delete_warning_date == None )).select(db.user_vm_map.user_id,db.vm_data.vm_name):
send_email=1
vm_users.append(user.user_vm_map.user_id)
vm_name=user.vm_data.vm_name
if (send_email == 1):
vm_delete_time=send_email_vm_warning(VM_TASK_WARNING_DELETE,vm_users,vm_name,vm_shutdown_time)
logger.debug("Mail sent for vm_id:"+str(vm_details.vm_id)+"|vm_name:"+str(vm_name)+"|delete time:"+ str(vm_delete_time))
db(db.vm_data.id == vm_details.vm_id).update(locked=True, delete_warning_date=vm_delete_time)
send_email=0
else:
logger.debug("Email has already been sent to VM_ID:"+str(vm_details.vm_id))
else:
logger.info("VM:"+str(vm_details.vm_id)+" is not shutdown for: "+str(vmShutDownDays)+"(configured) days")
except:
log_exception()
pass
finally:
db.commit()
logger.debug("EXITING Send warning to shutdown vm........")
def process_vmdaily_checks():
"""
Function will check for the shutdown VM's and sends email to the user"""
logger.info("Entering VM's Daily Checks........")
try:
vmShutDownDays = config.get("GENERAL_CONF", "shutdown_vm_days")
send_email=0
for vm_id in db().select(db.vm_event_log.vm_id, distinct=True):
for vm_details in db(db.vm_event_log.vm_id==vm_id['vm_id']).select(db.vm_event_log.ALL,orderby = ~db.vm_event_log.id,limitby=(0,1)):
daysDiff=(get_datetime()-vm_details.timestamp).days
vm_shutdown_time=vm_details.timestamp
logger.info("VM details are VM_ID:" + str(vm_details['vm_id'])+ "|ID:"+str(vm_details['id'])+"|new_values is:"+str(vm_details['new_value'])+"|daysDiff:" + str(daysDiff)+"|vmShutDownDays:"+vmShutDownDays+"|vm_shutdown_time :"+str(vm_shutdown_time))
if (vm_details.new_value == "Shutdown" and int(daysDiff)>=int(vmShutDownDays)):
vm_users = []
vm_name = ""
for user in db((db.user_vm_map.vm_id == vm_details.vm_id) & (db.user_vm_map.vm_id == db.vm_data.id) & (db.vm_data.locked !='T') & (db.vm_data.delete_warning_date == None )).select(db.user_vm_map.user_id,db.vm_data.vm_name):
send_email=1
vm_users.append(user.user_vm_map.user_id)
vm_name=user.vm_data.vm_name
if (send_email == 1):
vm_delete_time = send_email_delete_vm_warning(vm_users,vm_name,vm_shutdown_time)
logger.debug("Mail sent for vm_name:"+str(vm_name)+"|delete time returned from the function:"+ str(vm_delete_time))
db(db.vm_data.id == vm_details.vm_id).update(locked=True, delete_warning_date=vm_delete_time)
else:
logger.debug("Email has already been sent to VM_ID:"+str(vm_details.vm_id))
else:
logger.info("VM:"+str(vm_details.vm_id)+" is not shutdown ..")
except:
log_exception()
pass
finally:
db.commit()
logger.debug("EXITING VM DAILY CHECKS........")
def check_vm_sanity(host_id = 0):
"""
Checks if database information of VM status is in sync with ground reality.
In case of discrepancy database is updated.
- For each host, get list of the domains(running and not running).
- If VM has migrated to another host; host information updated in database.
- If VM not in sync with actual state of VM; status of VM updated in DB.
- If VM is not present in database, it is marked Orphan.
- If VM is not found on any of the hosts, it is marked Undefined.
"""
vmcheck=[]
vm_list = []
if host_id == 0:
hosts=db(db.host.status == HOST_STATUS_UP).select()
else:
hosts=db(db.host.id == host_id).select()
for host in hosts:
try:
logger.info('Starting sanity check for host %s' %(host.host_name))
#Get list of the domains(running and not running) on the hypervisor
domains = get_host_domains(host.host_ip.private_ip)
for dom in domains:
try:
domain_name = dom.name()
vm = db((db.vm_data.vm_identity == domain_name) &
(db.vm_data.status.belongs(VM_STATUS_RUNNING, VM_STATUS_SUSPENDED, VM_STATUS_SHUTDOWN))).select().first()
vm_state = dom.info()[0]
status = vminfo_to_state(vm_state)
if(vm):
if(vm.host_id != host.id):
vmcheck.append({'vm_id':vm.id,
'host':host.host_name,
'host_id':host.id,
'vmname':vm.vm_name,
'status':status,
'message':'Moved from '+vm.host_id.host_name+' to '+host.host_name, 'operation':'None'})#Bad VMs
#If VM has been migrated to another host; Host information updated
db(db.vm_data.vm_identity == domain_name).update(host_id = host.id)
else:
vmcheck.append({'vm_id':vm.id,
'host':host.host_name,
'host_id':host.id,
'vmname':vm.vm_name,
'status':status,
'message':'VM is on expected host '+vm.host_id.host_name, 'operation':'None'})#Good VMs
if vm_state_map[vm_state] != vm.status:
logger.info("vm_identity="+str(domain_name)+" vm_state_map[vm_state]="+str(vm_state_map[vm_state])+"and vm.status is" + str(vm.status))
#If not in sync with actual state of VM; status of VM updated in DB
db(db.vm_data.vm_identity == domain_name).update(status = vm_state_map[vm_state])
#Adding into vm_event_log about the vm details
db.vm_event_log.insert(vm_id = vm.id,
attribute = 'VM Status',
requester_id = SYSTEM_USER,
old_value = get_vm_status(vm.status),
new_value = get_vm_status(vm_state_map[vm_state]))
vm_list.append(vm.vm_identity)
elif vm_state != VIR_DOMAIN_CRASHED:
vmcheck.append({'host':host.host_name,
'host_id':host.id,
'vmname':dom.name(),
'status':status,
'message':'Orphan, VM is not in database',
'operation':'Orphan'})#Orphan VMs
except Exception:
log_exception()
if(vm):
vmcheck.append({'vmname':vm.vm_name,
'host':'Unknown',
'host_id':'0',
'status':'Unknown',
'message':'Some Error Occurred',
'operation':'Error'})
except:pass
db.commit()
if host_id == 0:
db_vms=db(db.vm_data.status.belongs(VM_STATUS_RUNNING, VM_STATUS_SUSPENDED, VM_STATUS_SHUTDOWN)).select()
for db_vm in db_vms:
if(db_vm.vm_identity not in vm_list):
vmcheck.append({'vmname':db_vm.vm_identity,
'host':db_vm.host_id.host_name,
'host_id':db_vm.host_id,
'status':'Undefined',
'message':'VM not found',
'operation':'Undefined'})
return vmcheck
def check_vm_sanity(host_id=0):
vmcheck = []
vm_list = []
if host_id == 0:
hosts = db(db.host.status == HOST_STATUS_UP).select()
else:
hosts = db(db.host.id == host_id).select()
for host in hosts:
try:
logger.info('Starting sanity check for host %s' % (host.host_name))
#Get list of the domains(running and not running) on the hypervisor
domains = get_host_domains(host.host_ip.private_ip)
for dom in domains:
try:
domain_name = dom.name()
vm = db((db.vm_data.vm_identity == domain_name)
& (db.vm_data.status.belongs(
VM_STATUS_RUNNING, VM_STATUS_SUSPENDED,
VM_STATUS_SHUTDOWN))).select().first()
vm_state = dom.info()[0]
status = vminfo_to_state(vm_state)
if (vm):
if (vm.host_id != host.id):
vmcheck.append({
'vm_id':
vm.id,
'host':
host.host_name,
'host_id':
host.id,
'vmname':
vm.vm_name,
'status':
status,
'message':
'Moved from ' + vm.host_id.host_name + ' to ' +
host.host_name,
'operation':
'None'
}) #Bad VMs
#If VM has been migrated to another host; Host information updated
db(db.vm_data.vm_identity == domain_name).update(
host_id=host.id)
else:
vmcheck.append({
'vm_id':
vm.id,
'host':
host.host_name,
'host_id':
host.id,
'vmname':
vm.vm_name,
'status':
status,
'message':
'VM is on expected host ' +
vm.host_id.host_name,
'operation':
'None'
}) #Good VMs
if vm_state_map[vm_state] != vm.status:
#If not in sync with actual state of VM; status of VM updated in DB
db(db.vm_data.vm_identity == domain_name).update(
status=vm_state_map[vm_state])
#Adding into vm_event_log about the vm details
db.vm_event_log.insert(vm_id=vm.id,
attribute='VM Status',
requester_id=SYSTEM_USER,
old_value=get_vm_status(
vm.status),
new_value=get_vm_status(
vm_state_map[vm_state]))
vm_list.append(vm.vm_identity)
elif vm_state != VIR_DOMAIN_CRASHED:
vmcheck.append({
'host': host.host_name,
'host_id': host.id,
'vmname': dom.name(),
'status': status,
'message': 'Orphan, VM is not in database',
'operation': 'Orphan'
}) #Orphan VMs
except Exception:
log_exception()
if (vm):
vmcheck.append({
'vmname': vm.vm_name,
'host': 'Unknown',
'host_id': '0',
'status': 'Unknown',
'message': 'Some Error Occurred',
'operation': 'Error'
})
except:
pass
db.commit()
if host_id == 0:
db_vms = db(
db.vm_data.status.belongs(VM_STATUS_RUNNING, VM_STATUS_SUSPENDED,
VM_STATUS_SHUTDOWN)).select()
for db_vm in db_vms:
if (db_vm.vm_identity not in vm_list):
vmcheck.append({
'vmname': db_vm.vm_identity,
'host': db_vm.host_id.host_name,
'host_id': db_vm.host_id,
'status': 'Undefined',
'message': 'VM not found',
'operation': 'Undefined'
})
return vmcheck
