def vm_utilization_rrd(host_ip, m_type=None): """ Handles periodic collection of VM and Host utilization data and updates of respective RRD file. """ logger.info("ENTERING RRD UPDATION/CREATION........on host: %s" % host_ip) try: rrd_logger.debug("Starting RRD Processing for Host: %s" % host_ip) rrd_logger.debug(host_ip) if is_pingable(host_ip): update_rrd(host_ip, m_type) else: rrd_logger.error("UNABLE TO UPDATE RRDs for host : %s" % host_ip) except Exception as e: rrd_logger.debug("ERROR OCCURED: %s" % e) finally: rrd_logger.debug("Completing RRD Processing for Host: %s" % host_ip) logger.debug("EXITING RRD UPDATION/CREATION........on host: %s" % host_ip)
def vm_utilization_rrd(host_ip,m_type=None): """ Handles periodic collection of VM and Host utilization data and updation of respective RRD file.""" logger.info("ENTERING RRD UPDATION/CREATION........on host: %s" % host_ip) try: rrd_logger.debug("Starting RRD Processing for Host: %s" % host_ip) rrd_logger.debug(host_ip) if is_pingable(host_ip): if m_type is None: update_rrd(host_ip) else: update_rrd(host_ip,m_type) else: rrd_logger.error("UNABLE TO UPDATE RRDs for host : %s" % host_ip) except Exception as e: rrd_logger.debug("ERROR OCCURED: %s" % e) finally: rrd_logger.debug("Completing RRD Processing for Host: %s" % host_ip) logger.debug("EXITING RRD UPDATION/CREATION........on host: %s" % host_ip)
def shutdown_baadal(): logger.info('Starting Baadal Shutdown') update_constant('baadal_status', BAADAL_STATUS_DOWN_IN_PROGRESS) current.db.commit() vms = current.db( current.db.vm_data.status.belongs( current.VM_STATUS_RUNNING, current.VM_STATUS_SUSPENDED)).select() # pool = ThreadPool(THREAD_POOL_COUNT) for vm_detail in vms: snapshot_and_destroy(vm_detail.id, vm_detail.vm_identity, vm_detail.status) current.db.vm_event_log.insert(vm_id=vm_detail.id, attribute='VM Status', requester_id=-1, old_value=vm_detail.status, new_value='System Shutdown') current.db.commit() # pool.add_task(snapshot_and_suspend, vm_detail.id, vm_detail.vm_identity) # pool.wait_completion() update_constant('baadal_status', BAADAL_STATUS_DOWN) current.db.commit()
def bootup_host(host_id_list): logger.info('Starting Host Bootup') # pool = ThreadPool(THREAD_POOL_COUNT) vms = current.db(~current.db.vm_data.status.belongs(current.VM_STATUS_UNKNOWN, current.VM_STATUS_IN_QUEUE) & current.db.vm_data.host_id.belongs(host_id_list)).select() for vm_detail in vms: revert_and_resume(vm_detail.id, vm_detail.vm_identity)
def process_loadbalancer(): logger.info("ENTERING PROCESS LOADBALANCER VM ........") try: (host_list, vm_list) = find_host_and_guest_list() loadbalance_vm(host_list, vm_list) except: log_exception() pass finally: logger.debug("EXITING PROCESS LOADBALANCER VM......")
def process_loadbalancer(): logger.info("ENTERING PROCESS LOADBALANCER VM ........") try: (host_list,vm_list)=find_host_and_guest_list() loadbalance_vm(host_list,vm_list) except: log_exception() pass finally: logger.debug("EXITING PROCESS LOADBALANCER VM......")
def process_container_queue(task_event_id): """ Invoked when scheduler runs task of type 'Container_Task' For every task, function calls the corresponding handler and updates the database on the basis of the response """ logger.info("\n ENTERING Container_Task........") task_event_data = db.task_queue_event[task_event_id] task_queue_data = db.task_queue[task_event_data.task_id] container_data = db.container_data[ task_event_data.cont_id] if task_event_data.cont_id != None else None try: #Update attention_time for task in the event table task_event_data.update_record(attention_time=get_datetime(), status=TASK_QUEUE_STATUS_PROCESSING) #Call the corresponding function from vm_helper logger.debug("Starting Container_Task processing...") ret = task[task_queue_data.task_type](task_queue_data.parameters) logger.debug("Completed Container_Task processing...") #On return, update the status and end time in task event table task_event_data.update_record(status=ret[0], message=ret[1], end_time=get_datetime()) if ret[0] == TASK_QUEUE_STATUS_FAILED: logger.debug("Container_Task FAILED") logger.debug("Container_Task Error Message: %s" % ret[1]) task_queue_data.update_record(status=TASK_QUEUE_STATUS_FAILED) elif ret[0] == TASK_QUEUE_STATUS_SUCCESS: # Create log event for the task logger.debug("Container_Task SUCCESSFUL") if container_data: _log_vm_event(container_data, task_queue_data) # For successful task, delete the task from queue if db.task_queue[task_queue_data.id]: del db.task_queue[task_queue_data.id] if 'request_id' in task_queue_data.parameters: del db.request_queue[task_queue_data.parameters['request_id']] if task_event_data.task_type not in (VM_TASK_MIGRATE_HOST, VM_TASK_MIGRATE_DS): _send_cont_task_complete_mail(task_event_data) except: msg = log_exception() task_event_data.update_record(status=TASK_QUEUE_STATUS_FAILED, message=msg) finally: db.commit() logger.info("EXITING Container_Task........\n")
def bootup_host(host_id_list): logger.info('Starting Host Bootup') # pool = ThreadPool(THREAD_POOL_COUNT) vms = current.db( ~current.db.vm_data.status.belongs(current.VM_STATUS_UNKNOWN, current.VM_STATUS_IN_QUEUE) & current.db.vm_data.host_id.belongs(host_id_list)).select() for vm_detail in vms: revert_and_resume(vm_detail.id, vm_detail.vm_identity)
def send_object_task_complete_mail(task_event, object_name): vm_users = [] vm_id = task_event.parameters['vm_id'] if 'vm_id' in task_event.parameters else None if vm_id: for user in db(db.user_object_map.ob_id == vm_id).select(db.user_object_map.user_id): vm_users.append(user['user_id']) else: vm_users.append(task_event.requester_id) logger.info("\n %s" %vm_users) send_email_to_vm_user(task_event.task_type, object_name, task_event.start_time, vm_users)
def process_unusedvm(): logger.info("ENTERING PROCESS UNUSED VM ........") try: process_shutdown_unusedvm() process_purge_shutdownvm() except: log_exception() pass finally: db.commit() logger.debug("EXITING PROCESS UNUSED VM......")
def check_vnc_access(): """ Clears all timed out VNC Mappings Invoked when scheduler runs task of type 'vnc_access'""" logger.info("ENTERNING CLEAR ALL TIMEDOUT VNC MAPPINGS") try: clear_all_timedout_vnc_mappings() except: log_exception() pass finally: logger.debug("EXITING CLEAR ALL TIMEDOUT VNC MAPPINGS........")
def host_sanity_check(): """ Handles periodic Host sanity check Invoked when scheduler runs task of type 'host_sanity'""" logger.info("ENTERNING HOST SANITY CHECK........") try: host_status_sanity_check() except: log_exception() pass finally: logger.debug("EXITING HOST SANITY CHECK........")
def vm_sanity_check(): """ Handles periodic VM sanity check Invoked when scheduler runs task of type 'vm_sanity'""" logger.info("ENTERNING VM SANITY CHECK........") try: check_vm_sanity() except: log_exception() pass finally: logger.debug("EXITING VM SANITY CHECK........")
def process_purge_shutdownvm(): logger.info("ENTERING PURGE SHUTDOWN VM ........") vmShutDownDays = config.get("GENERAL_CONF", "shutdown_vm_days") try: # Fetch all the VM's which are locked and whose delete warning date is not null. for vm_data in db(db.vm_data.locked == True and db.vm_data.delete_warning_date!=None).select(db.vm_data.ALL): daysDiff=0 daysDiff=(get_datetime()-vm_data.delete_warning_date).days if(daysDiff >=0 ): for vm_details in db(db.vm_event_log.vm_id==vm_data.id).select(db.vm_event_log.ALL,orderby = ~db.vm_event_log.id,limitby=(0,1)): daysDiff=(get_datetime()-vm_details.timestamp).days if(vm_details.new_value == "Shutdown" and int(daysDiff)>=int(vmShutDownDays)): logger.info("Need to delete the VM ID:"+str(vm_data.id)) add_vm_task_to_queue(vm_data.id,VM_TASK_DELETE) # make an entry in task queue so that scheduler can pick up and delete the VM. else: logger.info("No need to delete the VM ID:"+str(vm_data.id)+" as it is in use now. ") db(db.vm_data.id == vm_details.vm_id).update(locked='F',delete_warning_date=None) else: logger.info("No need to process shutdown VM :"+str(vm_data.id)) except: log_exception() pass finally: db.commit() logger.debug("EXITING PURGE SHUTDOWN VM ........")
def process_unusedvm(): """ Purge/shutdown the unused VM's """ logger.info("ENTERING PROCESS UNUSED VM ........") try: process_shutdown_unusedvm() process_purge_shutdownvm() except: log_exception() pass finally: db.commit() logger.debug("EXITING PROCESS UNUSED VM......")
def process_task_queue(task_event_id): """ Invoked when scheduler runs task of type 'vm_task' For every task, function calls the corresponding handler and updates the database on the basis of the response """ logger.info("\n ENTERING VM_TASK........") task_event_data = db.task_queue_event[task_event_id] task_queue_data = db.task_queue[task_event_data.task_id] vm_data = db.vm_data[task_event_data.vm_id] if task_event_data.vm_id != None else None try: #Update attention_time for task in the event table task_event_data.update_record(attention_time=get_datetime(), status=TASK_QUEUE_STATUS_PROCESSING) #Call the corresponding function from vm_helper logger.debug("Starting VM_TASK processing...") ret = task[task_queue_data.task_type](task_queue_data.parameters) logger.debug("Completed VM_TASK processing...") #On return, update the status and end time in task event table task_event_data.update_record(status=ret[0], message=ret[1], end_time=get_datetime()) if ret[0] == TASK_QUEUE_STATUS_FAILED: logger.debug("VM_TASK FAILED") logger.debug("VM_TASK Error Message: %s" % ret[1]) task_queue_data.update_record(status=TASK_QUEUE_STATUS_FAILED) elif ret[0] == TASK_QUEUE_STATUS_SUCCESS: # Create log event for the task logger.debug("VM_TASK SUCCESSFUL") if vm_data: _log_vm_event(vm_data, task_queue_data) # For successful task, delete the task from queue if db.task_queue[task_queue_data.id]: del db.task_queue[task_queue_data.id] if 'request_id' in task_queue_data.parameters: del db.request_queue[task_queue_data.parameters['request_id']] if task_event_data.task_type not in (VM_TASK_MIGRATE_HOST, VM_TASK_MIGRATE_DS): _send_task_complete_mail(task_event_data) except: msg = log_exception() task_event_data.update_record(status=TASK_QUEUE_STATUS_FAILED, message=msg) finally: db.commit() logger.info("EXITING VM_TASK........\n")
def process_vmdaily_checks(): """ Function will check for the shutdown VM's and sends email to the user""" logger.info("Entering VM's Daily Checks........") try: process_sendwarning_unusedvm() process_sendwarning_shutdownvm() except: log_exception() pass finally: db.commit() logger.debug("EXITING VM DAILY CHECKS........")
def bootup_baadal(): logger.info('Starting Baadal Bootup') update_constant('baadal_status', BAADAL_STATUS_UP_IN_PROGRESS) current.db.commit() # pool = ThreadPool(THREAD_POOL_COUNT) vms = current.db(~current.db.vm_data.status.belongs(current.VM_STATUS_UNKNOWN, current.VM_STATUS_IN_QUEUE)).select() for vm_detail in vms: revert_and_resume(vm_detail.id, vm_detail.vm_identity) # pool.add_task(revert_and_delete_snapshot, vm_detail.id, vm_detail.vm_identity, sys_snapshot.id, sys_snapshot.snapshot_name) # pool.wait_completion() update_constant('baadal_status', BAADAL_STATUS_UP) current.db.commit()
def shutdown_host(host_id_list): logger.info('Starting Host Shutdown') vms = current.db((current.db.vm_data.status.belongs(current.VM_STATUS_RUNNING, current.VM_STATUS_SUSPENDED)) & current.db.vm_data.host_id.belongs(host_id_list)).select() # pool = ThreadPool(THREAD_POOL_COUNT) for vm_detail in vms: snapshot_and_destroy(vm_detail.id, vm_detail.vm_identity, vm_detail.status) current.db.vm_event_log.insert(vm_id = vm_detail.id, attribute = 'VM Status', requester_id = -1, old_value = vm_detail.status, new_value = 'System Shutdown') current.db.commit()
def process_vmdaily_checks(): """ Check for the shutdown VM's and unused VM's and sends warning email to the user """ logger.info("Entering VM's Daily Checks........") try: process_sendwarning_unusedvm() process_sendwarning_shutdownvm() except: log_exception() pass finally: db.commit() logger.debug("EXITING VM DAILY CHECKS........")
def shutdown_host(host_id_list): logger.info('Starting Host Shutdown') vms = current.db( (current.db.vm_data.status.belongs(current.VM_STATUS_RUNNING, current.VM_STATUS_SUSPENDED)) & current.db.vm_data.host_id.belongs(host_id_list)).select() # pool = ThreadPool(THREAD_POOL_COUNT) for vm_detail in vms: snapshot_and_destroy(vm_detail.id, vm_detail.vm_identity, vm_detail.status) current.db.vm_event_log.insert(vm_id=vm_detail.id, attribute='VM Status', requester_id=-1, old_value=vm_detail.status, new_value='System Shutdown') current.db.commit()
def bootup_baadal(): logger.info('Starting Baadal Bootup') update_constant('baadal_status', BAADAL_STATUS_UP_IN_PROGRESS) current.db.commit() # pool = ThreadPool(THREAD_POOL_COUNT) vms = current.db(~current.db.vm_data.status.belongs( current.VM_STATUS_UNKNOWN, current.VM_STATUS_IN_QUEUE)).select() for vm_detail in vms: revert_and_resume(vm_detail.id, vm_detail.vm_identity) # pool.add_task(revert_and_delete_snapshot, vm_detail.id, vm_detail.vm_identity, sys_snapshot.id, sys_snapshot.snapshot_name) # pool.wait_completion() update_constant('baadal_status', BAADAL_STATUS_UP) current.db.commit()
def process_unusedvm_purge(): logger.info("ENTERING PURGE UNUSED VM ........") try: # Fetch all the VM's which are locked and whose delete warning date=today. for vm_data in db(db.vm_data.locked == True).select(db.vm_data.ALL): for vm_details in db(db.vm_event_log.vm_id==vm_data.id).select(db.vm_event_log.ALL,orderby = ~db.vm_event_log.id,limitby=(0,1)): daysDiff=(get_datetime()-vm_data.delete_warning_date).days if(vm_details.new_value == "Shutdown" and daysDiff >= 0): logger.info("Need to delete the VM ID:"+str(vm_data.id)) add_vm_task_to_queue(vm_data.id,VM_TASK_DELETE) # make an entry in task queue so that scheduler can pick up and delete the VM. except: log_exception() pass finally: db.commit() logger.debug("EXITING PURGE UNUSED VM ........")
def shutdown_baadal(): logger.info('Starting Baadal Shutdown') update_constant('baadal_status', BAADAL_STATUS_DOWN_IN_PROGRESS) current.db.commit() vms = current.db(current.db.vm_data.status.belongs(current.VM_STATUS_RUNNING, current.VM_STATUS_SUSPENDED)).select() # pool = ThreadPool(THREAD_POOL_COUNT) for vm_detail in vms: snapshot_and_destroy(vm_detail.id, vm_detail.vm_identity, vm_detail.status) current.db.vm_event_log.insert(vm_id = vm_detail.id, attribute = 'VM Status', requester_id = -1, old_value = vm_detail.status, new_value = 'System Shutdown') current.db.commit() # pool.add_task(snapshot_and_suspend, vm_detail.id, vm_detail.vm_identity) # pool.wait_completion() update_constant('baadal_status', BAADAL_STATUS_DOWN) current.db.commit()
def process_shutdown_unusedvm(): logger.info("ENTERING SHUTDOWN UNUSED VM ........") try: # Fetch all the VM's which are locked and whose shutdown_warning_date=today. vmCPUThreshold = config.get("GENERAL_CONF", "cpu_threshold_limit") vmreadThreshold = config.get("GENERAL_CONF", "nwRead_threshold_limit") vmwriteThreshold = config.get("GENERAL_CONF", "nwWrite_threshold_limit") thresholdcontext = dict(CPUThreshold=vmCPUThreshold, ReadThreshold=vmreadThreshold, WriteThreshold=vmwriteThreshold) for vmData in db(db.vm_data.shutdown_warning_date!=None).select(db.vm_data.ALL): daysDiff=(get_datetime()-vmData.shutdown_warning_date).days if(daysDiff >= 0): '''Again compare the data for last 20 days from rrd logs ''' retVal=compare_rrd_data_with_threshold(vmData.vm_identity,thresholdcontext) logger.info(" DaysDiff are "+str(daysDiff)+" return value is "+str(retVal)) if(retVal == True): logger.info("Need to shutdown the VM ID:"+str(vmData.id)) add_vm_task_to_queue(vmData.id,VM_TASK_DESTROY) # make an entry in task queue so that scheduler can pick up and shutdown the VM. else: logger.info("No Need to shutdown the VM ID:"+str(vmData.id)+" as VM is in use now. ") #update db to clean the shutdown warning date db(db.vm_data.id == vmData.id).update(shutdown_warning_date=None) else: logger.info("No need to process purge for the VM:"+str(vmData.id)) except: log_exception() pass finally: db.commit() logger.debug("EXITING SHUTDOWN UNUSED VM ........")
def process_sendwarning_unusedvm(): logger.info("Entering send warning to unused VM........") try: ''' performing daily checks for network usage ''' vmCPUThreshold = config.get("GENERAL_CONF", "cpu_threshold_limit") vmreadThreshold = config.get("GENERAL_CONF", "nwRead_threshold_limit") vmwriteThreshold = config.get("GENERAL_CONF", "nwWrite_threshold_limit") thresholdcontext = dict(CPUThreshold=vmCPUThreshold, ReadThreshold=vmreadThreshold, WriteThreshold=vmwriteThreshold) logger.info("checking network usage with threshold values as CPUThreshold is:"+str(thresholdcontext['CPUThreshold'])+" WriteThreshold is :"+str(thresholdcontext['WriteThreshold'])+" ReadThreshold is :"+ str(thresholdcontext['ReadThreshold'])) vms = db(db.vm_data.status.belongs(VM_STATUS_RUNNING, VM_STATUS_SUSPENDED) & (db.vm_data.shutdown_warning_date == None) & (db.vm_data.start_time < (get_datetime() - timedelta(days=20)))).select() '''check vm should have been created 20days back''' for vm in vms: logger.info("comparing threshold for the vm "+ str(vm.vm_identity)) send_email=0 retVal=compare_rrd_data_with_threshold(vm.vm_identity,thresholdcontext) if(retVal == True): vm_users = [] vm_name = "" for user in db((db.user_vm_map.vm_id == vm.id) & (db.user_vm_map.vm_id == db.vm_data.id) & (db.vm_data.shutdown_warning_date == None )).select(db.user_vm_map.user_id,db.vm_data.vm_name): send_email=1 vm_users.append(user.user_vm_map.user_id) vm_name=user.vm_data.vm_name if (send_email == 1): vm_shutdown_time=send_email_vm_warning(VM_TASK_WARNING_SHUTDOWN,vm_users,vm_name,'') logger.debug("Mail sent for vm_name:"+str(vm_name)+"|shutdown time returned from the function:"+ str(vm_shutdown_time)) db(db.vm_data.id == vm.id).update(shutdown_warning_date=vm_shutdown_time) db.commit() else: logger.debug("Warning Email to use the VM has already been sent to VM_ID:"+str(vm.id)) else: logger.info("VM:"+str(vm.id)+" is in use.. no need to send shutdown warning mail ...") except: log_exception() pass finally: db.commit() logger.debug("EXITING send warning to unused VM........")
def process_object_task(task_event_id): """Invoked when scheduler runs task of type 'object_task' For every task, function calls the corresponding handler and updates the database on the basis of the response """ logger.info("\n ENTERING OBJECT_TASK ........") task_event_data = db.task_queue_event[task_event_id] task_queue_data = db.task_queue[task_event_data.task_id] object_data = db.object_store_data[task_event_data.parameters[ 'vm_id']] if task_event_data.parameters['vm_id'] != None else None try: #Update attention_time for task in the event table task_event_data.update_record(attention_time=get_datetime(), status=TASK_QUEUE_STATUS_PROCESSING) #Call the corresponding function from vm_helper logger.debug("Starting OBJECT_TASK processing...") ret = create_object_store(task_event_id, object_data) logger.debug("Completed OBJECT_TASK processing...") #On return, update the status and end time in task event table task_event_data.update_record(status=ret[0], message=ret[1], end_time=get_datetime()) if ret[0] == TASK_QUEUE_STATUS_FAILED: logger.debug("OBJECT_TASK FAILED") logger.debug("OBJECT_TASK Error Message: %s" % ret[1]) task_queue_data.update_record(status=TASK_QUEUE_STATUS_FAILED) elif ret[0] == TASK_QUEUE_STATUS_SUCCESS: # Create log event for the task logger.debug("OBJECT_TASK SUCCESSFUL") if object_data: logger.info("\n object_data: %s" % object_data) # For successful task, delete the task from queue if db.task_queue[task_queue_data.id]: del db.task_queue[task_queue_data.id] if 'request_id' in task_queue_data.parameters: del db.request_queue[task_queue_data.parameters['request_id']] _send_object_task_complete_mail(task_event_data, object_data['object_store_name']) except: msg = log_exception() task_event_data.update_record(status=TASK_QUEUE_STATUS_FAILED, message=msg) finally: db.commit() logger.info("EXITING OBJECT_TASK........\n")
def process_sendwarning_shutdownvm(): logger.info("Entering Process send warning mail to shutdown vm........") try: vmShutDownDays = config.get("GENERAL_CONF", "shutdown_vm_days") send_email=0 for vm_id in db().select(db.vm_event_log.vm_id, distinct=True): for vm_details in db(db.vm_event_log.vm_id==vm_id['vm_id']).select(db.vm_event_log.ALL,orderby = ~db.vm_event_log.id,limitby=(0,1)): daysDiff=(get_datetime()-vm_details.timestamp).days vm_shutdown_time=vm_details.timestamp logger.info("VM details are VM_ID:" + str(vm_details['vm_id'])+ "|ID:"+str(vm_details['id'])+"|new_values is:"+str(vm_details['new_value'])+"|daysDiff:" + str(daysDiff)+"|vmShutDownDays:"+vmShutDownDays+"|vm_shutdown_time :"+str(vm_shutdown_time)) if (vm_details.new_value == "Shutdown" and int(daysDiff)>=int(vmShutDownDays)): vm_users = [] vm_name = "" for user in db((db.user_vm_map.vm_id == vm_details.vm_id) & (db.user_vm_map.vm_id == db.vm_data.id) & (db.vm_data.locked != True) & (db.vm_data.delete_warning_date == None )).select(db.user_vm_map.user_id,db.vm_data.vm_name): send_email=1 vm_users.append(user.user_vm_map.user_id) vm_name=user.vm_data.vm_name if (send_email == 1): vm_delete_time=send_email_vm_warning(VM_TASK_WARNING_DELETE,vm_users,vm_name,vm_shutdown_time) logger.debug("Mail sent for vm_id:"+str(vm_details.vm_id)+"|vm_name:"+str(vm_name)+"|delete time:"+ str(vm_delete_time)) db(db.vm_data.id == vm_details.vm_id).update(locked=True, delete_warning_date=vm_delete_time) send_email=0 else: logger.debug("Email has already been sent to VM_ID:"+str(vm_details.vm_id)) else: logger.info("VM:"+str(vm_details.vm_id)+" is not shutdown for: "+str(vmShutDownDays)+"(configured) days") except: log_exception() pass finally: db.commit() logger.debug("EXITING Send warning to shutdown vm........")
def process_vmdaily_checks(): """ Function will check for the shutdown VM's and sends email to the user""" logger.info("Entering VM's Daily Checks........") try: vmShutDownDays = config.get("GENERAL_CONF", "shutdown_vm_days") send_email=0 for vm_id in db().select(db.vm_event_log.vm_id, distinct=True): for vm_details in db(db.vm_event_log.vm_id==vm_id['vm_id']).select(db.vm_event_log.ALL,orderby = ~db.vm_event_log.id,limitby=(0,1)): daysDiff=(get_datetime()-vm_details.timestamp).days vm_shutdown_time=vm_details.timestamp logger.info("VM details are VM_ID:" + str(vm_details['vm_id'])+ "|ID:"+str(vm_details['id'])+"|new_values is:"+str(vm_details['new_value'])+"|daysDiff:" + str(daysDiff)+"|vmShutDownDays:"+vmShutDownDays+"|vm_shutdown_time :"+str(vm_shutdown_time)) if (vm_details.new_value == "Shutdown" and int(daysDiff)>=int(vmShutDownDays)): vm_users = [] vm_name = "" for user in db((db.user_vm_map.vm_id == vm_details.vm_id) & (db.user_vm_map.vm_id == db.vm_data.id) & (db.vm_data.locked !='T') & (db.vm_data.delete_warning_date == None )).select(db.user_vm_map.user_id,db.vm_data.vm_name): send_email=1 vm_users.append(user.user_vm_map.user_id) vm_name=user.vm_data.vm_name if (send_email == 1): vm_delete_time = send_email_delete_vm_warning(vm_users,vm_name,vm_shutdown_time) logger.debug("Mail sent for vm_name:"+str(vm_name)+"|delete time returned from the function:"+ str(vm_delete_time)) db(db.vm_data.id == vm_details.vm_id).update(locked=True, delete_warning_date=vm_delete_time) else: logger.debug("Email has already been sent to VM_ID:"+str(vm_details.vm_id)) else: logger.info("VM:"+str(vm_details.vm_id)+" is not shutdown ..") except: log_exception() pass finally: db.commit() logger.debug("EXITING VM DAILY CHECKS........")
def check_vm_sanity(host_id = 0): """ Checks if database information of VM status is in sync with ground reality. In case of discrepancy database is updated. - For each host, get list of the domains(running and not running). - If VM has migrated to another host; host information updated in database. - If VM not in sync with actual state of VM; status of VM updated in DB. - If VM is not present in database, it is marked Orphan. - If VM is not found on any of the hosts, it is marked Undefined. """ vmcheck=[] vm_list = [] if host_id == 0: hosts=db(db.host.status == HOST_STATUS_UP).select() else: hosts=db(db.host.id == host_id).select() for host in hosts: try: logger.info('Starting sanity check for host %s' %(host.host_name)) #Get list of the domains(running and not running) on the hypervisor domains = get_host_domains(host.host_ip.private_ip) for dom in domains: try: domain_name = dom.name() vm = db((db.vm_data.vm_identity == domain_name) & (db.vm_data.status.belongs(VM_STATUS_RUNNING, VM_STATUS_SUSPENDED, VM_STATUS_SHUTDOWN))).select().first() vm_state = dom.info()[0] status = vminfo_to_state(vm_state) if(vm): if(vm.host_id != host.id): vmcheck.append({'vm_id':vm.id, 'host':host.host_name, 'host_id':host.id, 'vmname':vm.vm_name, 'status':status, 'message':'Moved from '+vm.host_id.host_name+' to '+host.host_name, 'operation':'None'})#Bad VMs #If VM has been migrated to another host; Host information updated db(db.vm_data.vm_identity == domain_name).update(host_id = host.id) else: vmcheck.append({'vm_id':vm.id, 'host':host.host_name, 'host_id':host.id, 'vmname':vm.vm_name, 'status':status, 'message':'VM is on expected host '+vm.host_id.host_name, 'operation':'None'})#Good VMs if vm_state_map[vm_state] != vm.status: logger.info("vm_identity="+str(domain_name)+" vm_state_map[vm_state]="+str(vm_state_map[vm_state])+"and vm.status is" + str(vm.status)) #If not in sync with actual state of VM; status of VM updated in DB db(db.vm_data.vm_identity == domain_name).update(status = vm_state_map[vm_state]) #Adding into vm_event_log about the vm details db.vm_event_log.insert(vm_id = vm.id, attribute = 'VM Status', requester_id = SYSTEM_USER, old_value = get_vm_status(vm.status), new_value = get_vm_status(vm_state_map[vm_state])) vm_list.append(vm.vm_identity) elif vm_state != VIR_DOMAIN_CRASHED: vmcheck.append({'host':host.host_name, 'host_id':host.id, 'vmname':dom.name(), 'status':status, 'message':'Orphan, VM is not in database', 'operation':'Orphan'})#Orphan VMs except Exception: log_exception() if(vm): vmcheck.append({'vmname':vm.vm_name, 'host':'Unknown', 'host_id':'0', 'status':'Unknown', 'message':'Some Error Occurred', 'operation':'Error'}) except:pass db.commit() if host_id == 0: db_vms=db(db.vm_data.status.belongs(VM_STATUS_RUNNING, VM_STATUS_SUSPENDED, VM_STATUS_SHUTDOWN)).select() for db_vm in db_vms: if(db_vm.vm_identity not in vm_list): vmcheck.append({'vmname':db_vm.vm_identity, 'host':db_vm.host_id.host_name, 'host_id':db_vm.host_id, 'status':'Undefined', 'message':'VM not found', 'operation':'Undefined'}) return vmcheck
def check_vm_sanity(host_id=0): vmcheck = [] vm_list = [] if host_id == 0: hosts = db(db.host.status == HOST_STATUS_UP).select() else: hosts = db(db.host.id == host_id).select() for host in hosts: try: logger.info('Starting sanity check for host %s' % (host.host_name)) #Get list of the domains(running and not running) on the hypervisor domains = get_host_domains(host.host_ip.private_ip) for dom in domains: try: domain_name = dom.name() vm = db((db.vm_data.vm_identity == domain_name) & (db.vm_data.status.belongs( VM_STATUS_RUNNING, VM_STATUS_SUSPENDED, VM_STATUS_SHUTDOWN))).select().first() vm_state = dom.info()[0] status = vminfo_to_state(vm_state) if (vm): if (vm.host_id != host.id): vmcheck.append({ 'vm_id': vm.id, 'host': host.host_name, 'host_id': host.id, 'vmname': vm.vm_name, 'status': status, 'message': 'Moved from ' + vm.host_id.host_name + ' to ' + host.host_name, 'operation': 'None' }) #Bad VMs #If VM has been migrated to another host; Host information updated db(db.vm_data.vm_identity == domain_name).update( host_id=host.id) else: vmcheck.append({ 'vm_id': vm.id, 'host': host.host_name, 'host_id': host.id, 'vmname': vm.vm_name, 'status': status, 'message': 'VM is on expected host ' + vm.host_id.host_name, 'operation': 'None' }) #Good VMs if vm_state_map[vm_state] != vm.status: #If not in sync with actual state of VM; status of VM updated in DB db(db.vm_data.vm_identity == domain_name).update( status=vm_state_map[vm_state]) #Adding into vm_event_log about the vm details db.vm_event_log.insert(vm_id=vm.id, attribute='VM Status', requester_id=SYSTEM_USER, old_value=get_vm_status( vm.status), new_value=get_vm_status( vm_state_map[vm_state])) vm_list.append(vm.vm_identity) elif vm_state != VIR_DOMAIN_CRASHED: vmcheck.append({ 'host': host.host_name, 'host_id': host.id, 'vmname': dom.name(), 'status': status, 'message': 'Orphan, VM is not in database', 'operation': 'Orphan' }) #Orphan VMs except Exception: log_exception() if (vm): vmcheck.append({ 'vmname': vm.vm_name, 'host': 'Unknown', 'host_id': '0', 'status': 'Unknown', 'message': 'Some Error Occurred', 'operation': 'Error' }) except: pass db.commit() if host_id == 0: db_vms = db( db.vm_data.status.belongs(VM_STATUS_RUNNING, VM_STATUS_SUSPENDED, VM_STATUS_SHUTDOWN)).select() for db_vm in db_vms: if (db_vm.vm_identity not in vm_list): vmcheck.append({ 'vmname': db_vm.vm_identity, 'host': db_vm.host_id.host_name, 'host_id': db_vm.host_id, 'status': 'Undefined', 'message': 'VM not found', 'operation': 'Undefined' }) return vmcheck