def test_procesS_user_connections(self): user_experience_tools.process_user_connections(self.app_node) us_exp1 = UserExperience.objects.filter(user=self.user3).filter(connection_opened__isnull=False) self.failIfEqual(us_exp1[0].connection_opened, None) log.debug("****TEST**** connection opened = " + str(us_exp1[0].connection_opened)) self.app_node.sessions.remove(self.session) user_experience_tools.process_user_connections(self.app_node) us_exp2 = UserExperience.objects.filter(user=self.user3).filter(connection_closed__isnull=False) self.failIfEqual(us_exp2[0].connection_closed, None) log.debug("****TEST**** connection closed = " + str(us_exp2[0].connection_closed))
def test_procesS_user_connections(self): user_experience_tools.process_user_connections(self.app_node) us_exp1 = UserExperience.objects.filter(user=self.user3).filter( connection_opened__isnull=False) self.failIfEqual(us_exp1[0].connection_opened, None) log.debug("****TEST**** connection opened = " + str(us_exp1[0].connection_opened)) self.app_node.sessions.remove(self.session) user_experience_tools.process_user_connections(self.app_node) us_exp2 = UserExperience.objects.filter(user=self.user3).filter( connection_closed__isnull=False) self.failIfEqual(us_exp2[0].connection_closed, None) log.debug("****TEST**** connection closed = " + str(us_exp2[0].connection_closed))
def run(self, app): # Create an instance of the logger log = get_logger('vdi') # Create the cluster object to help us manage the cluster cluster = AppCluster(app.pk) # Clean up all idle users on all nodes for this application cluster log.debug('APP NAME %s' % app.name) cluster.logout_idle_users() log.debug("Checking for active clusters") for node in cluster.active: log.debug("Found active host") osutil_node = osutils.get_os_object( node.ip, settings.MEDIA_ROOT + str(node.application.ssh_key)) user_experience_tools.process_user_connections(osutil_node) # Handle vms we were waiting on to boot up booting = driver_tools.get_instances(cluster.booting) for vm in booting: dns_name = vm.public_addresses[0] log.debug('ASDF = %s' % dns_name) if dns_name.find("amazonaws.com") > -1: # Resolve the domain name into an IP address # This adds a dependancy on the 'host' command output = Popen(["host", dns_name], stdout=PIPE).communicate()[0] ip = '.'.join( re.findall( '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)', output)[0]) try: # TODO: remove the hard coded '3389' & '22' below. '3389' is for RDP and '22' is for SSH # TODO: remove the arbitrary '3' second timeout below socket.create_connection((ip, 22), 3) #socket.create_connection((ip,3389),3) except Exception as e: log.debug("Server %s is not yet available" % ip) pass else: instance = Instance.objects.filter(instanceId=vm.id)[0] booting.remove(vm) instance.ip = ip instance.state = 2 instance.save() log.debug( "Moving instance %s into enabled state with ip %s" % (instance.instanceId, ip)) num_booting = len(booting) if num_booting > 0: log.debug( "Application cluster '%s' is still waiting for %s cluster nodes to boot" % (cluster.name, len(booting))) # Consider if the cluster needs to be scaled log.debug('Considering %s app cluster for scaling ...' % cluster.name) # Should I scale up? log.debug( '%s is avail (%s) < req (%s)?' % (cluster.app.name, cluster.avail_headroom, cluster.req_headroom)) if cluster.avail_headroom < cluster.req_headroom: # Yes I should scale up space_needed = cluster.req_headroom - cluster.avail_headroom servers_needed = int( math.ceil(space_needed / float(cluster.app.users_per_small))) log.debug( 'Available headroom (%s) is less than the cluster headroom goal (%s). Starting %s additional cluster nodes now' % (cluster.avail_headroom, cluster.req_headroom, servers_needed)) for i in range(servers_needed): cluster.start_node() # Handle instances we are supposed to shut down toTerminate = [] for host in cluster.shutting_down: log.debug('ASDASDASD %s' % host.instanceId) try: osutil_node = osutils.get_os_object( host.ip, settings.MEDIA_ROOT + str(self.host.application.ssh_key)) log.debug( 'Node %s is waiting to be shut down and has %s connections' % (host.ip, osutil_node.sessions)) if osutil_node.sessions == []: toTerminate.append(host) host.shutdownDateTime = datetime.now() host.save() except HostNotConnectableError: # Ignore this host that doesn't seem to be ssh'able, but log it as an error log.warning( 'Node %s is NOT sshable and should be looked into. It is currently waiting to shutdown' ) driver_tools.terminate_instances(toTerminate) # Should I scale down? overprov_num = cluster.avail_headroom - cluster.req_headroom log.debug('overprov (%s) avail (%s) required(%s)' % (overprov_num, cluster.avail_headroom, cluster.req_headroom)) # Reverse the list to try to remove the servers at the end of the waterfall inuse_reverse = cluster.inuse_map inuse_reverse.reverse() for (host, inuse) in inuse_reverse: # The node must have 0 sessions and the cluster must be able to be smaller while still leaving enough headroom if int(inuse) == 0 and overprov_num >= cluster.app.users_per_small: overprov_num = overprov_num - cluster.app.users_per_small host.state = 4 host.save() log.debug( 'Application Server %s has no sessions. Removing that node from the cluster!' % host.ip) return 'scaling complete @TODO put scaling event summary in this output'
def run(self, app): # Create an instance of the logger log = get_logger('vdi') # Create the cluster object to help us manage the cluster cluster = AppCluster(app.pk) # Clean up all idle users on all nodes for this application cluster log.debug('APP NAME %s'%app.name) cluster.logout_idle_users() log.debug("Checking for active clusters") for node in cluster.active: log.debug("Found active host") osutil_node = osutils.get_os_object(node.ip, settings.MEDIA_ROOT + str(node.application.ssh_key)) user_experience_tools.process_user_connections(osutil_node) # Handle vms we were waiting on to boot up booting = driver_tools.get_instances(cluster.booting) for vm in booting: dns_name = vm.public_addresses[0] log.debug('ASDF = %s' % dns_name) if dns_name.find("amazonaws.com") > -1: # Resolve the domain name into an IP address # This adds a dependancy on the 'host' command output = Popen(["host", dns_name], stdout=PIPE).communicate()[0] ip = '.'.join(re.findall('(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)', output)[0]) try: # TODO: remove the hard coded '3389' & '22' below. '3389' is for RDP and '22' is for SSH # TODO: remove the arbitrary '3' second timeout below socket.create_connection((ip,22),3) #socket.create_connection((ip,3389),3) except Exception as e: log.debug("Server %s is not yet available" % ip) pass else: instance = Instance.objects.filter(instanceId=vm.id)[0] booting.remove(vm) instance.ip = ip instance.state = 2 instance.save() log.debug("Moving instance %s into enabled state with ip %s" % (instance.instanceId,ip)) num_booting = len(booting) if num_booting > 0: log.debug("Application cluster '%s' is still waiting for %s cluster nodes to boot" % (cluster.name,len(booting))) # Consider if the cluster needs to be scaled log.debug('Considering %s app cluster for scaling ...' % cluster.name) # Should I scale up? log.debug('%s is avail (%s) < req (%s)?' % (cluster.app.name, cluster.avail_headroom, cluster.req_headroom)) if cluster.avail_headroom < cluster.req_headroom: # Yes I should scale up space_needed = cluster.req_headroom - cluster.avail_headroom servers_needed = int(math.ceil(space_needed / float(cluster.app.users_per_small))) log.debug('Available headroom (%s) is less than the cluster headroom goal (%s). Starting %s additional cluster nodes now' % (cluster.avail_headroom,cluster.req_headroom,servers_needed)) for i in range(servers_needed): cluster.start_node() # Handle instances we are supposed to shut down toTerminate = [] for host in cluster.shutting_down: log.debug('ASDASDASD %s' % host.instanceId) try: osutil_node = osutils.get_os_object(host.ip, settings.MEDIA_ROOT + str(self.host.application.ssh_key)) log.debug('Node %s is waiting to be shut down and has %s connections' % (host.ip, osutil_node.sessions)) if osutil_node.sessions == []: toTerminate.append(host) host.shutdownDateTime = datetime.now() host.save() except HostNotConnectableError: # Ignore this host that doesn't seem to be ssh'able, but log it as an error log.warning('Node %s is NOT sshable and should be looked into. It is currently waiting to shutdown') driver_tools.terminate_instances(toTerminate) # Should I scale down? overprov_num = cluster.avail_headroom - cluster.req_headroom log.debug('overprov (%s) avail (%s) required(%s)' % (overprov_num,cluster.avail_headroom,cluster.req_headroom)) # Reverse the list to try to remove the servers at the end of the waterfall inuse_reverse = cluster.inuse_map inuse_reverse.reverse() for (host,inuse) in inuse_reverse: # The node must have 0 sessions and the cluster must be able to be smaller while still leaving enough headroom if int(inuse) == 0 and overprov_num >= cluster.app.users_per_small: overprov_num = overprov_num - cluster.app.users_per_small host.state = 4 host.save() log.debug('Application Server %s has no sessions. Removing that node from the cluster!' % host.ip) return 'scaling complete @TODO put scaling event summary in this output'