예제 #1
0
	def get_data(self, context):
		log = logging.getLogger(__name__)

		starting_color = 0xfa5858
		step_change = 0x2010

		# get instances with listed user
		current_owner = ''
		current_cost = 0.0
		idx = 0
		data = []
		pricecache = {}
		caller_groups = remove_managergroup(set(self.request.user.groups.all()))

		for inst in Instance.objects.all().exclude(userid='').filter(state='running').order_by('userid'):
			if inst.userid.upper() != self.request.user.username.upper():
				# check if caller and instance owner share a group
				try:
					if caller_groups.isdisjoint(set(User.objects.get(username=inst.userid.upper()).groups.all())):
						continue
				except User.DoesNotExist:
					continue

			if current_owner != inst.owner:
				data.append({
					'value': round(current_cost, 4),
					'color': '#{:x}'.format(starting_color + (idx * step_change)),
					'highlight': "#FFC870",
					'label': current_owner
				})
				idx += 1
				current_owner = inst.owner
				current_cost = 0.0

			if inst.instance_type not in pricecache:
				iprice = getPrice(inst.instance_type) * (1.0 - AWS_DISCOUNT)
				if iprice > 0.0:
					current_cost += iprice
					pricecache[inst.instance_type] = iprice
				else:
					log.error("Instance of type {} not found in price db.".format(inst.instance_type))
			else:
				current_cost += pricecache[inst.instance_type]

		# last user
		data.append({
			'value': round(current_cost, 4),
			'color': '#{:x}'.format(starting_color + (idx * step_change)),
			'highlight': "#FFC870",
			'label': current_owner
		})

		return data
예제 #2
0
	def get(self, db_instances, message='', force_set_display_state=''):
		try:
			response = []
			for instance in db_instances:
				# On aws failure, make it clear that the state is unknown
				if force_set_display_state:
					self.logger.info("Instance state forced set: " + force_set_display_state)
					state = force_set_display_state
				else:
					state = instance.state
				inst = {
					'id': instance.instance_id,
					'type': instance.instance_type,
					'private_ip': instance.private_ip,
					'state': {
						'Name': state,
						'progress': instance.progress_status,
						'integer': instance.progress_integer,
					},
					'time': get_uptime(instance.start_at),
					'tags': list(instance.tag_set.values('Name', 'Value')),
					'price': '{:.2}'.format(getPrice(instance.instance_type) * (1.0 - AWS_DISCOUNT)),
					'sc': instance.software_config.name,
					'dns_url': instance.dns_url,
					'owner': instance.owner,
					'message': message
				}
				try:
					extra_values = list(DisplayValue.objects.filter(instance=instance))
					for display_object in extra_values:
						inst[display_object.table_header] = display_object.table_value
				except DisplayValue.DoesNotExist:
					self.logger.info("No extra DisplayValues", extra={"instance_id": instance.instance_id})
				response.append(inst)
			self.logger.debug(str(response))
			return response
		except Exception as e:
			self.logger.exception(e)
		return []
예제 #3
0
    def get_data(self, context):
        log = self.logger
        action = self.kwargs['action']
        instanceid = self.kwargs['instanceid']

        # make sure user owns this instance

        inst = get_object_or_404(Instance, instance_id=instanceid)

        is_authorized = has_permission(self.request.user, inst.userid)

        if not is_authorized:
            return {'action': 'invalid', 'status': 'unauthorized'}

        conn = boto3.resource('ec2', region_name=AWS_REGION)
        botoinstance = conn.Instance(id=instanceid)
        # ====================================
        if action == 'start':
            log.info("Entering " + str(action))
            if botoinstance.state['Name'] != 'stopped':
                if inst.state == "stopped":
                    inst.state = botoinstance.state['Name']
                    inst.start_at = datetime.now(timezone(TIME_ZONE))
                    inst.save()
                return {
                    'action': 'invalid',
                    'status': 'failed',
                    'exception': 'Instance started already.'
                }
            try:
                botoinstance.start()
                bill = BillingData()
                bill.ongoing = True
                bill.instance_type = inst.instance_type
                bill.instance_name = inst.instance_id
                if inst.current_bill is not None:
                    bill.charge_name = inst.current_bill.charge_name
                else:
                    bill.charge_name = DEFAULT_CHARGE_CODE

                bill.user = User.objects.get(username=inst.userid)
                bill.price = getPrice(
                    inst.instance_type) * (1.0 - AWS_DISCOUNT)
                bill.start_time = datetime.now(timezone('UTC'))
                bill.save()
                inst.current_bill = bill
                inst.state = 'starting'
                inst.start_at = datetime.now(timezone(TIME_ZONE))
                inst.save()
                log.info("Exiting " + str(action))
            except Exception as e:
                log.exception(e)
                return {
                    'action': 'invalid',
                    'status': 'failed',
                    'exception': str(e)
                }
        # ====================================
        elif action == 'stop':
            log.info("Entering " + str(action))

            if botoinstance.state['Name'] != 'running':
                if inst.state == "running":
                    inst.state = botoinstance.state['Name']
                    inst.start_at = datetime.now(timezone(TIME_ZONE))
                    inst.save()
                return {
                    'action': 'invalid',
                    'status': 'failed',
                    'exception': "Instance is not running."
                }
            try:
                botoinstance.stop()
                bill = inst.current_bill
                if bill is not None:
                    bill.ongoing = False
                    bill.end_time = datetime.now(timezone('UTC'))
                    bill.save()
                inst.state = 'stopping'
                inst.stop_at = datetime.now(timezone(TIME_ZONE))
                inst.save()
                log.info("Exiting " + str(action))
            except Exception as e:
                log.exception(e)
                return {
                    'action': 'invalid',
                    'status': 'failed',
                    'exception': str(e)
                }
        # ====================================
        elif action == 'restart':
            log.info("Entering " + str(action))
            if botoinstance.state['Name'] != 'running':
                return {
                    'action': 'invalid',
                    'status': 'failed',
                    'exception': 'Instance is not running.'
                }
            try:
                botoinstance.reboot()
                inst.stop_at = datetime.now(timezone(TIME_ZONE))
                inst.start_at = datetime.now(timezone(TIME_ZONE))
                inst.state = 'restarting'
                inst.save()
                log.info("Exiting " + str(action))
            except Exception as e:
                log.exception(e)
                return {
                    'action': 'invalid',
                    'status': 'failed',
                    'exception': str(e)
                }
        # ====================================
        elif action == 'terminate':
            log.info("Entering " + str(action))
            if botoinstance.state['Name'].startswith('termin'):
                return {
                    'action': 'invalid',
                    'status': 'failed',
                    'exception': 'Instance already terminated.'
                }
            try:
                # kill the Cloudformation stack
                stack = inst.stack_id
                if stack is not None:
                    client = boto3.client("cloudformation",
                                          region_name=AWS_REGION)
                    ec2_client = boto3.client('ec2', region_name=AWS_REGION)
                    client.delete_stack(StackName=stack.stack_id)
                    stack.delete()
                    state = ''
                    while state != 'shutting-down':
                        state = ec2_client.describe_instances(
                            InstanceIds=[inst.instance_id]
                        )['Reservations'][0]['Instances'][0]['State']['Name']
                        time.sleep(3)
                else:
                    return {
                        'action':
                        'invalid',
                        'status':
                        'failed',
                        'exception':
                        'Instance has no stack and will not be terminated.'
                    }
                inst.stack_id = None
                bill = inst.current_bill
                if bill is not None:
                    bill.ongoing = False
                    bill.end_time = datetime.now(timezone('UTC'))
                    bill.save()
                deleteDnsEntry(inst.instance_id, inst.private_ip)
                inst.state = 'terminating'
                inst.progress_integer = 100
                inst.progress_status = 'done'
                inst.archived = True
                inst.stop_at = datetime.now(timezone(TIME_ZONE))
                inst.save()
                log.info("Exiting " + str(action))
            except Exception as e:
                log.exception(e)
                return {
                    'action': 'invalid',
                    'status': 'failed',
                    'exception': str(e)
                }
        # ====================================
        elif action == 'archive':
            log.info("Entering " + str(action))
            try:
                # Check for CF
                stack = inst.stack_id
                if stack:
                    client = boto3.client("cloudformation",
                                          region_name=AWS_REGION)
                    client.delete_stack(StackName=stack.stack_id)
                    stack.delete()
                inst.stack_id = None
                inst.archived = True
                inst.stop_at = datetime.now(timezone(TIME_ZONE))
                deleteDnsEntry(inst.instance_id, inst.private_ip)

                bill = inst.current_bill
                if bill:
                    bill.ongoing = False
                    bill.end_time = datetime.now(timezone('UTC'))
                    bill.save()
                inst.save()
                log.info("Exiting " + str(action))
            except Exception as e:
                log.exception(e)
                return {
                    'action': 'invalid',
                    'status': 'failed',
                    'exception': str(e)
                }
        # ====================================
        elif action == 'fakeok':
            # here for UI testing
            pass
        # ====================================
        else:
            log.error("Invalid verb passed to changeInstanceState")
            return {
                'action': 'invalid',
                'status': 'failed',
                'exception': "Invalid verb passed to changeInstanceState"
            }

        # add a short delay in return to try to address non-changing ui
        time.sleep(2)
        return {'action': action, 'status': 'ok'}
예제 #4
0
파일: launch.py 프로젝트: FINRAOS/MLiy
    def launch_cluster(self, form):

        cluster = None

        try:
            NETWORK_ID = self.user.username
            UNIQUE_ID = self.launch_id

            AGS = self.group_config.ags
            SDLC = self.group_config.sdlc
            COST_CENTER = self.group_config.cost_center
            PURPOSE = self.purpose
            BID = self.bid

            if self.software_config.master_security_group is not None:
                MASTER_SG = self.software_config.master_security_group.sgid
                SLAVE_SG = self.software_config.slave_security_group.sgid
                logging.debug(self.software_config.name)
            else:
                MASTER_SG = ""
                SLAVE_SG = ""
                logging.debug(self.software_config.name)

            ADDITIONAL_MASTER_SG = ""
            ADDITIONAL_SLAVE_SG = ""

            if self.software_config.additional_master_security_groups is not None:
                for security_group in self.software_config.additional_master_security_groups.all(
                ):
                    ADDITIONAL_MASTER_SG = ADDITIONAL_MASTER_SG + security_group.sgid + ','
                ADDITIONAL_MASTER_SG = ADDITIONAL_MASTER_SG.rstrip(',')
            if self.software_config.additional_slave_security_groups is not None:
                for security_group in self.software_config.additional_slave_security_groups.all(
                ):
                    ADDITIONAL_SLAVE_SG = ADDITIONAL_SLAVE_SG + security_group.sgid + ','
                ADDITIONAL_SLAVE_SG = ADDITIONAL_SLAVE_SG.rstrip(',')

            subnets = []

            if SDLC == "PRODY":
                subnets = getSubnets(findVPCID('analytics'), 'dynamic')
            else:
                subnets = getSubnets(findVPCID('general'), 'dynamic')

            SUBNET_NUM = random.randint(0, len(subnets) - 1)
            SUBNET_ID = subnets[SUBNET_NUM]['SubnetId']

            OWNER = "{} {}".format(self.user.first_name, self.user.last_name)

            group_name = self.group_config.override_instance_name
            software_name = self.software_config.instance_name
            if group_name:
                name = group_name + "-" + self.market_type + "-" + UNIQUE_ID + "-" + NETWORK_ID
            else:
                name = software_name + "-" + self.market_type + "-" + UNIQUE_ID + "-" + NETWORK_ID

            stack_name = name[0:40].replace("_", "-")

            template_body = self.software_config.cloud_formation.body

            # Find out the instance type from

            self.logger.info("launching cluster with name = " + stack_name)

            # Parameters that come from user input, Group Config, and the Software Config
            included_parameters = [
                {
                    'ParameterKey': "paramClusterName",
                    'ParameterValue': stack_name
                },
                {
                    'ParameterKey': "paramEnvironment",
                    'ParameterValue': SDLC
                },
                {
                    'ParameterKey': "paramTimeZone",
                    'ParameterValue': TIME_ZONE
                },
                {
                    'ParameterKey': "paramAGS",
                    'ParameterValue': AGS
                },
                {
                    'ParameterKey': "paramInstanceType",
                    'ParameterValue': self.instance_type
                },
                {
                    'ParameterKey': "paramCoreInstanceCount",
                    'ParameterValue': str(self.core_nodes)
                },
                {
                    'ParameterKey': "paramTaskInstanceCount",
                    'ParameterValue': str(self.task_nodes)
                },
                {
                    'ParameterKey': "paramBidPrice",
                    'ParameterValue': str(BID)
                },
                {
                    'ParameterKey': "paramSubnetID",
                    'ParameterValue': SUBNET_ID
                },
                {
                    'ParameterKey': "paramEc2KeyName",
                    'ParameterValue': self.group_config.key_name
                },
                {
                    'ParameterKey': "paramEMRManagedMasterSecurityGroup",
                    'ParameterValue': MASTER_SG
                },
                {
                    'ParameterKey': "paramEMRManagedSlaveSecurityGroup",
                    'ParameterValue': SLAVE_SG
                },
                {
                    'ParameterKey': "paramAdditionalMasterSecurityGroups",
                    'ParameterValue': ADDITIONAL_MASTER_SG
                },
                {
                    'ParameterKey': "paramAdditionalSlaveSecurityGroups",
                    'ParameterValue': ADDITIONAL_SLAVE_SG
                },
                {
                    'ParameterKey': "paramIdleSeconds",
                    'ParameterValue': str(self.auto_terminate_seconds)
                },
                {
                    'ParameterKey':
                    "paramJobFlowRole",
                    'ParameterValue':
                    str(self.group_config.iam_instance_profile_name)
                },
            ]
            if self.on_demand:
                self.logger.info("Setting on-demand Parameters")
                included_parameters.append({
                    'ParameterKey': "paramMarketType",
                    'ParameterValue': 'ON_DEMAND'
                })
                included_parameters.append({
                    'ParameterKey': "paramBidPrice",
                    'ParameterValue': '1.00'
                })

            else:
                self.logger.info("Setting spot Parameters")
                included_parameters.append({
                    'ParameterKey': "paramMarketType",
                    'ParameterValue': 'SPOT'
                })

            try:
                cloudformation = json.loads(
                    self.software_config.cloud_formation.body)
            except json.JSONDecodeError:
                logging.info('JSON Parse failed, attempting YAML parsing')
                cloudformation = yaml.load(
                    self.software_config.cloud_formation.body)

            # Cloudformation parameters must be dynamically set from models.param to allow more flexibility
            parameters = []

            # Params that come from MLiy
            for param in included_parameters:
                if param['ParameterKey'] in cloudformation['Parameters']:
                    parameters.append(param)

            # User provided params
            for param in self.software_config.params.all():
                if param.token in cloudformation["Parameters"]:
                    replacement = param.replacement
                    if param.token == "paramLdapGroup":
                        # This is to allow AD Groups from the Group Config to be used for cluster Auth.
                        # The AD group from Group Config takes precedence.
                        if len(self.group_config.AD_groupname) != 0:
                            self.logger.info(
                                "Using AD group found in Group Config: " +
                                str(self.group_config.AD_groupname))
                            replacement = self.group_config.AD_groupname
                    parameters.append({
                        'ParameterKey': param.token,
                        'ParameterValue': replacement
                    })

            tags = [{
                'Key': "Cost Center",
                'Value': COST_CENTER
            }, {
                'Key': "AGS",
                'Value': AGS
            }, {
                'Key': "SDLC",
                'Value': SDLC
            }, {
                'Key': "Purpose",
                'Value': PURPOSE
            }, {
                'Key': "Owner",
                'Value': OWNER
            }, {
                'Key': "Name",
                'Value': stack_name
            }, {
                'Key': "UserGroup",
                'Value': self.group_config.group.name
            }, {
                'Key': "Creator",
                'Value': NETWORK_ID
            }, {
                'Key': "Launched By",
                'Value': 'MLiy'
            }]

            client = boto3.client('cloudformation', region_name=AWS_REGION)

            response = client.create_stack(TemplateBody=template_body,
                                           StackName=stack_name,
                                           Parameters=parameters,
                                           DisableRollback=False,
                                           Tags=tags)
            stack_id = response["StackId"]

            cluster = Cluster()
            cluster.master_ip = "To Be Determined"
            cluster.userid = NETWORK_ID
            cluster.stack_id = stack_id
            cluster.cluster_id = "To Be Determined"
            cluster.owner = OWNER
            cluster.purpose = PURPOSE
            cluster.node_max = str(self.total_nodes)
            cluster.on_demand = self.on_demand
            cluster.software_config = self.software_config
            cluster.node_count = 0
            cluster.updated_at = datetime.now(timezone('UTC'))
            cluster.task_node = self.task_nodes
            cluster.state = "CREATE_IN_PROGRESS"
            cluster.save()

            self.logger.info("stack created with stack name = " + stack_name)
            response = client.describe_stacks(StackName=stack_name)

            while response['Stacks'][0]['StackStatus'] == 'CREATE_IN_PROGRESS':
                time.sleep(5)
                response = client.describe_stacks(StackName=stack_name)

            if response['Stacks'][0]['StackStatus'] != 'CREATE_COMPLETE':
                cluster.state = "An Error has occurred"
                cluster.save()
                self.logger.debug('State isn\'t CREATE_COMPLETE, exiting')
                raise Exception('Something went terribly wrong: ' +
                                str(response))

            # Use the ClusterId to describe cluster to get Ip
            cluster_id = response['Stacks'][0]['Outputs'][0]['OutputValue']
            self.logger.info("cluster created with cluster id = " + cluster_id)
            emr = boto3.client('emr', region_name=AWS_REGION)

            response = emr.describe_cluster(ClusterId=cluster_id)
            state = response['Cluster']['Status']['State']

            self.logger.debug("state = {}".format(str(state)))

            response = emr.list_instances(ClusterId=cluster_id,
                                          InstanceGroupTypes=['MASTER'])
            ip = response['Instances'][0]['PrivateIpAddress']

            self.logger.debug("ip = {}".format(ip))

            response = emr.list_instance_groups(ClusterId=cluster_id)
            inst_groups = response['InstanceGroups']
            nodes_count = 0
            for grp in inst_groups:
                nodes_count = nodes_count + grp['RunningInstanceCount']

            self.logger.debug("nodes = {}".format(nodes_count))
            cluster.cluster_id = cluster_id
            cluster.owner = OWNER
            cluster.userid = NETWORK_ID
            cluster.stack_id = stack_id
            cluster.purpose = PURPOSE
            cluster.state = 'Initializing'
            cluster.master_ip = ip
            cluster.node_count = nodes_count
            cluster.node_max = str(self.total_nodes)
            cluster.on_demand = self.on_demand
            cluster.software_config = self.software_config
            cluster.task_node = self.task_nodes
            cluster.save()

            # Look at step last-step to complete before notifying users that the cluster is ready to use
            interval = 20
            RETRY_EXCEPTIONS = ('ProvisionedThroughputExceededException',
                                'ThrottlingException')

            last_step = False
            while not last_step:
                time.sleep(interval)
                try:
                    response = emr.list_steps(ClusterId=cluster_id)
                    list_step = response['Steps']
                    all_completed = True
                    for step in list_step:
                        all_completed = all_completed and step['Status'][
                            'State'] == 'COMPLETED'
                    if all_completed:
                        last_step = True
                except ClientError as err:
                    if err.response['Error']['Code'] not in RETRY_EXCEPTIONS:
                        raise
                    interval = min(60, interval + 20)

            response = emr.describe_cluster(ClusterId=cluster_id)
            state = response['Cluster']['Status']['State']
            availability_zone = response['Cluster']['Ec2InstanceAttributes'][
                'Ec2AvailabilityZone']

            # Billing Data
            if self.on_demand:
                base_price = getPrice(self.instance_type)
            else:
                base_price = getSpotPrice(self.instance_type,
                                          availability_zone=availability_zone)
            emr_price = getEmrPrice(self.instance_type)

            if not cluster.current_bill:
                self.logger.info("Bill does not exist for Cluster: " +
                                 cluster_id + ". Creating new BillingData.")

                bill = BillingData()
                bill.ongoing = True
                bill.instance_name = cluster.cluster_id
                bill.charge_name = COST_CENTER
                bill.user = User.objects.get(username=self.user.username)
                bill.start_time = datetime.now(timezone('UTC'))
                bill.price = (base_price + emr_price) * self.total_nodes * (
                    1.0 - AWS_DISCOUNT)
                bill.instance_type = self.market_type.replace('_',' ') + ' | ' \
                      + self.instance_type + ' | ' \
                      + str(self.total_nodes) + ' nodes'

                bill.save()
                cluster.current_bill = bill
            # Set DNS here:
            # Clusters can't be stopped/started, and therefore don't have to worry about changing public IPs
            update_dns(self.logger, cluster.master_ip, cluster, cluster_id)

            cluster.state = state
            cluster.save()

            self.logger.info("Cluster object created")
        except Exception as e:
            # Case: launch_cluster errors out before the cluster object gets initialized
            if cluster is None:
                cluster = Cluster()
                cluster.cluster_id = "Error:" + e.__class__.__name__ + ":" + self.launch_id
                cluster.core_nodes = self.core_nodes
                cluster.master_ip = "An Error has occurred"
                cluster.node_max = self.total_nodes
                cluster.on_demand = self.on_demand
                cluster.owner = self.user.first_name + " " + self.user.last_name
                cluster.purpose = self.purpose
                cluster.software_config = self.software_config
                cluster.task_node = self.task_nodes
                cluster.userid = self.user.username
                cluster.stack_id = "launch id: " + self.launch_id + " error: " + str(
                    e)
            cluster.state = "An Error has occurred"
            cluster.save()
            self.logger.error(e)
            raise e
예제 #5
0
파일: launch.py 프로젝트: FINRAOS/MLiy
    def provide_instance_details(self, instance_id, stack_record):
        logger = self.logger
        logger.debug(instance_id)
        logger.debug(stack_record)

        ec2 = boto3.resource('ec2', region_name=AWS_REGION)
        error = True if "Error" in instance_id else False

        instance_aws = ec2.Instance(instance_id)
        id = instance_aws.id
        private_ip_address = instance_aws.private_ip_address
        state = instance_aws.state['Name']
        logger.debug("private_ip_address set to: " + str(private_ip_address))
        logger.debug("state set to: " + str(state))

        instance_record, created = Instance.objects.get_or_create(
            instance_id=id,
            private_ip=private_ip_address,
            userid=self.user.username,
            owner="{} {}".format(self.user.first_name, self.user.last_name),
            instance_type=self.instance_type,
            software_config=self.sw_config)

        logger.info("Ran get_or_create on Instance. Entry for instance " +
                    instance_id + " did not exist before: " + str(created))

        instance_record.stack_id = stack_record
        instance_record.state = state
        instance_record.updated_at = datetime.now(timezone('UTC'))

        if instance_record.current_bill is None and not error:
            logger.info("Creating new BillingData for the Instance: " +
                        instance_id)
            # Billing Data
            bill = BillingData()

            bill.ongoing = True
            bill.instance_type = self.instance_type
            bill.instance_name = instance_record.instance_id
            bill.charge_name = self.group_config.cost_center
            bill.user = self.user
            bill.price = getPrice(self.instance_type) * (1.0 - AWS_DISCOUNT)
            bill.start_time = datetime.now(timezone('UTC'))
            bill.save()

            instance_record.current_bill = bill
            logger.info("BillingData created for Instance: " + instance_id)

        if created:
            logger.info(
                "Instance is new. Setting initial state and DNS entry.")

            if self.sw_config.has_progress_bar:
                instance_record.progress_status = 'Initializing system'
                instance_record.progress_integer = 0

            instance_record.start_at = datetime.now(timezone('UTC'))
            instance_record.state = 'Initializing'

            # create DNS entry
            try:
                logger.info("Creating DNS entry for instance: " +
                            instance_record.instance_id)

                r = createDnsEntry(instance_record.instance_id,
                                   instance_record.private_ip)

            except Exception as e:
                logger.exception(e)

        instance_record.save()
예제 #6
0
파일: clusters.py 프로젝트: FINRAOS/MLiy
    def update(self, db_clusters):

        emr_client = boto3.client('emr', region_name=AWS_REGION)
        cf_client = boto3.client('cloudformation', region_name=AWS_REGION)
        self.logger.debug("Retrieved the emr client")

        try:
            for cluster in db_clusters:
                self.logger.info("{}: Updating".format(cluster.cluster_id))
                # We only want to make AWS calls if the session status is okay (x seconds interval between AWS calls)
                try:
                    # Cluster SDK doesn't have an option to get cluster information in batches, only individually.
                    if re.match('^j-\w{9,}', cluster.cluster_id):
                        aws_cluster = emr_client.describe_cluster(
                            ClusterId=cluster.cluster_id)
                        if aws_cluster['Cluster']:
                            cluster.state = aws_cluster['Cluster']['Status'][
                                'State']
                            # start_at = aws_cluster['Cluster']['Status']['Timeline']['CreationDateTime']
                            self.logger.info(
                                "Retrieved cluster information from EMR",
                                extra={"cluster_id": cluster.cluster_id})
                            self.logger.debug(
                                "{}: aws_cluster: {}" + str(aws_cluster),
                                extra={"cluster_id": cluster.cluster_id})
                        else:
                            stop_billing(cluster)
                            cluster.state = "An Error has occurred"
                    else:
                        self.logger.info(
                            "Cluster ID did not match regex. Attempting to find Cloudformation stack."
                        )

                        if cluster.stack_id:
                            self.logger.info(
                                "Stack ID found. Attempting update from CF stack: ",
                                extra={"stack_id": cluster.stack_id})
                            aws_stack = cf_client.describe_stacks(
                                StackName=cluster.stack_id)
                            self.logger.info(
                                "Retrieved stack info",
                                extra={"stack_id": cluster.stack_id})
                            self.logger.debug("Stacks: " + str(aws_stack))

                            ok_statuses = [
                                'CREATE_IN_PROGRESS', 'CREATE_COMPLETE',
                                'UPDATE_COMPLETE', 'UPDATE_IN_PROGRESS'
                            ]

                            # creation_time = aws_stack['Stacks'][0]['CreationTime']
                            if aws_stack['Stacks'][0][
                                    'StackStatus'] not in ok_statuses:
                                self.delete_cluster_and_stack(
                                    cf_client, cluster)
                                continue
                            else:
                                cluster.state = aws_stack['Stacks'][0][
                                    'StackStatus']
                                self.logger.info("Cluster stack state: " +
                                                 cluster.state +
                                                 ", proceeding.")
                                if 'Outputs' in aws_stack['Stacks'][0]:
                                    cluster.cluster_id = aws_stack['Stacks'][
                                        0]['Outputs'][0]['OutputValue']
                                    self.logger.info("Retrieved cluster ID: " +
                                                     cluster.cluster_id)
                                else:
                                    self.logger.info(
                                        "Cluster ID not set yet. ")
                                    cluster.cluster_id = "To Be Determined"

                except ClientError as e:
                    self.logger.error(e)
                    self.delete_cluster_and_stack(cf_client, cluster)
                    self.logger.info("Setting state to 'TERMINATED'")
                    cluster.state = "TERMINATED"

                # Cluster instance information (boto3 SDK doesn't have all the information in one place)
                if re.match('^j-\w{9,}', cluster.cluster_id):
                    aws_cluster_instances = emr_client.list_instances(
                        ClusterId=cluster.cluster_id,
                        InstanceGroupTypes=['MASTER'])

                    # Billing
                    # TODO Boto3 EMR api is extremely lacking. Get the information from tags if possible.
                    # Let launch take care of billing. If billing doesn't exist by the time the cluster is WAITING,
                    # it can be assumed that launch.py failed/crashed before billing was created.
                    if cluster.state == 'WAITING' and not cluster.current_bill:
                        self.logger.info(
                            "Setting billing data for cluster",
                            extra={"cluster_id": cluster.cluster_id})
                        aws_cluster = emr_client.describe_cluster(
                            ClusterId=cluster.cluster_id)

                        instance_type = aws_cluster_instances['Instances'][0][
                            'InstanceType']
                        market_type = aws_cluster_instances['Instances'][0][
                            'Market']
                        availability_zone = aws_cluster['Cluster'][
                            'Ec2InstanceAttributes']['Ec2AvailabilityZone']

                        if market_type == "ON_DEMAND":
                            base_price = getPrice(instance_type)
                        else:
                            base_price = getSpotPrice(
                                instance_type,
                                availability_zone=availability_zone)
                        emr_price = getEmrPrice(instance_type)

                        cost_center = self.get_cost_center(
                            aws_cluster['Cluster']['Tags'])

                        if not cost_center:
                            self.logger.info(
                                "No cost center found, deleting cluster.",
                                extra={"cluster_id": cluster.cluster_id})
                            self.delete_cluster_and_stack(cf_client, cluster)
                            continue
                        bill = BillingData()
                        bill.ongoing = True
                        bill.instance_name = cluster.cluster_id
                        bill.charge_name = cost_center
                        bill.user = User.objects.get(username=cluster.userid)
                        bill.start_time = aws_cluster['Cluster']['Status'][
                            'Timeline']['CreationDateTime']
                        bill.price = round(
                            (base_price + emr_price) * cluster.node_max *
                            (1.0 - AWS_DISCOUNT), 2)
                        bill.instance_type = market_type.replace('_', ' ') + ' | ' \
                              + instance_type + ' | ' \
                              + str(cluster.node_max) + ' nodes'
                        bill.save()
                        cluster.current_bill = bill

                    #  master_ip
                    # public_ip = aws_cluster_instances['Instances'][0]['PublicIpAddress']
                    private_ip = aws_cluster_instances['Instances'][0][
                        'PrivateIpAddress']
                    cluster.master_ip = private_ip

                    #  node count
                    if cluster.node_count == 0:
                        response = emr_client.list_instance_groups(
                            ClusterId=cluster.cluster_id)
                        instance_groups = response['InstanceGroups']
                        nodes_count = 0
                        for group in instance_groups:
                            nodes_count = nodes_count + group[
                                'RunningInstanceCount']
                        cluster.node_count = nodes_count

                    #  dns_url
                    if cluster.dns_url is None or cluster.dns_url == "":
                        self.logger.info(
                            "Setting DNS entry for cluster.",
                            extra={"cluster_id": cluster.cluster_id})
                        update_dns(self.logger, cluster.master_ip, cluster,
                                   cluster.cluster_id)
                        self.logger.info(
                            "DNS entry set for cluster.",
                            extra={"cluster_id": cluster.cluster_id})

                cluster.save()

            return self.get(db_clusters)

        except Cluster.DoesNotExist:
            self.logger.exception("Cluster does not exist.")
        except Exception as e:
            self.logger.exception(str(e))
        return []