def get_data(self, context):
log = logging.getLogger(__name__)
starting_color = 0xfa5858
step_change = 0x2010
# get instances with listed user
current_owner = ''
current_cost = 0.0
idx = 0
data = []
pricecache = {}
caller_groups = remove_managergroup(set(self.request.user.groups.all()))
for inst in Instance.objects.all().exclude(userid='').filter(state='running').order_by('userid'):
if inst.userid.upper() != self.request.user.username.upper():
# check if caller and instance owner share a group
try:
if caller_groups.isdisjoint(set(User.objects.get(username=inst.userid.upper()).groups.all())):
continue
except User.DoesNotExist:
continue
if current_owner != inst.owner:
data.append({
'value': round(current_cost, 4),
'color': '#{:x}'.format(starting_color + (idx * step_change)),
'highlight': "#FFC870",
'label': current_owner
})
idx += 1
current_owner = inst.owner
current_cost = 0.0
if inst.instance_type not in pricecache:
iprice = getPrice(inst.instance_type) * (1.0 - AWS_DISCOUNT)
if iprice > 0.0:
current_cost += iprice
pricecache[inst.instance_type] = iprice
else:
log.error("Instance of type {} not found in price db.".format(inst.instance_type))
else:
current_cost += pricecache[inst.instance_type]
# last user
data.append({
'value': round(current_cost, 4),
'color': '#{:x}'.format(starting_color + (idx * step_change)),
'highlight': "#FFC870",
'label': current_owner
})
return data
def get(self, db_instances, message='', force_set_display_state=''):
try:
response = []
for instance in db_instances:
# On aws failure, make it clear that the state is unknown
if force_set_display_state:
self.logger.info("Instance state forced set: " + force_set_display_state)
state = force_set_display_state
else:
state = instance.state
inst = {
'id': instance.instance_id,
'type': instance.instance_type,
'private_ip': instance.private_ip,
'state': {
'Name': state,
'progress': instance.progress_status,
'integer': instance.progress_integer,
},
'time': get_uptime(instance.start_at),
'tags': list(instance.tag_set.values('Name', 'Value')),
'price': '{:.2}'.format(getPrice(instance.instance_type) * (1.0 - AWS_DISCOUNT)),
'sc': instance.software_config.name,
'dns_url': instance.dns_url,
'owner': instance.owner,
'message': message
}
try:
extra_values = list(DisplayValue.objects.filter(instance=instance))
for display_object in extra_values:
inst[display_object.table_header] = display_object.table_value
except DisplayValue.DoesNotExist:
self.logger.info("No extra DisplayValues", extra={"instance_id": instance.instance_id})
response.append(inst)
self.logger.debug(str(response))
return response
except Exception as e:
self.logger.exception(e)
return []
def get_data(self, context):
log = self.logger
action = self.kwargs['action']
instanceid = self.kwargs['instanceid']
# make sure user owns this instance
inst = get_object_or_404(Instance, instance_id=instanceid)
is_authorized = has_permission(self.request.user, inst.userid)
if not is_authorized:
return {'action': 'invalid', 'status': 'unauthorized'}
conn = boto3.resource('ec2', region_name=AWS_REGION)
botoinstance = conn.Instance(id=instanceid)
# ====================================
if action == 'start':
log.info("Entering " + str(action))
if botoinstance.state['Name'] != 'stopped':
if inst.state == "stopped":
inst.state = botoinstance.state['Name']
inst.start_at = datetime.now(timezone(TIME_ZONE))
inst.save()
return {
'action': 'invalid',
'status': 'failed',
'exception': 'Instance started already.'
}
try:
botoinstance.start()
bill = BillingData()
bill.ongoing = True
bill.instance_type = inst.instance_type
bill.instance_name = inst.instance_id
if inst.current_bill is not None:
bill.charge_name = inst.current_bill.charge_name
else:
bill.charge_name = DEFAULT_CHARGE_CODE
bill.user = User.objects.get(username=inst.userid)
bill.price = getPrice(
inst.instance_type) * (1.0 - AWS_DISCOUNT)
bill.start_time = datetime.now(timezone('UTC'))
bill.save()
inst.current_bill = bill
inst.state = 'starting'
inst.start_at = datetime.now(timezone(TIME_ZONE))
inst.save()
log.info("Exiting " + str(action))
except Exception as e:
log.exception(e)
return {
'action': 'invalid',
'status': 'failed',
'exception': str(e)
}
# ====================================
elif action == 'stop':
log.info("Entering " + str(action))
if botoinstance.state['Name'] != 'running':
if inst.state == "running":
inst.state = botoinstance.state['Name']
inst.start_at = datetime.now(timezone(TIME_ZONE))
inst.save()
return {
'action': 'invalid',
'status': 'failed',
'exception': "Instance is not running."
}
try:
botoinstance.stop()
bill = inst.current_bill
if bill is not None:
bill.ongoing = False
bill.end_time = datetime.now(timezone('UTC'))
bill.save()
inst.state = 'stopping'
inst.stop_at = datetime.now(timezone(TIME_ZONE))
inst.save()
log.info("Exiting " + str(action))
except Exception as e:
log.exception(e)
return {
'action': 'invalid',
'status': 'failed',
'exception': str(e)
}
# ====================================
elif action == 'restart':
log.info("Entering " + str(action))
if botoinstance.state['Name'] != 'running':
return {
'action': 'invalid',
'status': 'failed',
'exception': 'Instance is not running.'
}
try:
botoinstance.reboot()
inst.stop_at = datetime.now(timezone(TIME_ZONE))
inst.start_at = datetime.now(timezone(TIME_ZONE))
inst.state = 'restarting'
inst.save()
log.info("Exiting " + str(action))
except Exception as e:
log.exception(e)
return {
'action': 'invalid',
'status': 'failed',
'exception': str(e)
}
# ====================================
elif action == 'terminate':
log.info("Entering " + str(action))
if botoinstance.state['Name'].startswith('termin'):
return {
'action': 'invalid',
'status': 'failed',
'exception': 'Instance already terminated.'
}
try:
# kill the Cloudformation stack
stack = inst.stack_id
if stack is not None:
client = boto3.client("cloudformation",
region_name=AWS_REGION)
ec2_client = boto3.client('ec2', region_name=AWS_REGION)
client.delete_stack(StackName=stack.stack_id)
stack.delete()
state = ''
while state != 'shutting-down':
state = ec2_client.describe_instances(
InstanceIds=[inst.instance_id]
)['Reservations'][0]['Instances'][0]['State']['Name']
time.sleep(3)
else:
return {
'action':
'invalid',
'status':
'failed',
'exception':
'Instance has no stack and will not be terminated.'
}
inst.stack_id = None
bill = inst.current_bill
if bill is not None:
bill.ongoing = False
bill.end_time = datetime.now(timezone('UTC'))
bill.save()
deleteDnsEntry(inst.instance_id, inst.private_ip)
inst.state = 'terminating'
inst.progress_integer = 100
inst.progress_status = 'done'
inst.archived = True
inst.stop_at = datetime.now(timezone(TIME_ZONE))
inst.save()
log.info("Exiting " + str(action))
except Exception as e:
log.exception(e)
return {
'action': 'invalid',
'status': 'failed',
'exception': str(e)
}
# ====================================
elif action == 'archive':
log.info("Entering " + str(action))
try:
# Check for CF
stack = inst.stack_id
if stack:
client = boto3.client("cloudformation",
region_name=AWS_REGION)
client.delete_stack(StackName=stack.stack_id)
stack.delete()
inst.stack_id = None
inst.archived = True
inst.stop_at = datetime.now(timezone(TIME_ZONE))
deleteDnsEntry(inst.instance_id, inst.private_ip)
bill = inst.current_bill
if bill:
bill.ongoing = False
bill.end_time = datetime.now(timezone('UTC'))
bill.save()
inst.save()
log.info("Exiting " + str(action))
except Exception as e:
log.exception(e)
return {
'action': 'invalid',
'status': 'failed',
'exception': str(e)
}
# ====================================
elif action == 'fakeok':
# here for UI testing
pass
# ====================================
else:
log.error("Invalid verb passed to changeInstanceState")
return {
'action': 'invalid',
'status': 'failed',
'exception': "Invalid verb passed to changeInstanceState"
}
# add a short delay in return to try to address non-changing ui
time.sleep(2)
return {'action': action, 'status': 'ok'}
def launch_cluster(self, form):
cluster = None
try:
NETWORK_ID = self.user.username
UNIQUE_ID = self.launch_id
AGS = self.group_config.ags
SDLC = self.group_config.sdlc
COST_CENTER = self.group_config.cost_center
PURPOSE = self.purpose
BID = self.bid
if self.software_config.master_security_group is not None:
MASTER_SG = self.software_config.master_security_group.sgid
SLAVE_SG = self.software_config.slave_security_group.sgid
logging.debug(self.software_config.name)
else:
MASTER_SG = ""
SLAVE_SG = ""
logging.debug(self.software_config.name)
ADDITIONAL_MASTER_SG = ""
ADDITIONAL_SLAVE_SG = ""
if self.software_config.additional_master_security_groups is not None:
for security_group in self.software_config.additional_master_security_groups.all(
):
ADDITIONAL_MASTER_SG = ADDITIONAL_MASTER_SG + security_group.sgid + ','
ADDITIONAL_MASTER_SG = ADDITIONAL_MASTER_SG.rstrip(',')
if self.software_config.additional_slave_security_groups is not None:
for security_group in self.software_config.additional_slave_security_groups.all(
):
ADDITIONAL_SLAVE_SG = ADDITIONAL_SLAVE_SG + security_group.sgid + ','
ADDITIONAL_SLAVE_SG = ADDITIONAL_SLAVE_SG.rstrip(',')
subnets = []
if SDLC == "PRODY":
subnets = getSubnets(findVPCID('analytics'), 'dynamic')
else:
subnets = getSubnets(findVPCID('general'), 'dynamic')
SUBNET_NUM = random.randint(0, len(subnets) - 1)
SUBNET_ID = subnets[SUBNET_NUM]['SubnetId']
OWNER = "{} {}".format(self.user.first_name, self.user.last_name)
group_name = self.group_config.override_instance_name
software_name = self.software_config.instance_name
if group_name:
name = group_name + "-" + self.market_type + "-" + UNIQUE_ID + "-" + NETWORK_ID
else:
name = software_name + "-" + self.market_type + "-" + UNIQUE_ID + "-" + NETWORK_ID
stack_name = name[0:40].replace("_", "-")
template_body = self.software_config.cloud_formation.body
# Find out the instance type from
self.logger.info("launching cluster with name = " + stack_name)
# Parameters that come from user input, Group Config, and the Software Config
included_parameters = [
{
'ParameterKey': "paramClusterName",
'ParameterValue': stack_name
},
{
'ParameterKey': "paramEnvironment",
'ParameterValue': SDLC
},
{
'ParameterKey': "paramTimeZone",
'ParameterValue': TIME_ZONE
},
{
'ParameterKey': "paramAGS",
'ParameterValue': AGS
},
{
'ParameterKey': "paramInstanceType",
'ParameterValue': self.instance_type
},
{
'ParameterKey': "paramCoreInstanceCount",
'ParameterValue': str(self.core_nodes)
},
{
'ParameterKey': "paramTaskInstanceCount",
'ParameterValue': str(self.task_nodes)
},
{
'ParameterKey': "paramBidPrice",
'ParameterValue': str(BID)
},
{
'ParameterKey': "paramSubnetID",
'ParameterValue': SUBNET_ID
},
{
'ParameterKey': "paramEc2KeyName",
'ParameterValue': self.group_config.key_name
},
{
'ParameterKey': "paramEMRManagedMasterSecurityGroup",
'ParameterValue': MASTER_SG
},
{
'ParameterKey': "paramEMRManagedSlaveSecurityGroup",
'ParameterValue': SLAVE_SG
},
{
'ParameterKey': "paramAdditionalMasterSecurityGroups",
'ParameterValue': ADDITIONAL_MASTER_SG
},
{
'ParameterKey': "paramAdditionalSlaveSecurityGroups",
'ParameterValue': ADDITIONAL_SLAVE_SG
},
{
'ParameterKey': "paramIdleSeconds",
'ParameterValue': str(self.auto_terminate_seconds)
},
{
'ParameterKey':
"paramJobFlowRole",
'ParameterValue':
str(self.group_config.iam_instance_profile_name)
},
]
if self.on_demand:
self.logger.info("Setting on-demand Parameters")
included_parameters.append({
'ParameterKey': "paramMarketType",
'ParameterValue': 'ON_DEMAND'
})
included_parameters.append({
'ParameterKey': "paramBidPrice",
'ParameterValue': '1.00'
})
else:
self.logger.info("Setting spot Parameters")
included_parameters.append({
'ParameterKey': "paramMarketType",
'ParameterValue': 'SPOT'
})
try:
cloudformation = json.loads(
self.software_config.cloud_formation.body)
except json.JSONDecodeError:
logging.info('JSON Parse failed, attempting YAML parsing')
cloudformation = yaml.load(
self.software_config.cloud_formation.body)
# Cloudformation parameters must be dynamically set from models.param to allow more flexibility
parameters = []
# Params that come from MLiy
for param in included_parameters:
if param['ParameterKey'] in cloudformation['Parameters']:
parameters.append(param)
# User provided params
for param in self.software_config.params.all():
if param.token in cloudformation["Parameters"]:
replacement = param.replacement
if param.token == "paramLdapGroup":
# This is to allow AD Groups from the Group Config to be used for cluster Auth.
# The AD group from Group Config takes precedence.
if len(self.group_config.AD_groupname) != 0:
self.logger.info(
"Using AD group found in Group Config: " +
str(self.group_config.AD_groupname))
replacement = self.group_config.AD_groupname
parameters.append({
'ParameterKey': param.token,
'ParameterValue': replacement
})
tags = [{
'Key': "Cost Center",
'Value': COST_CENTER
}, {
'Key': "AGS",
'Value': AGS
}, {
'Key': "SDLC",
'Value': SDLC
}, {
'Key': "Purpose",
'Value': PURPOSE
}, {
'Key': "Owner",
'Value': OWNER
}, {
'Key': "Name",
'Value': stack_name
}, {
'Key': "UserGroup",
'Value': self.group_config.group.name
}, {
'Key': "Creator",
'Value': NETWORK_ID
}, {
'Key': "Launched By",
'Value': 'MLiy'
}]
client = boto3.client('cloudformation', region_name=AWS_REGION)
response = client.create_stack(TemplateBody=template_body,
StackName=stack_name,
Parameters=parameters,
DisableRollback=False,
Tags=tags)
stack_id = response["StackId"]
cluster = Cluster()
cluster.master_ip = "To Be Determined"
cluster.userid = NETWORK_ID
cluster.stack_id = stack_id
cluster.cluster_id = "To Be Determined"
cluster.owner = OWNER
cluster.purpose = PURPOSE
cluster.node_max = str(self.total_nodes)
cluster.on_demand = self.on_demand
cluster.software_config = self.software_config
cluster.node_count = 0
cluster.updated_at = datetime.now(timezone('UTC'))
cluster.task_node = self.task_nodes
cluster.state = "CREATE_IN_PROGRESS"
cluster.save()
self.logger.info("stack created with stack name = " + stack_name)
response = client.describe_stacks(StackName=stack_name)
while response['Stacks'][0]['StackStatus'] == 'CREATE_IN_PROGRESS':
time.sleep(5)
response = client.describe_stacks(StackName=stack_name)
if response['Stacks'][0]['StackStatus'] != 'CREATE_COMPLETE':
cluster.state = "An Error has occurred"
cluster.save()
self.logger.debug('State isn\'t CREATE_COMPLETE, exiting')
raise Exception('Something went terribly wrong: ' +
str(response))
# Use the ClusterId to describe cluster to get Ip
cluster_id = response['Stacks'][0]['Outputs'][0]['OutputValue']
self.logger.info("cluster created with cluster id = " + cluster_id)
emr = boto3.client('emr', region_name=AWS_REGION)
response = emr.describe_cluster(ClusterId=cluster_id)
state = response['Cluster']['Status']['State']
self.logger.debug("state = {}".format(str(state)))
response = emr.list_instances(ClusterId=cluster_id,
InstanceGroupTypes=['MASTER'])
ip = response['Instances'][0]['PrivateIpAddress']
self.logger.debug("ip = {}".format(ip))
response = emr.list_instance_groups(ClusterId=cluster_id)
inst_groups = response['InstanceGroups']
nodes_count = 0
for grp in inst_groups:
nodes_count = nodes_count + grp['RunningInstanceCount']
self.logger.debug("nodes = {}".format(nodes_count))
cluster.cluster_id = cluster_id
cluster.owner = OWNER
cluster.userid = NETWORK_ID
cluster.stack_id = stack_id
cluster.purpose = PURPOSE
cluster.state = 'Initializing'
cluster.master_ip = ip
cluster.node_count = nodes_count
cluster.node_max = str(self.total_nodes)
cluster.on_demand = self.on_demand
cluster.software_config = self.software_config
cluster.task_node = self.task_nodes
cluster.save()
# Look at step last-step to complete before notifying users that the cluster is ready to use
interval = 20
RETRY_EXCEPTIONS = ('ProvisionedThroughputExceededException',
'ThrottlingException')
last_step = False
while not last_step:
time.sleep(interval)
try:
response = emr.list_steps(ClusterId=cluster_id)
list_step = response['Steps']
all_completed = True
for step in list_step:
all_completed = all_completed and step['Status'][
'State'] == 'COMPLETED'
if all_completed:
last_step = True
except ClientError as err:
if err.response['Error']['Code'] not in RETRY_EXCEPTIONS:
raise
interval = min(60, interval + 20)
response = emr.describe_cluster(ClusterId=cluster_id)
state = response['Cluster']['Status']['State']
availability_zone = response['Cluster']['Ec2InstanceAttributes'][
'Ec2AvailabilityZone']
# Billing Data
if self.on_demand:
base_price = getPrice(self.instance_type)
else:
base_price = getSpotPrice(self.instance_type,
availability_zone=availability_zone)
emr_price = getEmrPrice(self.instance_type)
if not cluster.current_bill:
self.logger.info("Bill does not exist for Cluster: " +
cluster_id + ". Creating new BillingData.")
bill = BillingData()
bill.ongoing = True
bill.instance_name = cluster.cluster_id
bill.charge_name = COST_CENTER
bill.user = User.objects.get(username=self.user.username)
bill.start_time = datetime.now(timezone('UTC'))
bill.price = (base_price + emr_price) * self.total_nodes * (
1.0 - AWS_DISCOUNT)
bill.instance_type = self.market_type.replace('_',' ') + ' | ' \
+ self.instance_type + ' | ' \
+ str(self.total_nodes) + ' nodes'
bill.save()
cluster.current_bill = bill
# Set DNS here:
# Clusters can't be stopped/started, and therefore don't have to worry about changing public IPs
update_dns(self.logger, cluster.master_ip, cluster, cluster_id)
cluster.state = state
cluster.save()
self.logger.info("Cluster object created")
except Exception as e:
# Case: launch_cluster errors out before the cluster object gets initialized
if cluster is None:
cluster = Cluster()
cluster.cluster_id = "Error:" + e.__class__.__name__ + ":" + self.launch_id
cluster.core_nodes = self.core_nodes
cluster.master_ip = "An Error has occurred"
cluster.node_max = self.total_nodes
cluster.on_demand = self.on_demand
cluster.owner = self.user.first_name + " " + self.user.last_name
cluster.purpose = self.purpose
cluster.software_config = self.software_config
cluster.task_node = self.task_nodes
cluster.userid = self.user.username
cluster.stack_id = "launch id: " + self.launch_id + " error: " + str(
e)
cluster.state = "An Error has occurred"
cluster.save()
self.logger.error(e)
raise e
def provide_instance_details(self, instance_id, stack_record):
logger = self.logger
logger.debug(instance_id)
logger.debug(stack_record)
ec2 = boto3.resource('ec2', region_name=AWS_REGION)
error = True if "Error" in instance_id else False
instance_aws = ec2.Instance(instance_id)
id = instance_aws.id
private_ip_address = instance_aws.private_ip_address
state = instance_aws.state['Name']
logger.debug("private_ip_address set to: " + str(private_ip_address))
logger.debug("state set to: " + str(state))
instance_record, created = Instance.objects.get_or_create(
instance_id=id,
private_ip=private_ip_address,
userid=self.user.username,
owner="{} {}".format(self.user.first_name, self.user.last_name),
instance_type=self.instance_type,
software_config=self.sw_config)
logger.info("Ran get_or_create on Instance. Entry for instance " +
instance_id + " did not exist before: " + str(created))
instance_record.stack_id = stack_record
instance_record.state = state
instance_record.updated_at = datetime.now(timezone('UTC'))
if instance_record.current_bill is None and not error:
logger.info("Creating new BillingData for the Instance: " +
instance_id)
# Billing Data
bill = BillingData()
bill.ongoing = True
bill.instance_type = self.instance_type
bill.instance_name = instance_record.instance_id
bill.charge_name = self.group_config.cost_center
bill.user = self.user
bill.price = getPrice(self.instance_type) * (1.0 - AWS_DISCOUNT)
bill.start_time = datetime.now(timezone('UTC'))
bill.save()
instance_record.current_bill = bill
logger.info("BillingData created for Instance: " + instance_id)
if created:
logger.info(
"Instance is new. Setting initial state and DNS entry.")
if self.sw_config.has_progress_bar:
instance_record.progress_status = 'Initializing system'
instance_record.progress_integer = 0
instance_record.start_at = datetime.now(timezone('UTC'))
instance_record.state = 'Initializing'
# create DNS entry
try:
logger.info("Creating DNS entry for instance: " +
instance_record.instance_id)
r = createDnsEntry(instance_record.instance_id,
instance_record.private_ip)
except Exception as e:
logger.exception(e)
instance_record.save()
def update(self, db_clusters):
emr_client = boto3.client('emr', region_name=AWS_REGION)
cf_client = boto3.client('cloudformation', region_name=AWS_REGION)
self.logger.debug("Retrieved the emr client")
try:
for cluster in db_clusters:
self.logger.info("{}: Updating".format(cluster.cluster_id))
# We only want to make AWS calls if the session status is okay (x seconds interval between AWS calls)
try:
# Cluster SDK doesn't have an option to get cluster information in batches, only individually.
if re.match('^j-\w{9,}', cluster.cluster_id):
aws_cluster = emr_client.describe_cluster(
ClusterId=cluster.cluster_id)
if aws_cluster['Cluster']:
cluster.state = aws_cluster['Cluster']['Status'][
'State']
# start_at = aws_cluster['Cluster']['Status']['Timeline']['CreationDateTime']
self.logger.info(
"Retrieved cluster information from EMR",
extra={"cluster_id": cluster.cluster_id})
self.logger.debug(
"{}: aws_cluster: {}" + str(aws_cluster),
extra={"cluster_id": cluster.cluster_id})
else:
stop_billing(cluster)
cluster.state = "An Error has occurred"
else:
self.logger.info(
"Cluster ID did not match regex. Attempting to find Cloudformation stack."
)
if cluster.stack_id:
self.logger.info(
"Stack ID found. Attempting update from CF stack: ",
extra={"stack_id": cluster.stack_id})
aws_stack = cf_client.describe_stacks(
StackName=cluster.stack_id)
self.logger.info(
"Retrieved stack info",
extra={"stack_id": cluster.stack_id})
self.logger.debug("Stacks: " + str(aws_stack))
ok_statuses = [
'CREATE_IN_PROGRESS', 'CREATE_COMPLETE',
'UPDATE_COMPLETE', 'UPDATE_IN_PROGRESS'
]
# creation_time = aws_stack['Stacks'][0]['CreationTime']
if aws_stack['Stacks'][0][
'StackStatus'] not in ok_statuses:
self.delete_cluster_and_stack(
cf_client, cluster)
continue
else:
cluster.state = aws_stack['Stacks'][0][
'StackStatus']
self.logger.info("Cluster stack state: " +
cluster.state +
", proceeding.")
if 'Outputs' in aws_stack['Stacks'][0]:
cluster.cluster_id = aws_stack['Stacks'][
0]['Outputs'][0]['OutputValue']
self.logger.info("Retrieved cluster ID: " +
cluster.cluster_id)
else:
self.logger.info(
"Cluster ID not set yet. ")
cluster.cluster_id = "To Be Determined"
except ClientError as e:
self.logger.error(e)
self.delete_cluster_and_stack(cf_client, cluster)
self.logger.info("Setting state to 'TERMINATED'")
cluster.state = "TERMINATED"
# Cluster instance information (boto3 SDK doesn't have all the information in one place)
if re.match('^j-\w{9,}', cluster.cluster_id):
aws_cluster_instances = emr_client.list_instances(
ClusterId=cluster.cluster_id,
InstanceGroupTypes=['MASTER'])
# Billing
# TODO Boto3 EMR api is extremely lacking. Get the information from tags if possible.
# Let launch take care of billing. If billing doesn't exist by the time the cluster is WAITING,
# it can be assumed that launch.py failed/crashed before billing was created.
if cluster.state == 'WAITING' and not cluster.current_bill:
self.logger.info(
"Setting billing data for cluster",
extra={"cluster_id": cluster.cluster_id})
aws_cluster = emr_client.describe_cluster(
ClusterId=cluster.cluster_id)
instance_type = aws_cluster_instances['Instances'][0][
'InstanceType']
market_type = aws_cluster_instances['Instances'][0][
'Market']
availability_zone = aws_cluster['Cluster'][
'Ec2InstanceAttributes']['Ec2AvailabilityZone']
if market_type == "ON_DEMAND":
base_price = getPrice(instance_type)
else:
base_price = getSpotPrice(
instance_type,
availability_zone=availability_zone)
emr_price = getEmrPrice(instance_type)
cost_center = self.get_cost_center(
aws_cluster['Cluster']['Tags'])
if not cost_center:
self.logger.info(
"No cost center found, deleting cluster.",
extra={"cluster_id": cluster.cluster_id})
self.delete_cluster_and_stack(cf_client, cluster)
continue
bill = BillingData()
bill.ongoing = True
bill.instance_name = cluster.cluster_id
bill.charge_name = cost_center
bill.user = User.objects.get(username=cluster.userid)
bill.start_time = aws_cluster['Cluster']['Status'][
'Timeline']['CreationDateTime']
bill.price = round(
(base_price + emr_price) * cluster.node_max *
(1.0 - AWS_DISCOUNT), 2)
bill.instance_type = market_type.replace('_', ' ') + ' | ' \
+ instance_type + ' | ' \
+ str(cluster.node_max) + ' nodes'
bill.save()
cluster.current_bill = bill
# master_ip
# public_ip = aws_cluster_instances['Instances'][0]['PublicIpAddress']
private_ip = aws_cluster_instances['Instances'][0][
'PrivateIpAddress']
cluster.master_ip = private_ip
# node count
if cluster.node_count == 0:
response = emr_client.list_instance_groups(
ClusterId=cluster.cluster_id)
instance_groups = response['InstanceGroups']
nodes_count = 0
for group in instance_groups:
nodes_count = nodes_count + group[
'RunningInstanceCount']
cluster.node_count = nodes_count
# dns_url
if cluster.dns_url is None or cluster.dns_url == "":
self.logger.info(
"Setting DNS entry for cluster.",
extra={"cluster_id": cluster.cluster_id})
update_dns(self.logger, cluster.master_ip, cluster,
cluster.cluster_id)
self.logger.info(
"DNS entry set for cluster.",
extra={"cluster_id": cluster.cluster_id})
cluster.save()
return self.get(db_clusters)
except Cluster.DoesNotExist:
self.logger.exception("Cluster does not exist.")
except Exception as e:
self.logger.exception(str(e))
return []