def _update(self):
# only updates InstanceCount
desired_instances = self.desired_state_definition.get("Instances").get(
"InstanceGroups")
# instance pool
if not desired_instances:
desired_instances = self.desired_state_definition.get(
"Instances").get("InstanceFleets")
# TODO neither pool or group
if not desired_instances:
return True
desired_instance_dict = pcf_util.list_to_dict("Name",
desired_instances)
desired_instance_dict = {
k: pcf_util.param_filter(v, EMRCluster.FILTERED_UPDATE_PARAMS)
for k, v in desired_instance_dict.items()
}
current_instance_dict = {
k:
pcf_util.keep_and_replace_keys(v,
EMRCluster.UPDATE_PARAM_CONVERSIONS)
for k, v in self.current_state_definition.get("Instances").items()
}
diff = pcf_util.diff_dict(current_instance_dict, desired_instance_dict)
for k, v in diff.items():
curr_instance = self.current_state_definition["Instances"].get(
k, {}).get('Id')
#instance group
if curr_instance and self.current_state_definition[
"Instances"].get(k, {}).get('InstanceGroupType'):
self.client.modify_instance_groups(
ClusterId=self._get_cluster_id(),
InstanceGroups=[{
"InstanceGroupId":
curr_instance,
"InstanceCount":
v["InstanceCount"]["updated"]
}])
#instance pool
elif curr_instance:
od_cap = v.get("TargetOnDemandCapacity", {}).get("updated", 0)
spot_cap = v.get("TargetSpotCapacity", {}).get("updated", 0)
self.client.modify_instance_fleet(
ClusterId=self._get_cluster_id(),
InstanceFleet={
"InstanceFleetId": curr_instance,
"TargetOnDemandCapacity": od_cap,
"TargetSpotCapacity": spot_cap
})
def get_status(self):
"""
Calls boto3 describe_clusters using arn.
Returns:
status or {"status":"missing"}
"""
try:
cluster_id = self._get_cluster_id()
if not cluster_id:
return {"status": "missing"}
cluster_status_resp = self.client.describe_cluster(
ClusterId=cluster_id)
cluster_type = cluster_status_resp["Cluster"].get(
"InstanceCollectionType", "INSTANCE_GROUP")
if cluster_type == "INSTANCE_GROUP":
instance_groups = self.client.list_instance_groups(
ClusterId=cluster_id)
cluster_status_resp["Cluster"][
"Instances"] = pcf_util.list_to_dict(
key_name='Name',
dict_list=instance_groups["InstanceGroups"])
# INSTANCE_FLEET
else:
instance_fleets = self.client.list_instance_fleets(
ClusterId=cluster_id)
cluster_status_resp["Cluster"][
"Instances"] = pcf_util.list_to_dict(
key_name='Name',
dict_list=instance_fleets["InstanceFleets"])
except NoResourceException:
return {"status": "missing"}
return cluster_status_resp.get("Cluster")
def is_state_definition_equivalent(self):
"""
Determines if current state is equivalent to the desired state.
Returns:
bool
"""
desired_instances = self.desired_state_definition.get("Instances").get(
"InstanceGroups")
# instance pool
if not desired_instances:
desired_instances = self.desired_state_definition.get(
"Instances").get("InstanceFleets")
# TODO neither pool or group
if not desired_instances:
return True
desired_instance_dict = pcf_util.list_to_dict("Name",
desired_instances)
desired_instance_dict = {
k: pcf_util.param_filter(v, EMRCluster.FILTERED_UPDATE_PARAMS)
for k, v in desired_instance_dict.items()
}
current_instance_dict = {
k:
pcf_util.keep_and_replace_keys(v,
EMRCluster.UPDATE_PARAM_CONVERSIONS)
for k, v in self.current_state_definition.get("Instances").items()
}
# moto bug with instance fleets
if not current_instance_dict:
return True
diff = pcf_util.diff_dict(current_instance_dict, desired_instance_dict)
return diff == {}