def terminate_incomplete_clusters(self, ctx):
if CONF.cleanup_time_for_incomplete_clusters <= 0:
return
LOG.debug('Terminating old clusters in non-final state')
# NOTE(alazarev) Retrieving all clusters once in hour for now.
# Criteria support need to be implemented in sahara db API to
# have SQL filtering.
all_clusters = [
cluster for cluster in conductor.cluster_get_all(ctx)
if (cluster.status not in [
c_u.CLUSTER_STATUS_ACTIVE, c_u.CLUSTER_STATUS_ERROR,
c_u.CLUSTER_STATUS_DELETING
])
]
clusters_to_manage = self.hr.get_subset(all_clusters)
for cluster in clusters_to_manage:
spacing = get_time_since_last_update(cluster)
if spacing < CONF.cleanup_time_for_incomplete_clusters * 3600:
continue
terminate_cluster(ctx, cluster, description='incomplete')
# Add event log info cleanup
context.ctx().current_instance_info = context.InstanceInfo()
def test_instance_context_manager(self):
fake_instances = [FakeInstance() for _ in range(50)]
# check that InstanceContextManager works fine sequentially
for instance in fake_instances:
info = context.InstanceInfo(None, instance.id, instance.name, None)
with context.InstanceInfoManager(info):
self._make_checks(info, sleep=False)
# check that InstanceContextManager works fine in parallel
with context.ThreadGroup() as tg:
for instance in fake_instances:
info = context.InstanceInfo(None, instance.id, instance.name,
None)
with context.InstanceInfoManager(info):
tg.spawn("make_checks", self._make_checks, info)
def _start_instance(self, cluster, node_group, idx, aa_group,
old_aa_groups=None):
instance_name = g.generate_instance_name(
cluster.name, node_group.name, idx)
current_instance_info = context.InstanceInfo(
cluster.id, None, instance_name, node_group.id)
with context.InstanceInfoManager(current_instance_info):
instance_id = self._run_instance(
cluster, node_group, idx,
aa_group=aa_group, old_aa_groups=old_aa_groups)
return instance_id
def terminate_incomplete_clusters(self, ctx):
if CONF.cleanup_time_for_incomplete_clusters <= 0:
return
LOG.debug('Terminating old clusters in non-final state')
# NOTE(alazarev) Retrieving all clusters once in hour for now.
# Criteria support need to be implemented in sahara db API to
# have SQL filtering.
for cluster in conductor.cluster_get_all(ctx):
if cluster.status in ['Active', 'Error', 'Deleting']:
continue
spacing = get_time_since_last_update(cluster)
if spacing < CONF.cleanup_time_for_incomplete_clusters * 3600:
continue
terminate_cluster(ctx, cluster, description='incomplete')
# Add event log info cleanup
context.ctx().current_instance_info = context.InstanceInfo()
def terminate_unneeded_transient_clusters(self, ctx):
LOG.debug('Terminating unneeded transient clusters')
for cluster in conductor.cluster_get_all(ctx, status='Active'):
if not cluster.is_transient:
continue
jc = conductor.job_execution_count(ctx,
end_time=None,
cluster_id=cluster.id)
if jc > 0:
continue
spacing = get_time_since_last_update(cluster)
if spacing < CONF.min_transient_cluster_active_time:
continue
terminate_cluster(ctx, cluster, description='transient')
# Add event log info cleanup
context.ctx().current_instance_info = context.InstanceInfo()
def terminate_unneeded_transient_clusters(self, ctx):
LOG.debug('Terminating unneeded transient clusters')
all_clusters = conductor.cluster_get_all(
ctx, status=c_u.CLUSTER_STATUS_ACTIVE, is_transient=True)
clusters_to_manage = self.hr.get_subset(all_clusters)
for cluster in clusters_to_manage:
jc = conductor.job_execution_count(ctx,
end_time=None,
cluster_id=cluster.id)
if jc > 0:
continue
spacing = get_time_since_last_update(cluster)
if spacing < CONF.min_transient_cluster_active_time:
continue
terminate_cluster(ctx, cluster, description='transient')
# Add event log info cleanup
context.ctx().current_instance_info = context.InstanceInfo()
def _get_info_from_cluster(arg):
if isinstance(arg, resource.ClusterResource):
return context.InstanceInfo(arg.id)
return None
