def create_cluster_model(creator=None, cpus=1, memory=1):
if creator is None:
creator = create_user_model()
cluster = Cluster(creator=creator, cpus=cpus, memory=memory)
cluster.save()
return cluster
def update_system_nodes():
k8s_manager = K8SManager(in_cluster=True)
nodes = k8s_manager.list_nodes()
cluster = Cluster.load()
nodes_to_update = {}
nodes_to_create = {node.metadata.name: node for node in nodes}
deprecated_nodes = []
for node in cluster.nodes.all():
if node.name in nodes_to_create:
nodes_to_update[node.name] = (node, nodes_to_create.pop(node.name))
else:
deprecated_nodes.append(node)
for node in deprecated_nodes:
node.is_current = False
node.save()
for node in nodes_to_create.values():
node_dict = ClusterNode.from_node_item(node)
node_dict['cluster'] = cluster
ClusterNode.objects.create(**node_dict)
for current_node, new_node in nodes_to_update.values():
node_dict = ClusterNode.from_node_item(new_node)
for k, v in node_dict.items():
setattr(current_node, k, v)
current_node.save()
def setUp(self):
super().setUp()
self.cluster = Cluster.load()
self.obj1 = ClusterNodeFactory(cluster=self.cluster)
self.obj2 = ClusterNodeFactory(cluster=self.cluster)
self.gpu_obj1 = GPUFactory(cluster_node=self.obj1)
self.gpu_obj2 = GPUFactory(cluster_node=self.obj2)
def get_node():
cluster = Cluster.load()
node = ClusterNode.objects.filter(cluster=cluster,
name=settings.K8S_NODE_NAME)
if node.count():
return node.first()
return None
def update_system_info():
k8s_manager = K8SManager(in_cluster=True)
version_api = k8s_manager.get_version()
cluster = Cluster.load()
if cluster.version_api != version_api:
cluster.version_api = version_api
cluster.save()
def setUp(self):
self.cluster = Cluster.load()
self.admin = UserFactory(is_staff=True, is_superuser=True)
self.user = UserFactory()
self.publisher = PublishTrackerService()
self.publisher.setup()
super(PublishTrackerTest, self).setUp()
def setUp(self):
super().setUp()
self.cluster_node = ClusterNodeFactory(cluster=Cluster.load())
self.url = '/{}/nodes/{}/gpus'.format(API_V1, self.cluster_node.sequence)
self.objects = [self.factory_class(cluster_node=self.cluster_node)
for _ in range(self.num_objects)]
self.queryset = self.model_class.objects.all()
def setUp(self):
super().setUp()
self.object = Cluster.load()
self.url = '/{}/cluster/'.format(API_V1)
# Create related fields
for _ in range(2):
ClusterNodeFactory(cluster=self.object)
def setUp(self):
super().setUp()
self.cluster = Cluster.load()
self.url = '/{}/cluster/nodes/'.format(API_V1)
self.objects = [
self.factory_class(cluster=self.cluster)
for _ in range(self.num_objects)
]
self.queryset = self.model_class.objects.filter(cluster=self.cluster)
def update_system_info():
k8s_manager = K8SManager(in_cluster=True)
version_api = k8s_manager.get_version()
cluster = Cluster.load()
if cluster.version_api != version_api:
cluster.version_api = version_api
cluster.save()
auditor.record(event_type=CLUSTER_UPDATED,
instance=cluster,
is_upgrade=settings.CHART_IS_UPGRADE)
def setUp(self):
super().setUp()
self.cluster = Cluster.load()
self.object = self.factory_class(cluster=self.cluster)
self.url = '/{}/nodes/{}/'.format(API_V1, self.object.sequence)
self.queryset = self.model_class.objects.all()
# Create related fields
for _ in range(2):
GPUFactory(cluster_node=self.object)
def test_update_works(self):
cluster = Cluster.load()
updated_cluster = Cluster.load()
cluster.version_api = {
'build_date': datetime.datetime.now().isoformat(),
'compiler': 'gc',
'git_commit': '17d7182a7ccbb167074be7a87f0a68bd00d58d91',
'git_tree_state': 'clean',
'git_version': 'v1.7.5',
'go_version': 'go1.8.3',
'major': '1',
'minor': '7',
'platform': 'linux/amd64'
}
updated_cluster.save()
loaded_version = Cluster.load()
assert loaded_version.version_api != cluster.version_api
assert loaded_version.version_api == updated_cluster.version_api
def setUp(self):
self.cluster = Cluster.load()
self.cluster_node = ClusterNodeFactory(cluster=self.cluster)
self.node_gpu = GPUFactory(cluster_node=self.cluster_node)
auditor.validate()
auditor.setup()
tracker.validate()
tracker.setup()
activitylogs.validate()
activitylogs.setup()
super(AuditorClusterTest, self).setUp()
def get_cluster_id(self):
if self.cluster_id:
return self.cluster_id
from clusters.models import Cluster
try:
cluster_uuid = Cluster.load().uuid.hex
self.cluster_id = cluster_uuid
except (Cluster.DoesNotExist, InterfaceError, ProgrammingError,
OperationalError):
pass
return self.cluster_id
def get_cluster_or_wait(self, log_sleep_interval):
max_trials = 10
trials = 0
while trials < max_trials:
try:
return Cluster.load()
except (InterfaceError, ProgrammingError, OperationalError) as e:
namespace.logger.exception("Database is not synced yet %s\n",
e)
trials += 1
time.sleep(log_sleep_interval * 2)
return None
def cluster_analytics():
cluster = Cluster.load()
notification = uuid.uuid4()
notification_url = settings.POLYAXON_NOTIFICATION_CLUSTER_ALIVE_URL.format(
url=settings.CLUSTER_NOTIFICATION_URL,
cluster_uuid=cluster.uuid.hex,
create_at=cluster.created_at.date(),
notification=notification,
version=settings.CHART_VERSION)
try:
requests.get(notification_url)
except requests.RequestException:
pass
def update_system_nodes():
k8s_manager = K8SManager(in_cluster=True)
nodes = k8s_manager.list_nodes()
cluster = Cluster.load()
nodes_to_update = {}
nodes_to_create = {node.metadata.name: node for node in nodes}
deprecated_nodes = []
for node in cluster.nodes.all():
if node.name in nodes_to_create:
nodes_to_update[node.name] = (node, nodes_to_create.pop(node.name))
else:
deprecated_nodes.append(node)
cluster_updated = False
for node in deprecated_nodes:
node.is_current = False
node.save()
cluster_updated = True
auditor.record(event_type=CLUSTER_NODE_DELETED, instance=node)
for node in nodes_to_create.values():
node_dict = ClusterNode.from_node_item(node)
node_dict['cluster'] = cluster
instance = ClusterNode.objects.create(**node_dict)
cluster_updated = True
auditor.record(event_type=CLUSTER_NODE_CREATED, instance=instance)
for current_node, new_node in nodes_to_update.values():
node_dict = ClusterNode.from_node_item(new_node)
node_updated = False
for k, v in node_dict.items():
if v != getattr(current_node, k):
setattr(current_node, k, v)
node_updated = True
if node_updated:
current_node.save()
cluster_updated = True
auditor.record(event_type=CLUSTER_NODE_UPDATED,
instance=current_node)
if cluster_updated:
cluster = get_cluster_resources()
auditor.record(event_type=CLUSTER_RESOURCES_UPDATED,
instance=cluster,
n_nodes=cluster.n_nodes,
memory=round(cluster.memory / (1000**3), 2),
n_cpus=cluster.n_cpus,
n_gpus=cluster.n_gpus)
def handle(self, *args, **options):
log_sleep_interval = options['log_sleep_interval']
self.stdout.write(
"Started a new namespace monitor with, "
"log sleep interval: `{}`.".format(log_sleep_interval),
ending='\n')
k8s_manager = K8SManager(namespace=settings.K8S_NAMESPACE,
in_cluster=True)
cluster = Cluster.load()
while True:
try:
namespace.run(k8s_manager, cluster)
except ApiException as e:
namespace.logger.error(
"Exception when calling CoreV1Api->list_event_for_all_namespaces: %s\n"
% e)
time.sleep(log_sleep_interval)
except Exception as e:
namespace.logger.exception(
"Unhandled exception occurred: %s\n" % e)
def test_cluster_invalid_without_creator():
with pytest.raises(ValidationError) as error:
cluster = Cluster(creator=None, cpus=1, memory=1)
cluster.save()
assert [*error.value.error_dict.keys()] == ['creator']
def cluster(request):
return {'cluster': Cluster.load()}
def perform_create(self, serializer):
serializer.save(cluster=Cluster.load())
def setUp(self):
super().setUp()
self.obj1 = self.factory_class(cluster=Cluster.load())
self.obj2 = self.factory_class(cluster=Cluster.load())
def setUp(self):
super().setUp()
node = ClusterNodeFactory(cluster=Cluster.load())
self.obj1 = self.factory_class(cluster_node=node)
self.obj2 = self.factory_class(cluster_node=node)
def setUp(self):
super().setUp()
self.cluster = Cluster.load()
ClusterNodeFactory(cluster=self.cluster)
ClusterNodeFactory(cluster=self.cluster)
def get_object(self):
return Cluster.load()
def test_loads_works(self):
assert Cluster.load() is not None
def get_cluster_node(**kwargs):
cluster = Cluster.load()
return ClusterNodeFactory(cluster=cluster, **kwargs)
def get_gpu(**kwargs):
if not kwargs.get('cluster_node'):
cluster = Cluster.load()
cluster['cluster_node'] = ClusterNodeFactory(cluster=cluster, **kwargs)
return GPUFactory(**kwargs)
