def setUp(self):
Cluster.load()
super().setUp()
self.tested_events = {
archives_events.ARCHIVE_BUILD_JOBS_VIEWED,
archives_events.ARCHIVE_JOBS_VIEWED,
archives_events.ARCHIVE_EXPERIMENTS_VIEWED,
archives_events.ARCHIVE_EXPERIMENT_GROUPS_VIEWED,
archives_events.ARCHIVE_PROJECTS_VIEWED,
}
def setUp(self):
Cluster.load()
super().setUp()
self.tested_events = {
bookmarks_events.BOOKMARK_BUILD_JOBS_VIEWED,
bookmarks_events.BOOKMARK_JOBS_VIEWED,
bookmarks_events.BOOKMARK_EXPERIMENTS_VIEWED,
bookmarks_events.BOOKMARK_EXPERIMENT_GROUPS_VIEWED,
bookmarks_events.BOOKMARK_PROJECTS_VIEWED,
}
def setUp(self):
Cluster.load()
auditor.validate()
auditor.setup()
tracker.validate()
tracker.setup()
activitylogs.validate()
activitylogs.setup()
notifier.validate()
notifier.setup()
super().setUp()
def get_node():
cluster = Cluster.load()
node = ClusterNode.objects.filter(cluster=cluster,
name=settings.K8S_NODE_NAME)
if node.count():
return node.first()
return None
def setUp(self):
super().setUp()
self.cluster_node = ClusterNodeFactory(cluster=Cluster.load())
self.url = '/{}/nodes/{}/gpus'.format(API_V1, self.cluster_node.sequence)
self.objects = [self.factory_class(cluster_node=self.cluster_node)
for _ in range(self.num_objects)]
self.queryset = self.model_class.objects.all()
def setUp(self):
super().setUp()
self.cluster = Cluster.load()
self.url = '/{}/cluster/nodes/'.format(API_V1)
self.objects = [self.factory_class(cluster=self.cluster) for _ in range(self.num_objects)]
self.queryset = self.model_class.objects.filter(cluster=self.cluster)
self.queryset = self.queryset.order_by('sequence')
def get_node():
cluster = Cluster.load()
node = ClusterNode.objects.filter(cluster=cluster,
name=conf.get('K8S_NODE_NAME'))
if node.exists():
return node.first()
return None
def setUp(self):
self.cluster = Cluster.load()
self.admin = UserFactory(is_staff=True, is_superuser=True)
self.user = UserFactory()
self.publisher = PublishTrackerService()
self.publisher.setup()
super(PublishTrackerTest, self).setUp()
def setUp(self):
self.cluster = Cluster.load()
self.admin = UserFactory(is_staff=True, is_superuser=True)
self.user = UserFactory()
self.publisher = PublishTrackerService()
self.publisher.setup()
super().setUp()
def setUp(self):
super().setUp()
self.cluster = Cluster.load()
self.obj1 = ClusterNodeFactory(cluster=self.cluster)
self.obj2 = ClusterNodeFactory(cluster=self.cluster)
self.gpu_obj1 = GPUFactory(cluster_node=self.obj1)
self.gpu_obj2 = GPUFactory(cluster_node=self.obj2)
def setUp(self):
super().setUp()
self.cluster = Cluster.load()
self.obj1 = ClusterNodeFactory(cluster=self.cluster)
self.obj2 = ClusterNodeFactory(cluster=self.cluster)
self.gpu_obj1 = GPUFactory(cluster_node=self.obj1)
self.gpu_obj2 = GPUFactory(cluster_node=self.obj2)
def setUp(self):
super().setUp()
self.object = Cluster.load()
self.url = '/{}/cluster/'.format(API_V1)
# Create related fields
for _ in range(2):
ClusterNodeFactory(cluster=self.object)
def setUp(self):
super().setUp()
self.owner = Cluster.get_or_create_owner(Cluster.load())
self.settings_service = DummySettingsService()
self.db_service = DummyDBService()
self.settings_service.option_manager = OptionManager()
self.db_service.option_manager = OptionManager()
self.settings_service.setup()
self.db_service.setup()
def setUp(self):
super().setUp()
self.owner = Owner.objects.get(name=Cluster.load().uuid)
self.settings_service = DummySettingsService()
self.db_service = DummyDBService()
self.settings_service.option_manager = OptionManager()
self.db_service.option_manager = OptionManager()
self.settings_service.setup()
self.db_service.setup()
def setUp(self):
super().setUp()
self.cluster = Cluster.load()
self.object = self.factory_class(cluster=self.cluster)
self.url = '/{}/nodes/{}/'.format(API_V1, self.object.sequence)
self.queryset = self.model_class.objects.all()
# Create related fields
for _ in range(2):
GPUFactory(cluster_node=self.object)
def update_system_info():
k8s_manager = K8SManager(in_cluster=True)
version_api = k8s_manager.get_version()
cluster = Cluster.load()
if cluster.version_api != version_api:
cluster.version_api = version_api
cluster.save()
auditor.record(event_type=CLUSTER_UPDATED,
instance=cluster,
is_upgrade=settings.CHART_IS_UPGRADE)
def update_system_info():
k8s_manager = K8SManager(in_cluster=True)
version_api = k8s_manager.get_version()
cluster = Cluster.load()
if version_api and cluster.version_api != version_api:
cluster.version_api = version_api
cluster.save()
auditor.record(event_type=CLUSTER_UPDATED,
instance=cluster,
is_upgrade=settings.CHART_IS_UPGRADE)
def test_update_works(self):
cluster = Cluster.load()
updated_cluster = Cluster.load()
cluster.version_api = {
'build_date': datetime.datetime.now().isoformat(),
'compiler': 'gc',
'git_commit': '17d7182a7ccbb167074be7a87f0a68bd00d58d91',
'git_tree_state': 'clean',
'git_version': 'v1.7.5',
'go_version': 'go1.8.3',
'major': '1',
'minor': '7',
'platform': 'linux/amd64'
}
updated_cluster.save()
loaded_version = Cluster.load()
assert loaded_version.version_api != cluster.version_api
assert loaded_version.version_api == updated_cluster.version_api
def get_cluster_or_wait(self, log_sleep_interval):
max_trials = 10
trials = 0
while trials < max_trials:
try:
return Cluster.load()
except (InterfaceError, ProgrammingError, OperationalError) as e:
monitor.logger.exception("Database is not synced yet %s\n", e)
trials += 1
time.sleep(log_sleep_interval * 2)
return None
def get_cluster_or_wait(self, log_sleep_interval):
max_trials = 10
trials = 0
while trials < max_trials:
try:
return Cluster.load()
except (InterfaceError, ProgrammingError, OperationalError) as e:
monitor.logger.exception("Database is not synced yet %s\n", e)
trials += 1
time.sleep(log_sleep_interval * 2)
return None
def setUp(self):
self.cluster = Cluster.load()
self.cluster_node = ClusterNodeFactory(cluster=self.cluster)
self.node_gpu = GPUFactory(cluster_node=self.cluster_node)
auditor.validate()
auditor.setup()
tracker.validate()
tracker.setup()
activitylogs.validate()
activitylogs.setup()
super(AuditorClusterTest, self).setUp()
def get_cluster_id(self):
if self.cluster_id:
return self.cluster_id
from db.models.clusters import Cluster
try:
cluster_uuid = Cluster.load().uuid.hex
self.cluster_id = cluster_uuid
except (Cluster.DoesNotExist, InterfaceError, ProgrammingError, OperationalError):
pass
return self.cluster_id
def get_cluster_id(self):
if self.cluster_id:
return self.cluster_id
from db.models.clusters import Cluster
try:
cluster_uuid = Cluster.load().uuid.hex
self.cluster_id = cluster_uuid
except (Cluster.DoesNotExist, InterfaceError, ProgrammingError, OperationalError):
pass
return self.cluster_id
def get_ref_id(self) -> str:
if self.ref_id:
return self.ref_id
from db.models.clusters import Cluster
from django.db import InterfaceError, OperationalError, ProgrammingError
try:
self.ref_id = Cluster.load().uuid
except (Cluster.DoesNotExist, InterfaceError, ProgrammingError,
OperationalError):
pass
return self.ref_id
def cluster_analytics() -> None:
cluster = Cluster.load()
notification = uuid.uuid4()
notification_url = conf.get(CLUSTER_NOTIFICATION_ALIVE_URL).format(
url=conf.get(CLUSTER_NOTIFICATION_URL),
cluster_uuid=cluster.uuid.hex,
created_at=cluster.created_at.date(),
notification=notification,
version=conf.get(CHART_VERSION))
try:
requests.get(notification_url)
except requests.RequestException:
pass
def cluster_analytics():
cluster = Cluster.load()
notification = uuid.uuid4()
notification_url = settings.POLYAXON_NOTIFICATION_CLUSTER_ALIVE_URL.format(
url=settings.CLUSTER_NOTIFICATION_URL,
cluster_uuid=cluster.uuid.hex,
create_at=cluster.created_at.date(),
notification=notification,
version=settings.CHART_VERSION)
try:
requests.get(notification_url)
except requests.RequestException:
pass
def cluster_owner(self) -> 'Owner':
if self._cluster_owner:
return self._cluster_owner
from db.models.clusters import Cluster
try:
cluster = Cluster.load()
self._cluster_owner = cluster.get_or_create_owner(cluster)
except (Cluster.DoesNotExist, InterfaceError, ProgrammingError,
OperationalError):
pass
return self._cluster_owner
def cluster_owner(self) -> 'Owner':
if self._cluster_owner:
return self._cluster_owner
from db.models.clusters import Cluster
try:
self._cluster_owner = self.owner_manager.get(
name=Cluster.load().uuid)
except (Cluster.DoesNotExist, InterfaceError, ProgrammingError,
OperationalError):
pass
return self._cluster_owner
def update_system_nodes() -> None:
k8s_manager = K8SManager(in_cluster=True)
nodes = k8s_manager.list_nodes()
cluster = Cluster.load()
nodes_to_update = {}
nodes_to_create = {node.metadata.name: node for node in nodes}
deprecated_nodes = []
for node in cluster.nodes.all():
if node.name in nodes_to_create:
nodes_to_update[node.name] = (node, nodes_to_create.pop(node.name))
elif node.is_current:
deprecated_nodes.append(node)
cluster_updated = False
for node in deprecated_nodes:
node.is_current = False
node.save()
cluster_updated = True
auditor.record(event_type=CLUSTER_NODE_DELETED, instance=node)
for node in nodes_to_create.values():
node_dict = ClusterNode.from_node_item(node)
node_dict['cluster'] = cluster
instance = ClusterNode.objects.create(**node_dict)
cluster_updated = True
auditor.record(event_type=CLUSTER_NODE_CREATED, instance=instance)
for current_node, new_node in nodes_to_update.values():
node_dict = ClusterNode.from_node_item(new_node)
node_updated = False
for k, v in node_dict.items():
if v != getattr(current_node, k):
setattr(current_node, k, v)
node_updated = True
if not current_node.is_current:
current_node.is_current = True
node_updated = True
if node_updated:
current_node.save()
cluster_updated = True
auditor.record(event_type=CLUSTER_NODE_UPDATED,
instance=current_node)
if cluster_updated:
cluster = get_cluster_resources()
auditor.record(event_type=CLUSTER_RESOURCES_UPDATED,
instance=cluster,
n_nodes=cluster.n_nodes,
memory=round(cluster.memory / (1000**3), 2),
n_cpus=cluster.n_cpus,
n_gpus=cluster.n_gpus)
def setUp(self):
super().setUp()
self.cluster = Cluster.load()
self.cluster_node = ClusterNodeFactory(cluster=self.cluster)
self.node_gpu = GPUFactory(cluster_node=self.cluster_node)
self.tested_events = {
cluster_events.CLUSTER_CREATED,
cluster_events.CLUSTER_UPDATED,
cluster_events.CLUSTER_RESOURCES_UPDATED,
cluster_events.CLUSTER_NODE_CREATED,
cluster_events.CLUSTER_NODE_UPDATED,
cluster_events.CLUSTER_NODE_DELETED,
cluster_events.CLUSTER_NODE_GPU,
}
def setUp(self):
super().setUp()
self.owner = Owner.objects.get(name=Cluster.load().uuid)
self.option_value1 = ConfigOption.objects.create(owner=self.owner,
key='value1',
value=2341)
self.option_value2 = ConfigOption.objects.create(owner=self.owner,
key='value2',
value={'foo': 'bar'})
self.option_secret1 = ConfigOption.objects.create(owner=self.owner,
key='secret1',
secret='secret1')
self.option_secret2 = ConfigOption.objects.create(owner=self.owner,
key='secret2',
secret='secret2')
def update_system_nodes():
k8s_manager = K8SManager(in_cluster=True)
nodes = k8s_manager.list_nodes()
cluster = Cluster.load()
nodes_to_update = {}
nodes_to_create = {node.metadata.name: node for node in nodes}
deprecated_nodes = []
for node in cluster.nodes.all():
if node.name in nodes_to_create:
nodes_to_update[node.name] = (node, nodes_to_create.pop(node.name))
else:
deprecated_nodes.append(node)
cluster_updated = False
for node in deprecated_nodes:
node.is_current = False
node.save()
cluster_updated = True
auditor.record(event_type=CLUSTER_NODE_DELETED, instance=node)
for node in nodes_to_create.values():
node_dict = ClusterNode.from_node_item(node)
node_dict['cluster'] = cluster
instance = ClusterNode.objects.create(**node_dict)
cluster_updated = True
auditor.record(event_type=CLUSTER_NODE_CREATED, instance=instance)
for current_node, new_node in nodes_to_update.values():
node_dict = ClusterNode.from_node_item(new_node)
node_updated = False
for k, v in node_dict.items():
if v != getattr(current_node, k):
setattr(current_node, k, v)
node_updated = True
if node_updated:
current_node.save()
cluster_updated = True
auditor.record(event_type=CLUSTER_NODE_UPDATED, instance=current_node)
if cluster_updated:
cluster = get_cluster_resources()
auditor.record(event_type=CLUSTER_RESOURCES_UPDATED,
instance=cluster,
n_nodes=cluster.n_nodes,
memory=round(cluster.memory / (1000 ** 3), 2),
n_cpus=cluster.n_cpus,
n_gpus=cluster.n_gpus)
def test_create(self):
data = {}
resp = self.auth_client.post(self.url, data)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
secret = K8SSecretFactory()
data = {
'name': 'new_config',
'description': 'some description',
'tags': ['foo', 'bar'],
'type': StoreTypes.S3,
'bucket': 'foo',
'k8s_secret': secret.id,
}
resp = self.auth_client.post(self.url, data)
assert resp.status_code == status.HTTP_201_CREATED
assert self.model_class.objects.count() == self.num_objects + 1
last_object = self.model_class.objects.last()
assert last_object.owner.owner == Cluster.load()
assert last_object.name == data['name']
assert last_object.description == data['description']
assert last_object.tags == data['tags']
assert last_object.type == data['type']
assert last_object.bucket == data['bucket']
assert last_object.k8s_secret.id == data['k8s_secret']
# Non admin
data = {}
resp = self.normal_client.post(self.url, data)
assert resp.status_code == status.HTTP_403_FORBIDDEN
data = {
'name': 'new_config',
'description': 'some description',
'tags': ['foo', 'bar'],
'type': StoreTypes.S3,
'bucket': 'foo',
'k8s_secret': secret.id,
}
resp = self.normal_client.post(self.url, data)
assert resp.status_code == status.HTTP_403_FORBIDDEN
def test_create(self):
data = {}
resp = self.auth_client.post(self.url, data)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
data = {
'name': 'new_config',
'description': 'some description',
'k8s_ref': 'k8s_resource',
'items': ['key1', 'key2'],
'tags': ['foo', 'bar']
}
resp = self.auth_client.post(self.url, data)
assert resp.status_code == status.HTTP_201_CREATED
assert self.model_class.objects.count() == self.num_objects + 1
last_object = self.model_class.objects.last()
assert last_object.owner.owner == Cluster.load()
assert last_object.name == data['name']
assert last_object.description == data['description']
assert last_object.k8s_ref == data['k8s_ref']
assert last_object.items == data['items']
assert last_object.tags == data['tags']
# Create with same name and owner should raise
resp = self.auth_client.post(self.url, data)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
# Non admin
data = {}
resp = self.normal_client.post(self.url, data)
assert resp.status_code == status.HTTP_403_FORBIDDEN
data = {
'name': 'new_config',
'description': 'some description',
'k8s_ref': 'k8s_ref1',
'items': ['key1', 'key2'],
'tags': ['foo', 'bar']
}
resp = self.normal_client.post(self.url, data)
assert resp.status_code == status.HTTP_403_FORBIDDEN
def cluster(request):
return {'cluster': Cluster.load()}
def perform_create(self, serializer):
serializer.save(cluster=Cluster.load())
def get_object(self):
return Cluster.load()
def get_cluster_node(**kwargs):
cluster = Cluster.load()
return ClusterNodeFactory(cluster=cluster, **kwargs)
def get_gpu(**kwargs):
if not kwargs.get('cluster_node'):
cluster = Cluster.load()
cluster['cluster_node'] = ClusterNodeFactory(cluster=cluster, **kwargs)
return GPUFactory(**kwargs)
def setUp(self):
super().setUp()
self.owner = Cluster.get_or_create_owner(Cluster.load())
self.cluster_options_handler = ClusterOptionsHandler()
def perform_create(self, serializer):
serializer.save(cluster=Cluster.load())
def setUp(self):
super().setUp()
self.cluster = Cluster.load()
ClusterNodeFactory(cluster=self.cluster)
ClusterNodeFactory(cluster=self.cluster)
def setUp(self):
super().setUp()
node = ClusterNodeFactory(cluster=Cluster.load())
self.obj1 = self.factory_class(cluster_node=node)
self.obj2 = self.factory_class(cluster_node=node)
def setUp(self):
super().setUp()
self.obj1 = self.factory_class(cluster=Cluster.load())
self.obj2 = self.factory_class(cluster=Cluster.load())
