def test_switching_search_path(self): dummies_tenant1_count, dummies_tenant2_count = 0, 0 tenant1 = Tenant(domain_url='test.com', schema_name='tenant1') tenant1.save() tenant2 = Tenant(domain_url='example.com', schema_name='tenant2') tenant2.save() # go to tenant1's path connection.set_tenant(tenant1) # add some data DummyModel(name="Schemas are").save() DummyModel(name="awesome!").save() dummies_tenant1_count = DummyModel.objects.count() # switch temporarily to tenant2's path with tenant_context(tenant2): # add some data DummyModel(name="Man,").save() DummyModel(name="testing").save() DummyModel(name="is great!").save() dummies_tenant2_count = DummyModel.objects.count() # we should be back to tenant1's path, test what we have self.assertEqual(DummyModel.objects.count(), dummies_tenant1_count) # switch back to tenant2's path with tenant_context(tenant2): self.assertEqual(DummyModel.objects.count(), dummies_tenant2_count)
def test_switching_search_path(self): tenant1 = Tenant(domain_url='something.test.com', schema_name='tenant1') tenant1.save() connection.set_schema_to_public() tenant2 = Tenant(domain_url='example.com', schema_name='tenant2') tenant2.save() # go to tenant1's path connection.set_tenant(tenant1) # add some data, 2 DummyModels for tenant1 DummyModel(name="Schemas are").save() DummyModel(name="awesome!").save() # switch temporarily to tenant2's path with tenant_context(tenant2): # add some data, 3 DummyModels for tenant2 DummyModel(name="Man,").save() DummyModel(name="testing").save() DummyModel(name="is great!").save() # we should be back to tenant1's path, test what we have self.assertEqual(2, DummyModel.objects.count()) # switch back to tenant2's path with tenant_context(tenant2): self.assertEqual(3, DummyModel.objects.count())
def setUpClass(cls): super(SharedAuthTest, cls).setUpClass() settings.SHARED_APPS = ('tenant_schemas', 'django.contrib.auth', 'django.contrib.contenttypes', ) settings.TENANT_APPS = ('dts_test_app', ) settings.INSTALLED_APPS = settings.SHARED_APPS + settings.TENANT_APPS cls.sync_shared() Tenant(domain_url='test.com', schema_name=get_public_schema_name()).save() # Create a tenant cls.tenant = Tenant(domain_url='tenant.test.com', schema_name='tenant') cls.tenant.save() # Create some users with schema_context(get_public_schema_name()): # this could actually also be executed inside a tenant cls.user1 = User(username='******', email="*****@*****.**") cls.user1.save() cls.user2 = User(username='******', email="*****@*****.**") cls.user2.save() # Create instances on the tenant that point to the users on public with tenant_context(cls.tenant): cls.d1 = ModelWithFkToPublicUser(user=cls.user1) cls.d1.save() cls.d2 = ModelWithFkToPublicUser(user=cls.user2) cls.d2.save()
def wrapper(*args, **kwargs): if settings.MULTI_TENANT: for tenant in get_tenant_model().objects.exclude(schema_name="public"): with tenant_context(tenant): f(*args, **kwargs) else: f(*args, **kwargs)
def receive(self, content, **kwargs): tenant_pk = self.message.channel_session.get('tenant') logger.info('tenant pk: {}'.format(tenant_pk)) if tenant_pk is None: logger.error('TransactionConsumer tenant not in session') return try: tenant = get_tenant_model().objects.get( pk=tenant_pk ) except get_tenant_model().DoesNotExist: return with tenant_context(tenant): more_blocks = Block.objects.exclude( height__isnull=True ).filter( height__lt=content.get('height') ).order_by( '-height' )[:20] kwargs.get('multiplexer').send( { "more_blocks": [ block.serialize() for block in more_blocks ] } )
def receive(self, content, multiplexer, **kwargs): logger.info(multiplexer) tenant_pk = self.message.channel_session.get('tenant') logger.info('tenant pk: {}'.format(tenant_pk)) if tenant_pk is None: logger.error('TransactionConsumer tenant not in session') logger.info(self.message.content) return try: tenant = get_tenant_model().objects.get( pk=tenant_pk ) except get_tenant_model().DoesNotExist: return with tenant_context(tenant): try: block = Block.objects.get(height=int(content.get('height'))) except Block.DoesNotExist: return multiplexer.send( { 'transactions': [ tx.serialize() for tx in block.transactions.all() ] } )
def receive(self, content, multiplexer, **kwargs): logger.info(multiplexer) logger.info(self.message) tenant_pk = self.message.channel_session.get('tenant') logger.info('tenant pk: {}'.format(tenant_pk)) if tenant_pk is None: logger.error('TransactionConsumer tenant not in session') logger.info(self.message.channel) return try: tenant = get_tenant_model().objects.get( pk=tenant_pk ) except get_tenant_model().DoesNotExist: return with tenant_context(tenant): try: block = Block.objects.get(height=content.get('height')) except Block.DoesNotExist: return multiplexer.send( { 'block': block.serialize(), 'next_block': block.next_block.height if block.next_block else None, 'previous_block': block.previous_block.height if block.previous_block else None, } )
def create_superuser(tenant, email): # raise Exception(email) with tenant_context(tenant): USER_MODEL = get_user_model() # Create a superuser for the given tenant # we set a dummy password for now ... # we should also send an activation mail, as we set no password USER_MODEL.objects.create_superuser(email=email, password="******")
def test_direct_relation_to_public(self): """ Tests that a forward relationship through a foreign key to public from a model inside TENANT_APPS works. """ with tenant_context(self.tenant): self.assertEqual(User.objects.get(pk=self.user1.id), ModelWithFkToPublicUser.objects.get(pk=self.d1.id).user) self.assertEqual(User.objects.get(pk=self.user2.id), ModelWithFkToPublicUser.objects.get(pk=self.d2.id).user)
def test_reverse_relation_to_public(self): """ Tests that a reverse relationship through a foreign keys to public from a model inside TENANT_APPS works. """ with tenant_context(self.tenant): users = User.objects.all().select_related().order_by('id') self.assertEqual(ModelWithFkToPublicUser.objects.get(pk=self.d1.id), users[0].modelwithfktopublicuser_set.all()[:1].get()) self.assertEqual(ModelWithFkToPublicUser.objects.get(pk=self.d2.id), users[1].modelwithfktopublicuser_set.all()[:1].get())
def test_switching_tenant_without_previous_tenant(self): tenant = Tenant(domain_url='something.test.com', schema_name='test') tenant.save() connection.tenant = None with tenant_context(tenant): DummyModel(name="No exception please").save() connection.tenant = None with schema_context(tenant.schema_name): DummyModel(name="Survived it!").save()
def event_generator(tenant_id, user_list, object_id, content_id): tenant = get_object_or_None(Customer, id=tenant_id) if not tenant: return with tenant_context(tenant): try: content_type_object = ContentType.objects.get(id=content_id) content_object = content_type_object\ .get_object_for_this_type(id=object_id) except ObjectDoesNotExist: return recipients = get_user_model().objects.filter(id__in=user_list) for recipient in recipients: content_object.create_event_object(user=recipient)
def _make_role(tenant, data, update=False): """Create the role object in the database.""" with tenant_context(tenant): name = data.pop('name') description = data.pop('description', None) access_list = data.pop('access') version = data.pop('version', 1) version_diff = False is_platform_default = data.pop('platform_default', False) if update: role, created = Role.objects.filter(name=name).get_or_create( name=name) version_diff = version != role.version if created or (not created and version_diff): logger.info('Updating role %s for tenant %s.', name, tenant.schema_name) role.description = description role.system = True role.version = version role.platform_default = is_platform_default role.save() role.access.all().delete() else: role = Role.objects.create(name=name, description=description, system=True, version=version, platform_default=is_platform_default) logger.info('Creating role %s for tenant %s.', name, tenant.schema_name) if not update or (update and version_diff): for access_item in access_list: resource_def_list = access_item.pop('resourceDefinitions', []) access_obj = Access.objects.create(**access_item, role=role) access_obj.save() for resource_def_item in resource_def_list: res_def = ResourceDefinition.objects.create( **resource_def_item, access=access_obj) res_def.save() return role
def test_get_cluster_capacity_daily_resolution_group_by_clusters(self): """Test that cluster capacity returns daily capacity by cluster.""" url = '?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=daily&group_by[cluster]=*' query_params = self.mocked_query_params(url, OCPCpuView) handler = OCPReportQueryHandler(query_params) query_data = handler.execute_query() daily_capacity_by_cluster = defaultdict(dict) total_capacity = Decimal(0) query_filter = handler.query_filter query_group_by = ['usage_start', 'cluster_id'] annotations = {'capacity': Max('cluster_capacity_cpu_core_hours')} cap_key = list(annotations.keys())[0] q_table = handler._mapper.query_table query = q_table.objects.filter(query_filter) with tenant_context(self.tenant): cap_data = query.values(*query_group_by).annotate(**annotations) for entry in cap_data: date = handler.date_to_string(entry.get('usage_start')) cluster_id = entry.get('cluster_id', '') if cluster_id in daily_capacity_by_cluster[date]: daily_capacity_by_cluster[date][cluster_id] += entry.get(cap_key, 0) else: daily_capacity_by_cluster[date][cluster_id] = entry.get(cap_key, 0) total_capacity += entry.get(cap_key, 0) for entry in query_data.get('data', []): date = entry.get('date') for cluster in entry.get('clusters', []): cluster_name = cluster.get('cluster', '') capacity = cluster.get('values')[0].get('capacity', {}).get('value') self.assertEqual( capacity, daily_capacity_by_cluster[date][cluster_name] ) self.assertEqual( query_data.get('total', {}).get('capacity', {}).get('value'), total_capacity )
def setUp(self): """Set up the role viewset tests.""" super().setUp() request = self.request_context['request'] user = User() user.username = self.user_data['username'] user.account = self.customer_data['account_id'] request.user = user sys_role_config = {'name': 'system_role', 'system': True} def_role_config = {'name': 'default_role', 'platform_default': True} self.display_fields = { 'applications', 'description', 'uuid', 'name', 'system', 'created', 'policyCount', 'accessCount', 'modified', 'platform_default' } with tenant_context(self.tenant): self.principal = Principal(username=self.user_data['username']) self.principal.save() self.policy = Policy.objects.create(name='policyA') self.group = Group(name='groupA', description='groupA description') self.group.save() self.group.principals.add(self.principal) self.group.policies.add(self.policy) self.group.save() self.sysRole = Role(**sys_role_config) self.sysRole.save() self.defRole = Role(**def_role_config) self.defRole.save() self.defRole.save() self.policy.roles.add(self.defRole, self.sysRole) self.policy.save() self.access = Access.objects.create(permission='app:*:*', role=self.defRole)
def test_execute_query_with_and_filter(self): """Test the filter[and:] param in the view.""" url = reverse("openshift-tags") client = APIClient() with tenant_context(self.tenant): projects = (OCPUsageLineItemDailySummary.objects.filter( usage_start__gte=self.ten_days_ago).values( "namespace").distinct()) projects = [project.get("namespace") for project in projects] params = { "filter[resolution]": "daily", "filter[time_scope_value]": "-10", "filter[time_scope_units]": "day", "filter[and:project]": projects, } url = url + "?" + urlencode(params, quote_via=quote_plus) response = client.get(url, **self.headers) self.assertEqual(response.status_code, status.HTTP_200_OK) response_data = response.json() self.assertEqual(response_data.get("data", []), [])
def tenant_view(request, tenant_schema_name): """View method for internal tenant requests. DELETE /_private/api/tenant/<schema_name>/ """ logger.info(f"Tenant view: {request.method} {request.user.username}") if request.method == "DELETE": if not destructive_ok(): return HttpResponse("Destructive operations disallowed.", status=400) tenant_obj = get_object_or_404(Tenant, schema_name=tenant_schema_name) with tenant_context(tenant_obj): if tenant_is_unmodified(): logger.warning( f"Deleting tenant {tenant_schema_name}. Requested by {request.user.username}" ) tenant_obj.delete() return HttpResponse(status=204) else: return HttpResponse("Tenant cannot be deleted.", status=400) return HttpResponse(f'Method "{request.method}" not allowed.', status=405)
def destroy(self, request, *args, **kwargs): """Delete a group. @api {delete} /api/v1/groups/:uuid Delete a group @apiName deleteGroup @apiGroup Group @apiVersion 1.0.0 @apiDescription Delete a group @apiHeader {String} token User authorization token @apiParam (Path) {String} uuid Group unique identifier @apiSuccessExample {json} Success-Response: HTTP/1.1 204 NO CONTENT """ validate_uuid(kwargs.get("uuid"), "group uuid validation") self.protect_default_groups("delete") group_name = Group.objects.get(uuid=kwargs.get("uuid")).name with tenant_context(Tenant.objects.get(schema_name="public")): Group.objects.filter(name=group_name, tenant=request.tenant).delete() return super().destroy(request=request, args=args, kwargs=kwargs)
def test_remove_aws_auth_billing_remain(self): """Remove aws provider.""" # Create Provider provider_authentication = ProviderAuthentication.objects.create(provider_resource_name='arn:aws:iam::2:role/mg') provider_authentication2 = ProviderAuthentication.objects.create( provider_resource_name='arn:aws:iam::3:role/mg' ) provider_billing = ProviderBillingSource.objects.create(bucket='my_s3_bucket') provider = Provider.objects.create(name='awsprovidername', created_by=self.user, customer=self.customer, authentication=provider_authentication, billing_source=provider_billing) provider2 = Provider.objects.create(name='awsprovidername2', created_by=self.user, customer=self.customer, authentication=provider_authentication2, billing_source=provider_billing) provider_uuid = provider2.uuid self.assertNotEqual(provider.uuid, provider2.uuid) new_user_dict = self._create_user_data() request_context = self._create_request_context(self.customer_data, new_user_dict, False) user_serializer = UserSerializer(data=new_user_dict, context=request_context) other_user = None if user_serializer.is_valid(raise_exception=True): other_user = user_serializer.save() with tenant_context(self.tenant): manager = ProviderManager(provider_uuid) manager.remove(self._create_delete_request(other_user)) auth_count = ProviderAuthentication.objects.count() billing_count = ProviderBillingSource.objects.count() provider_query = Provider.objects.all().filter(uuid=provider_uuid) self.assertFalse(provider_query) self.assertEqual(auth_count, 1) self.assertEqual(billing_count, 1)
def test_execute_query_ocp_aws_costs_group_by_project(self): """Test that grouping by project filters data.""" with tenant_context(self.tenant): # Force Django to do GROUP BY to get nodes projects = (OCPAWSCostLineItemDailySummary.objects.filter( usage_start__gte=self.ten_days_ago).values( *["namespace"]).annotate( project_count=Count("namespace")).all()) project_of_interest = projects[0].get("namespace") url = reverse("reports-openshift-aws-costs") client = APIClient() params = {"group_by[project]": project_of_interest} url = url + "?" + urlencode(params, quote_via=quote_plus) response = client.get(url, **self.headers) self.assertEqual(response.status_code, status.HTTP_200_OK) data = response.json() for entry in data.get("data", []): for project in entry.get("projects", []): self.assertEqual(project.get("project"), project_of_interest)
def test_tiered_rate_with_duplicate(self): """Test creating a tiered rate with duplicate tiers.""" self.ocp_data['rates'][0]['tiered_rates'] = [ { 'unit': 'USD', 'value': 0.22, 'usage': { 'usage_start': None, 'usage_end': 10.0 } }, { 'unit': 'USD', 'value': 0.26, 'usage': { 'usage_start': 10.0, 'usage_end': 20.0 } }, { 'unit': 'USD', 'value': 0.26, 'usage': { 'usage_start': 10.0, 'usage_end': 20.0 } }, { 'unit': 'USD', 'value': 0.26, 'usage': { 'usage_start': 20.0, 'usage_end': None } } ] with tenant_context(self.tenant): serializer = CostModelSerializer(data=self.ocp_data) with self.assertRaises(serializers.ValidationError): if serializer.is_valid(raise_exception=True): serializer.save()
def test_product_update(self, member1, client1, planninguser1): with tenant_context(member1.tenant): client1.force_login(planninguser1) product = factories.ProductFactory( name='test', identifier='bla', ) response = client1.put( reverse('purchase-product-detail', kwargs={'pk': product.id}), { 'name': 'test name', 'identifier': 'bla', 'price_purchase': '0.00', 'price_selling': '0.00', 'price_selling_alt': '0.00', 'price_purchase_ex': '0.00', 'price_selling_ex': '0.00', 'price_selling_alt_ex': '0.00', }) assert response.status_code == status.HTTP_200_OK assert response.data['name'] == 'test name'
def test_create_storage_no_tiers_rate(self): """Test creating a non tiered storage rate.""" storage_rates = ( metric_constants.OCP_METRIC_STORAGE_GB_REQUEST_MONTH, metric_constants.OCP_METRIC_STORAGE_GB_USAGE_MONTH, ) for storage_rate in storage_rates: ocp_data = { "name": "Test Cost Model", "description": "Test", "source_type": Provider.PROVIDER_OCP, "providers": [{"uuid": self.provider.uuid, "name": self.provider.name}], "rates": [{"metric": {"name": storage_rate}, "tiered_rates": [{"unit": "USD", "value": 0.22}]}], } with tenant_context(self.tenant): instance = None serializer = CostModelSerializer(data=ocp_data) if serializer.is_valid(raise_exception=True): instance = serializer.save() self.assertIsNotNone(instance) self.assertIsNotNone(instance.uuid)
def test_get_cluster_capacity_daily_resolution(self): """Test that total capacity is returned daily resolution.""" url = '?filter[time_scope_units]=month&filter[time_scope_value]=-1&filter[resolution]=daily' query_params = self.mocked_query_params(url, OCPCpuView) handler = OCPReportQueryHandler(query_params) query_data = handler.execute_query() daily_capacity = defaultdict(Decimal) total_capacity = Decimal(0) query_filter = handler.query_filter query_group_by = ['usage_start'] annotations = {'capacity': Max('total_capacity_cpu_core_hours')} cap_key = list(annotations.keys())[0] q_table = handler._mapper.provider_map.get('tables').get('query') query = q_table.objects.filter(query_filter) with tenant_context(self.tenant): cap_data = query.values(*query_group_by).annotate(**annotations) for entry in cap_data: date = handler.date_to_string(entry.get('usage_start')) daily_capacity[date] += entry.get(cap_key, 0) # This is a hack because the total capacity in the test data # is artificial but the total should still be a sum of # cluster capacities annotations = {'capacity': Max('cluster_capacity_cpu_core_hours')} cap_data = query.values(*query_group_by).annotate(**annotations) for entry in cap_data: total_capacity += entry.get(cap_key, 0) self.assertEqual( query_data.get('total', {}).get('capacity', {}).get('value'), total_capacity ) for entry in query_data.get('data', []): date = entry.get('date') values = entry.get('values') if values: capacity = values[0].get('capacity', {}).get('value') self.assertEqual(capacity, daily_capacity[date])
def test_delete_single_provider_with_cost_model(self, mock_delete_archived_data, mock_commit): """Assert the cost models are deleted upon provider instance delete.""" mock_commit.side_effect = mock_delete_archived_data.delay( self.schema, Provider.PROVIDER_AWS, self.aws_provider_uuid ) provider_uuid = self.aws_provider.uuid data = { "name": "Test Cost Model", "description": "Test", "rates": [], "markup": {"value": FAKE.pyint() % 100, "unit": "percent"}, "provider_uuids": [provider_uuid], } with tenant_context(self.tenant): manager = CostModelManager() with patch("cost_models.cost_model_manager.chain"): manager.create(**data) cost_model_map = CostModelMap.objects.filter(provider_uuid=provider_uuid) self.assertIsNotNone(cost_model_map) self.aws_provider.delete() self.assertEquals(0, CostModelMap.objects.filter(provider_uuid=provider_uuid).count()) mock_delete_archived_data.delay.assert_called_with(self.schema, Provider.PROVIDER_AWS, self.aws_provider_uuid)
def test_process_multiple_tag_query_params(self): """Test that grouping by multiple tag keys returns a valid response.""" with tenant_context(self.tenant): labels = (OCPAWSCostLineItemDailySummary.objects.filter( usage_start__gte=self.ten_days_ago).values(*["tags"]).first()) self.assertIsNotNone(labels) tags = labels.get("tags") qstr = "filter[limit]=2" # pick a random subset of tags kval = len(tags.keys()) if kval > 2: kval = random.randint(2, len(tags.keys())) selected_tags = random.choices(list(tags.keys()), k=kval) for tag in selected_tags: qstr += f"&group_by[tag:{tag}]=*" url = reverse("reports-openshift-aws-costs") + "?" + qstr client = APIClient() response = client.get(url, **self.headers) self.assertEqual(response.status_code, status.HTTP_200_OK)
def test_execute_query_ocp_aws_storage_with_group_by_tag_and_limit(self): """Test that data is grouped by tag key and limited.""" with tenant_context(self.tenant): labels = (OCPAWSCostLineItemDailySummary.objects.filter( usage_start__gte=self.dh.last_month_start).filter( usage_start__lte=self.dh.last_month_end).filter( product_family__contains="Storage").values( *["tags"]).first()) self.assertIsNotNone(labels) self.assertNotEqual(len(labels), 0) tags = labels.get("tags") group_by_key = list(tags.keys())[0] plural_key = group_by_key + "s" url = reverse("reports-openshift-aws-storage") client = APIClient() params = { "filter[resolution]": "monthly", "filter[time_scope_value]": "-2", "filter[time_scope_units]": "month", f"group_by[tag:{group_by_key}]": "*", "filter[limit]": 2, } url = url + "?" + urlencode(params, quote_via=quote_plus) response = client.get(url, **self.headers) self.assertEqual(response.status_code, status.HTTP_200_OK) data = response.json() data = data.get("data", []) # default ordered by usage previous_tag_usage = data[0].get(plural_key, [])[0].get( "values", [{}])[0].get("usage", {}).get("value") for entry in data[0].get(plural_key, []): current_tag_usage = entry.get("values", [{}])[0].get("usage", {}).get("value") if "Other" not in entry.get(group_by_key): self.assertTrue(current_tag_usage <= previous_tag_usage) previous_tag_usage = current_tag_usage
def test_execute_query_ocp_aws_storage_with_tag_filter(self): """Test that data is filtered by tag key.""" with tenant_context(self.tenant): labels = (OCPAWSCostLineItemDailySummary.objects.filter( usage_start__gte=self.ten_days_ago).filter( product_family__contains="Storage").values( *["tags"]).first()) self.assertIsNotNone(labels) tags = labels.get("tags") filter_key = list(tags.keys())[0] filter_value = tags.get(filter_key) totals = (OCPAWSCostLineItemDailySummary.objects.filter( usage_start__gte=self.ten_days_ago).filter(**{ f"tags__{filter_key}": filter_value }).filter(product_family__contains="Storage").aggregate( **{ "usage": Sum("usage_amount"), "cost": Sum(F("unblended_cost") + F("markup_cost")) })) url = reverse("reports-openshift-aws-storage") client = APIClient() params = {f"filter[tag:{filter_key}]": filter_value} url = url + "?" + urlencode(params, quote_via=quote_plus) response = client.get(url, **self.headers) self.assertEqual(response.status_code, status.HTTP_200_OK) data = response.json() data_totals = data.get("meta", {}).get("total", {}) for key in totals: expected = float(totals[key]) if key == "cost": result = data_totals.get(key, {}).get("total").get("value") else: result = data_totals.get(key, {}).get("value") self.assertEqual(result, expected)
def setUp(self): """Set up the rate view tests.""" super().setUp() request = self.request_context['request'] serializer = UserSerializer(data=self.user_data, context=self.request_context) if serializer.is_valid(raise_exception=True): user = serializer.save() request.user = user provider_data = { 'name': 'test_provider', 'type': Provider.PROVIDER_OCP, 'authentication': { 'provider_resource_name': self.fake.word() } } serializer = ProviderSerializer(data=provider_data, context=self.request_context) if serializer.is_valid(raise_exception=True): self.provider = serializer.save() self.fake_data = { 'provider_uuid': self.provider.uuid, 'metric': Rate.METRIC_MEM_GB_USAGE_HOUR, 'tiered_rate': [{ 'value': round(Decimal(random.random()), 6), 'unit': 'USD', 'usage_start': None, 'usage_end': None }] } with tenant_context(self.tenant): serializer = RateSerializer(data=self.fake_data, context=self.request_context) if serializer.is_valid(raise_exception=True): serializer.save()
def _get_tenant_provider_stats(self, provider, tenant, period_start): """Return provider statistics for schema.""" stats = {} query = None with tenant_context(tenant): if provider.type == 'OCP': query = OCPUsageReportPeriod.objects.filter(provider_id=provider.id, report_period_start=period_start).first() elif provider.type == 'AWS' or provider.type == 'AWS-local': query = AWSCostEntryBill.objects.filter(provider_id=provider.id, billing_period_start=period_start).first() elif provider.type == 'AZURE' or provider.type == 'AZURE-local': query = AzureCostEntryBill.objects.filter(provider_id=provider.id, billing_period_start=period_start).first() if query and query.summary_data_creation_datetime: stats['summary_data_creation_datetime'] = query.summary_data_creation_datetime.strftime(DATE_TIME_FORMAT) if query and query.summary_data_updated_datetime: stats['summary_data_updated_datetime'] = query.summary_data_updated_datetime.strftime(DATE_TIME_FORMAT) if query and query.derived_cost_datetime: stats['derived_cost_datetime'] = query.derived_cost_datetime.strftime(DATE_TIME_FORMAT) return stats
def test_execute_query_with_and_filter_project(self): """Test the filter[and:] param in the view for project.""" url = reverse("gcp-tags") client = APIClient() with tenant_context(self.tenant): subs = (GCPCostEntryLineItemDailySummary.objects.filter( usage_start__gte=self.ten_days_ago).values( "project_id").distinct()) project_id = [sub.get("project_id") for sub in subs] params = { "filter[resolution]": "daily", "filter[time_scope_value]": "-10", "filter[time_scope_units]": "day", "filter[and:project]": project_id, } url = url + "?" + urlencode(params, quote_via=quote_plus) response = client.get(url, **self.headers) self.assertEqual(response.status_code, status.HTTP_200_OK) data = response.json().get("data") self.assertEqual(data, [])
def clean_tenant_principals(tenant): """Check if all the principals in the tenant exist, remove non-existent principals.""" with tenant_context(tenant): removed_principals = [] principals = list(Principal.objects.all()) logger.info("Running clean up on %d principals for tenant %s.", len(principals), tenant.schema_name) for principal in principals: if principal.cross_account: continue logger.debug("Checking for username %s for tenant %s.", principal.username, tenant.schema_name) resp = proxy.request_filtered_principals([principal.username]) status_code = resp.get("status_code") data = resp.get("data") if status_code == status.HTTP_200_OK and data: logger.debug( "Username %s found for tenant %s, no change needed.", principal.username, tenant.schema_name) elif status_code == status.HTTP_200_OK and not data: removed_principals.append(principal.username) principal.delete() logger.info( "Username %s not found for tenant %s, principal removed.", principal.username, tenant.schema_name) else: logger.warn( "Unknown status %d when checking username %s" " for tenant %s, no change needed.", status_code, principal.username, tenant.schema_name, ) logger.info( "Completed clean up of %d principals for tenant %s, %d removed.", len(principals), tenant.schema_name, len(removed_principals), )
def test_execute_query_with_and_filter(self): """Test the filter[and:] param in the view.""" url = reverse('openshift-tags') client = APIClient() with tenant_context(self.tenant): projects = OCPUsageLineItemDailySummary.objects\ .filter(usage_start__gte=self.ten_days_ago)\ .values('namespace').distinct() projects = [project.get('namespace') for project in projects] params = { 'filter[resolution]': 'daily', 'filter[time_scope_value]': '-10', 'filter[time_scope_units]': 'day', 'filter[and:project]': projects } url = url + '?' + urlencode(params, quote_via=quote_plus) response = client.get(url, **self.headers) self.assertEqual(response.status_code, status.HTTP_200_OK) response_data = response.json() self.assertEqual(response_data.get('data', []), [])
def test_execute_query_ocp_aws_storage_group_by_node(self): """Test that grouping by node filters data.""" with tenant_context(self.tenant): # Force Django to do GROUP BY to get nodes nodes = (OCPAWSCostLineItemDailySummary.objects.values( *["node"]).filter(usage_start__gte=self.ten_days_ago).filter( product_family__contains="Storage").values( *["node"]).annotate(node_count=Count("node")).all()) node_of_interest = nodes[0].get("node") url = reverse("reports-openshift-aws-storage") client = APIClient() params = {"group_by[node]": node_of_interest} url = url + "?" + urlencode(params, quote_via=quote_plus) response = client.get(url, **self.headers) self.assertEqual(response.status_code, status.HTTP_200_OK) data = response.json() for entry in data.get("data", []): for node in entry.get("nodes", []): self.assertEqual(node.get("node"), node_of_interest)
def predict(self): """Define ORM query to run forecast and return prediction.""" cost_predictions = {} with tenant_context(self.params.tenant): data = (self.cost_summary_table.objects.filter( self.filters.compose()).order_by("usage_start").values( "usage_start").annotate( total_cost=self.total_cost_term, supplementary_cost=self.supplementary_cost_term, infrastructure_cost=self.infrastructure_cost_term, )) for fieldname in [ "total_cost", "infrastructure_cost", "supplementary_cost" ]: uniq_data = self._uniquify_qset(data.values( "usage_start", fieldname), field=fieldname) cost_predictions[fieldname] = self._predict(uniq_data) cost_predictions = self._key_results_by_date(cost_predictions) return self.format_result(cost_predictions)
def get_tag_keys(self, filters=True): """Get a list of tag keys to validate filters.""" type_filter = self.parameter_filter.get('type') tag_keys = [] with tenant_context(self.tenant): for source in self.data_sources: tag_keys_query = source.get('db_table').objects if filters is True: tag_keys_query = tag_keys_query.filter(self.query_filter) if type_filter and type_filter != source.get('type'): continue tag_keys_query = tag_keys_query.annotate(tag_keys=JSONBObjectKeys(source.get('db_column')))\ .values('tag_keys')\ .distinct()\ .all() tag_keys_query = [tag.get('tag_keys') for tag in tag_keys_query] for tag_key in tag_keys_query: tag_keys.append(tag_key) return list(set(tag_keys))
def test_create_storage_no_tiers_rate(self): """Test creating a non tiered storage rate.""" storage_rates = (Rate.METRIC_STORAGE_GB_REQUEST_MONTH, Rate.METRIC_STORAGE_GB_USAGE_MONTH) for storage_rate in storage_rates: rate = { 'provider_uuid': self.provider.uuid, 'metric': storage_rate, 'tiered_rate': [{ 'unit': 'USD', 'value': 0.22 }] } with tenant_context(self.tenant): instance = None serializer = RateSerializer(data=rate) if serializer.is_valid(raise_exception=True): instance = serializer.save() self.assertIsNotNone(instance) self.assertIsNotNone(instance.uuid)
def create_or_update_superuser(tenant, old_email, email): with tenant_context(tenant): USER_MODEL = get_user_model() # Create a superuser for the given tenant # we set a dummy password for now ... # we should also send an activation mail, as we set no password try: u = USER_MODEL.objects.get(email=old_email) u.is_superuser = True u.email = email u.save() except USER_MODEL.DoesNotExist: try: u = USER_MODEL.objects.get(email=email) u.is_superuser = True u.email = email u.save() except USER_MODEL.DoesNotExist: USER_MODEL.objects.create_superuser(email=email, password="******")
def copy_custom_groups_to_public(self, tenant): """Copy custom groups from provided tenant to the public schema.""" groups = Group.objects.filter(system=False) public_schema = Tenant.objects.get(schema_name="public") for group in groups: self.stdout.write(f"Copying group {group.name} to public schema for tenant {tenant}.") if not group.tenant: group.tenant = tenant group.save() principals = list(group.principals.all()) new_principals = [] with tenant_context(public_schema): self.clear_pk(group) try: group.save() except IntegrityError as err: self.stderr.write(f"Couldn't copy group {group.name}. Skipping due to:\n{err}") continue for principal in principals: new_principals.append(Principal.objects.get(username=principal.username, tenant=tenant)) group.principals.set(new_principals) group.save()
def get_tag_keys(self, filters=True): """Get a list of tag keys to validate filters.""" type_filter = self.parameters.get_filter("type") tag_keys = set() with tenant_context(self.tenant): for source in self.data_sources: tag_keys_query = source.get("db_table").objects annotations = source.get("annotations") if annotations: tag_keys_query = tag_keys_query.annotate(**annotations) if filters is True: tag_keys_query = tag_keys_query.filter(self.query_filter) if type_filter and type_filter != source.get("type"): continue exclusion = self._get_exclusions("key") tag_keys_query = tag_keys_query.exclude(exclusion).values( "key").distinct().all() tag_keys.update({tag.get("key") for tag in tag_keys_query}) return list(tag_keys)
def test_execute_query_ocp_aws_storage_with_group_by_tag_and_limit(self): """Test that data is grouped by tag key and limited.""" with tenant_context(self.tenant): labels = OCPAWSCostLineItemDailySummary.objects\ .filter(usage_start__gte=self.dh.last_month_start)\ .filter(usage_start__lte=self.dh.last_month_end)\ .filter(product_family__contains='Storage')\ .values(*['tags'])\ .first() tags = labels.get('tags') group_by_key = list(tags.keys())[0] plural_key = group_by_key + 's' url = reverse('reports-openshift-aws-storage') client = APIClient() params = { 'filter[resolution]': 'monthly', 'filter[time_scope_value]': '-2', 'filter[time_scope_units]': 'month', f'group_by[tag:{group_by_key}]': '*', 'filter[limit]': 2 } url = url + '?' + urlencode(params, quote_via=quote_plus) response = client.get(url, **self.headers) self.assertEqual(response.status_code, status.HTTP_200_OK) data = response.json() data = data.get('data', []) # default ordered by usage previous_tag_usage = data[0].get(plural_key, [])[0].get( 'values', [{}])[0].get('usage', {}).get('value') for entry in data[0].get(plural_key, []): current_tag_usage = entry.get('values', [{}])[0].get('usage', {}).get('value') if 'Other' not in entry.get(group_by_key): self.assertTrue(current_tag_usage <= previous_tag_usage) previous_tag_usage = current_tag_usage
def test_remove_ocp_added_via_sources(self): """Remove ocp provider added via sources.""" # Create Provider credentials = {"cluster_id": "cluster_id_1001"} provider_authentication = ProviderAuthentication.objects.create(credentials=credentials) with patch("masu.celery.tasks.check_report_updates"): provider = Provider.objects.create( name="ocpprovidername", created_by=self.user, customer=self.customer, authentication=provider_authentication, ) provider_uuid = provider.uuid sources = Sources.objects.create(source_id=1, auth_header="testheader", offset=1, koku_uuid=provider_uuid) sources.save() delete_request = self._create_delete_request(self.user, {"Sources-Client": "True"}) with tenant_context(self.tenant): manager = ProviderManager(provider_uuid) manager.remove(delete_request, from_sources=True) provider_query = Provider.objects.all().filter(uuid=provider_uuid) self.assertFalse(provider_query)
def test_error_neg_tier_usage_end(self): """Test error when trying to create a negative tiered usage_end.""" rate = { 'provider_uuid': self.provider.uuid, 'metric': Rate.METRIC_CPU_CORE_USAGE_HOUR, 'tiered_rate': [{ 'unit': 'USD', 'value': 1.0, 'usage_start': 10.0, 'usage_end': (round(Decimal(random.random()), 6) * -1) }] } with tenant_context(self.tenant): serializer = RateSerializer(data=rate) with self.assertRaises(serializers.ValidationError): if serializer.is_valid(raise_exception=True): serializer.save()
def connect(self, message, **kwargs): try: tenant = get_tenant_model().objects.get( domain_url=get_host(message.content) ) except get_tenant_model().DoesNotExist: logger.error( 'no tenant found for {}'.format( get_host(message.content) ) ) message.reply_channel.send({"close": True}) return message.channel_session['schema'] = tenant.schema_name super().connect(message, **kwargs) Group( '{}_latest_blocks'.format(tenant.schema_name) ).add( message.reply_channel ) with tenant_context(tenant): latest_blocks = Block.objects.exclude( height__isnull=True ).order_by( '-height' )[:20] kwargs.get('multiplexer').send( { "latest_blocks": [ block.serialize() for block in latest_blocks ] } )
def ws_receive(message): message_dict = json.loads(message['text']) domain_url = message_dict['payload']['host'] if domain_url == 'explorer.nubits.com': domain_url = 'nu.crypto-daio.co.uk' try: tenant = get_tenant_model().objects.get(domain_url=domain_url) except get_tenant_model().DoesNotExist: tenant = get_tenant_model().objects.get(domain_url='nu.crypto-test.co.uk') with tenant_context(tenant): if message['path'] == '/get_block_details/': get_block_details(message_dict, message) return if message['path'] == '/get_address_details/': try: address_object = Address.objects.get(address=message_dict.get('stream')) get_address_balance(address_object, message) get_address_details(address_object, message) except Address.DoesNotExist: pass return # if message['path'] == '/tx_browser/': # if message_dict.get('stream') == 'add_nodes': # add_onward_nodes(message_dict, message) # # if message_dict.get('stream') == 'stop_nodes': # try: # user_socket = UserSocket.objects.get( # reply_channel=message.reply_channel # ) # user_socket.tx_browser_running = False # user_socket.save() # except UserSocket.DoesNotExist: # pass # # return if message['path'] == '/get_current_grants/': get_current_grants(message) return if message['path'] == '/get_current_motions/': get_current_motions(message) return if message['path'] == '/all_blocks_list/': try: last_height = int(message_dict['payload']['last_height']) except ValueError: return get_next_blocks(message, last_height) return if message['path'] == '/latest_blocks/': get_latest_blocks(message)
def post_schema_save(sender, **kwargs): tenant = kwargs['tenant'] with tenant_context(tenant): create_superuser(tenant, tenant.email)