def ready(self): # structure from .backend import SugarCRMBackend SupportedServices.register_backend(SugarCRMBackend) # cost tracking from .cost_tracking import SugarCRMCostTrackingBackend CostTrackingRegister.register(self.label, SugarCRMCostTrackingBackend) # template from .template import CRMProvisionTemplateForm TemplateRegistry.register(CRMProvisionTemplateForm) from . import handlers CRM = self.get_model('CRM') signals.post_save.connect( handlers.update_user_limit_count_quota_on_crm_quota_change, sender=Quota, dispatch_uid='nodeconductor_sugarcrm.handlers.update_user_limit_count_quota_on_crm_quota_change', ) signals.pre_delete.connect( handlers.update_user_limit_count_quota_on_crm_deletion, sender=CRM, dispatch_uid='nodeconductor_sugarcrm.handlers.update_user_limit_count_quota_on_crm_deletion' )
def recalculate_estimate(recalculate_total=False): """ Recalculate price of consumables that were used by resource until now. Regular task. It is too expensive to calculate consumed price on each request, so we store cached price each hour. If recalculate_total is True - task also recalculates total estimate for current month. """ # Celery does not import server.urls and does not discover cost tracking modules. # So they should be discovered implicitly. CostTrackingRegister.autodiscover() # Step 1. Recalculate resources estimates. for resource_model in CostTrackingRegister.registered_resources: for resource in resource_model.objects.all(): _update_resource_consumed(resource, recalculate_total=recalculate_total) # Step 2. Move from down to top and recalculate consumed estimate for each # object based on its children. ancestors_models = [ m for m in models.PriceEstimate.get_estimated_models() if not issubclass(m, structure_models.ResourceMixin) ] for model in ancestors_models: for ancestor in model.objects.all(): _update_ancestor_consumed(ancestor)
def ready(self): from .cost_tracking import SaltStackCostTrackingBackend CostTrackingRegister.register(self.label, SaltStackCostTrackingBackend) # template from .template import SharepointTenantTemplateForm TemplateRegistry.register(SharepointTenantTemplateForm) # import it here in order to register as SaltStack backend from .backend import SharepointBackend from . import handlers SharepointTenant = self.get_model('SharepointTenant') User = self.get_model('User') SiteCollection = self.get_model('SiteCollection') signals.post_save.connect( handlers.increase_quotas_usage_on_tenant_creation, sender=SharepointTenant, dispatch_uid='nodeconductor.saltstack.sharepoint.handlers.increase_quotas_usage_on_tenant_creation', ) signals.pre_delete.connect( handlers.decrease_quotas_usage_on_tenant_deletion, sender=SharepointTenant, dispatch_uid='nodeconductor.saltstack.sharepoint.handlers.decrease_quotas_usage_on_tenant_deletion', )
def setUp(self): CostTrackingRegister.register_strategy( factories.TestNewInstanceCostTrackingStrategy) resource_content_type = ContentType.objects.get_for_model( TestNewInstance) self.price_list_item = models.DefaultPriceListItem.objects.create( item_type='storage', key='1 MB', value=0.5, resource_content_type=resource_content_type)
def ready(self): OpenStackService = self.get_model('OpenStackService') OpenStackServiceProjectLink = self.get_model('OpenStackServiceProjectLink') Instance = self.get_model('Instance') FloatingIP = self.get_model('FloatingIP') # structure from nodeconductor.openstack.backend import OpenStackBackend SupportedServices.register_backend(OpenStackService, OpenStackBackend) # cost tracking from nodeconductor.openstack.cost_tracking import OpenStackCostTrackingBackend CostTrackingRegister.register(self.label, OpenStackCostTrackingBackend) # template from nodeconductor.template import TemplateRegistry from nodeconductor.openstack.template import InstanceProvisionTemplateForm TemplateRegistry.register(InstanceProvisionTemplateForm) signals.post_save.connect( handlers.create_initial_security_groups, sender=OpenStackServiceProjectLink, dispatch_uid='nodeconductor.openstack.handlers.create_initial_security_groups', ) signals.post_save.connect( quotas_handlers.add_quotas_to_scope, sender=OpenStackServiceProjectLink, dispatch_uid='nodeconductor.openstack.handlers.add_quotas_to_service_project_link', ) signals.pre_save.connect( handlers.set_spl_default_availability_zone, sender=OpenStackServiceProjectLink, dispatch_uid='nodeconductor.openstack.handlers.set_spl_default_availability_zone', ) signals.post_save.connect( handlers.increase_quotas_usage_on_instance_creation, sender=Instance, dispatch_uid='nodeconductor.openstack.handlers.increase_quotas_usage_on_instance_creation', ) signals.post_delete.connect( handlers.decrease_quotas_usage_on_instances_deletion, sender=Instance, dispatch_uid='nodeconductor.openstack.handlers.decrease_quotas_usage_on_instances_deletion', ) signals.post_save.connect( handlers.change_floating_ip_quota_on_status_change, sender=FloatingIP, dispatch_uid='nodeconductor.openstack.handlers.change_floating_ip_quota_on_status_change', )
def init_from_registered_applications(self, request): created_items = [] for backend in CostTrackingRegister.get_registered_backends(): try: items = backend.get_default_price_list_items() except NotImplementedError: continue with transaction.atomic(): for item in items: item, created = models.DefaultPriceListItem.objects.update_or_create( resource_content_type=item.resource_content_type, item_type=item.item_type, key=item.key, defaults={ 'value': item.value, 'name': '{}: {}'.format(item.item_type, item.key), 'metadata': item.metadata, 'units': item.units }) if created: created_items.append(item) if created_items: message = ungettext( 'Price item was created: {}'.format(created_items[0].name), 'Price items were created: {}'.format(', '.join( item.name for item in created_items)), len(created_items)) self.message_user(request, message) else: self.message_user( request, "Price items for all registered applications have been updated" ) return redirect( reverse('admin:cost_tracking_defaultpricelistitem_changelist'))
def delete_not_registered(self, request): deleted_items_names = [] for price_list_item in models.DefaultPriceListItem.objects.all(): try: resource_class = price_list_item.resource_content_type.model_class( ) consumable_items = CostTrackingRegister.get_consumable_items( resource_class) next(item for item in consumable_items if item.key == price_list_item.key and item.item_type == price_list_item.item_type) except (ResourceNotRegisteredError, StopIteration): deleted_items_names.append(price_list_item.name) price_list_item.delete() if deleted_items_names: message = ungettext( _('Price item was deleted: %s.') % deleted_items_names[0], _('Price items were deleted: %s.') % ', '.join(item for item in deleted_items_names), len(deleted_items_names)) self.message_user(request, message) else: self.message_user( request, _('Nothing to delete. All default price items are registered.') ) return redirect( reverse('admin:cost_tracking_defaultpricelistitem_changelist'))
def init_from_registered_applications(self, request): created_items = [] for backend in CostTrackingRegister.get_registered_backends(): try: items = backend.get_default_price_list_items() except NotImplementedError: continue for item in items: if not models.DefaultPriceListItem.objects.filter(resource_content_type=item.resource_content_type, item_type=item.item_type, key=item.key).exists(): if not item.name: item.name = '{}: {}'.format(item.item_type, item.key) item.save() created_items.append(item) if created_items: message = ungettext( 'Price item was created: {}'.format(created_items[0].name), 'Price items were created: {}'.format(', '.join(item.name for item in created_items)), len(created_items) ) self.message_user(request, message) else: self.message_user(request, "Price items exist for all registered applications") return redirect(reverse('admin:cost_tracking_defaultpricelistitem_changelist'))
def setUp(self): resource_content_type = ContentType.objects.get_for_model( TestNewInstance) self.price_list_item = models.DefaultPriceListItem.objects.create( item_type='storage', key='1 MB', resource_content_type=resource_content_type, value=2) CostTrackingRegister.register_strategy( factories.TestNewInstanceCostTrackingStrategy) self.start_time = datetime.datetime(2016, 8, 8, 11, 0) with freeze_time(self.start_time): self.resource = structure_factories.TestNewInstanceFactory( disk=20 * 1024) self.spl = self.resource.service_project_link self.project = self.spl.project self.customer = self.project.customer self.service = self.spl.service
def update_price_for_resource(cls, resource, delete=False): try: cost_tracking_backend = CostTrackingRegister.get_resource_backend( resource) monthly_cost = float( cost_tracking_backend.get_monthly_cost_estimate(resource)) except ServiceBackendNotImplemented: return except ServiceBackendError as e: logger.error("Failed to get cost estimate for resource %s: %s", resource, e) except Exception as e: logger.exception("Failed to get cost estimate for resource %s: %s", resource, e) else: logger.info("Update cost estimate for resource %s: %s", resource, monthly_cost) now = timezone.now() created = resource.created days_in_month = calendar.monthrange(created.year, created.month)[1] month_start = created.replace(day=1, hour=0, minute=0, second=0) month_end = month_start + timezone.timedelta(days=days_in_month) seconds_in_month = (month_end - month_start).total_seconds() seconds_of_work = (month_end - created).total_seconds() creation_month_cost = round( monthly_cost * seconds_of_work / seconds_in_month, 2) update = functools.partial(cls.update_price_for_scope, resource) if delete: monthly_cost *= -1 creation_month_cost *= -1 if created.month == now.month and created.year == now.year: # update only current month estimate update(now.month, now.year, creation_month_cost) else: # update current month estimate update(now.month, now.year, monthly_cost) # update first month estimate update(created.month, created.year, creation_month_cost, update_if_exists=False) # update price estimate for previous months if it does not exist: date = now - relativedelta(months=+1) while not (date.month == created.month and date.year == created.year): update(date.month, date.year, monthly_cost, update_if_exists=False) date -= relativedelta(months=+1)
def ready(self): from .cost_tracking import SaltStackCostTrackingBackend CostTrackingRegister.register(self.label, SaltStackCostTrackingBackend) # template from .template import TenantProvisionTemplateForm TemplateRegistry.register(TenantProvisionTemplateForm) # import it here in order to register as SaltStack backend from .backend import ExchangeBackend from . import handlers ExchangeTenant = self.get_model('ExchangeTenant') User = self.get_model('User') # Tenants CRUD signals.post_save.connect( handlers.increase_exchange_storage_usage_on_tenant_creation, sender=ExchangeTenant, dispatch_uid='nodeconductor.saltstack.exchange.handlers.increase_exchange_storage_usage_on_tenant_creation', ) signals.post_delete.connect( handlers.decrease_exchange_storage_usage_on_tenant_deletion, sender=ExchangeTenant, dispatch_uid='nodeconductor.saltstack.exchange.handlers.decrease_exchange_storage_usage_on_tenant_deletion', ) # Users CRUD signals.post_save.connect( handlers.increase_global_mailbox_size_usage_on_user_creation_or_modification, sender=User, dispatch_uid=('nodeconductor.saltstack.exchange.handlers' '.increase_global_mailbox_size_usage_on_user_creation_or_modification'), ) signals.post_delete.connect( handlers.decrease_global_mailbox_size_usage_on_user_deletion, sender=User, dispatch_uid='nodeconductor.saltstack.exchange.handlers.decrease_global_mailbox_size_usage_on_user_deletion', )
def update_projected_estimate(customer_uuid=None, resource_uuid=None): if customer_uuid and resource_uuid: raise RuntimeError("Either customer_uuid or resource_uuid could be supplied, both received.") def get_resource_creation_month_cost(resource, monthly_cost): month_start = resource.created.replace(day=1, hour=0, minute=0, second=0) month_end = month_start.replace(month=month_start.month+1) seconds_in_month = (month_end - month_start).total_seconds() seconds_of_work = (month_end - resource.created).total_seconds() return round(monthly_cost * seconds_of_work / seconds_in_month, 2) for model in Resource.get_all_models(): queryset = model.objects.exclude(state=model.States.ERRED) if customer_uuid: queryset = queryset.filter(customer__uuid=customer_uuid) elif resource_uuid: queryset = queryset.filter(uuid=resource_uuid) for instance in queryset.iterator(): try: cost_tracking_backend = CostTrackingRegister.get_resource_backend(instance) if not cost_tracking_backend: continue monthly_cost = float(cost_tracking_backend.get_monthly_cost_estimate(instance)) except ServiceBackendNotImplemented as e: continue except ServiceBackendError as e: logger.error("Failed to get cost estimate for resource %s: %s", instance, e) except Exception as e: logger.exception("Failed to get cost estimate for resource %s: %s", instance, e) else: logger.info("Update cost estimate for resource %s: %s", instance, monthly_cost) creation_month_cost = get_resource_creation_month_cost(instance, monthly_cost) now = timezone.now() created = instance.created if created.month == now.month and created.year == now.year: # update only current month estimate PriceEstimate.update_price_for_scope(instance, now.month, now.year, creation_month_cost) else: # update current month estimate PriceEstimate.update_price_for_scope(instance, now.month, now.year, monthly_cost) # update first month estimate PriceEstimate.update_price_for_scope(instance, created.month, created.year, creation_month_cost, update_if_exists=False) # update price estimate for previous months if it does not exist: date = now - relativedelta(months=+1) while not (date.month == created.month and date.year == created.year): PriceEstimate.update_price_for_scope(instance, date.month, date.year, monthly_cost, update_if_exists=False) date -= relativedelta(months=+1)
def reinit_configurations(self, request): """ Re-initialize configuration for resource if it has been changed. This method should be called if resource consumption strategy was changed. """ now = timezone.now() # Step 1. Collect all resources with changed configuration. changed_resources = [] for resource_model in CostTrackingRegister.registered_resources: for resource in resource_model.objects.all(): try: pe = models.PriceEstimate.objects.get(scope=resource, month=now.month, year=now.year) except models.PriceEstimate.DoesNotExist: changed_resources.append(resource) else: new_configuration = CostTrackingRegister.get_configuration( resource) if new_configuration != pe.consumption_details.configuration: changed_resources.append(resource) # Step 2. Re-init configuration and recalculate estimate for changed resources. for resource in changed_resources: models.PriceEstimate.update_resource_estimate( resource, CostTrackingRegister.get_configuration(resource)) message = _( 'Configuration was reinitialized for %(count)s resources') % { 'count': len(changed_resources) } self.message_user(request, message) return redirect( reverse('admin:cost_tracking_defaultpricelistitem_changelist'))
def update_today_usage_of_resource(resource_str): # XXX: this method does ignores cases then VM was offline or online for small periods of time. # It could to be rewritten if more accurate calculation will be needed with transaction.atomic(): resource = next(Resource.from_string(resource_str)) cs_backend = CostTrackingRegister.get_resource_backend(resource) used_items = cs_backend.get_used_items(resource) if not resource.billing_backend_id: logger.warning( "Can't update usage for resource %s which is not subscribed to backend", resource_str) return numerical = ['storage', 'users' ] # XXX: use consistent method for usage calculation content_type = ContentType.objects.get_for_model(resource) units = {(item.item_type, None if item.item_type in numerical else item.key): item.units for item in DefaultPriceListItem.objects.filter( resource_content_type=content_type)} now = timezone.now() last_update_time = resource.last_usage_update_time or resource.created minutes_from_last_usage_update = ( now - last_update_time).total_seconds() / 60 usage = {} for item_type, key, val in used_items: if val: try: unit = units[item_type, None if item_type in numerical else key] usage_per_min = int( round(val * minutes_from_last_usage_update)) if usage_per_min: usage[unit] = usage_per_min except KeyError: logger.error("Can't find price for usage item %s:%s", key, val) kb_backend = KillBillBackend() kb_backend.add_usage_data(resource, usage) resource.last_usage_update_time = timezone.now() resource.save(update_fields=['last_usage_update_time'])
def handle(self, *args, **options): today = timezone.now() with transaction.atomic(): # Delete current month price estimates models.PriceEstimate.objects.filter(month=today.month, year=today.year).delete() # Create new estimates for resources and ancestors for resource_model in CostTrackingRegister.registered_resources: for resource in resource_model.objects.all(): configuration = CostTrackingRegister.get_configuration( resource) date = max(core_utils.month_start(today), resource.created) models.PriceEstimate.create_historical( resource, configuration, date) # recalculate consumed estimate tasks.recalculate_estimate()
class SharepointTenantStrategy(CostTrackingStrategy): resource_class = models.SharepointTenant class Types(object): SUPPORT = 'support' STORAGE = 'storage' class Keys(object): STORAGE = '1 GB' SUPPORT = 'premium' @classmethod def get_consumable_items(cls): return [ ConsumableItem(item_type=cls.Types.STORAGE, key=cls.Keys.STORAGE, name='1 GB of storage', units='GB'), ConsumableItem(item_type=cls.Types.SUPPORT, key=cls.Keys.SUPPORT, name='Support: premium'), ] @classmethod def get_configuration(cls, tenant): storage = tenant.quotas.get(name=models.SharepointTenant.Quotas.mailbox_size).usage return { ConsumableItem(item_type=cls.Types.STORAGE, key=cls.Keys.STORAGE): float(storage) / 1024, ConsumableItem(item_type=cls.Types.SUPPORT, key=cls.Keys.SUPPORT): 1, } CostTrackingRegister.register_strategy(SharepointTenantStrategy)
logger = logging.getLogger(__name__) class TenantStrategy(CostTrackingStrategy): resource_class = openstack_models.Tenant @classmethod def get_consumable_items(cls): for package_template in models.PackageTemplate.objects.all(): yield utils.get_consumable_item(package_template) @classmethod def get_configuration(cls, tenant): configuration = {} if tenant.state != tenant.States.ERRED: if 'package_name' not in tenant.extra_configuration: logger.warning( 'Package name is not defined in configuration of tenant %s, (PK: %s)', tenant.name, tenant.pk) else: package_name = tenant.extra_configuration['package_name'] configuration = { ConsumableItem(item_type=utils.Types.PACKAGE_TEMPLATE, key=package_name): 1, } return configuration CostTrackingRegister.register_strategy(TenantStrategy)
SUPPORT = 'support' USERS = 'users' class Keys(object): SUPPORT = 'premium' USERS = 'count' @classmethod def get_consumable_items(cls): return [ ConsumableItem(item_type=cls.Types.USERS, key=cls.Keys.USERS, name='Users count'), ConsumableItem(item_type=cls.Types.SUPPORT, key=cls.Keys.SUPPORT, name='Support: premium'), ] @classmethod def get_configuration(cls, crm): user_count = crm.quotas.get(name=crm.Quotas.user_count).limit return { ConsumableItem(item_type=cls.Types.USERS, key=cls.Keys.USERS): user_count, ConsumableItem(item_type=cls.Types.SUPPORT, key=cls.Keys.SUPPORT): 1, } CostTrackingRegister.register_strategy(CRMStrategy)
from django.conf.urls import include, url from django.contrib import admin from django.views.generic import TemplateView from nodeconductor.core import NodeConductorExtension from nodeconductor.core import views as core_views from nodeconductor.core.routers import SortedDefaultRouter as DefaultRouter from nodeconductor.core.schemas import WaldurSchemaView from nodeconductor.cost_tracking import urls as cost_tracking_urls, CostTrackingRegister from nodeconductor.logging import urls as logging_urls from nodeconductor.monitoring import urls as monitoring_urls from nodeconductor.quotas import urls as quotas_urls from nodeconductor.structure import urls as structure_urls from nodeconductor.users import urls as users_urls CostTrackingRegister.autodiscover() router = DefaultRouter() cost_tracking_urls.register_in(router) logging_urls.register_in(router) monitoring_urls.register_in(router) quotas_urls.register_in(router) structure_urls.register_in(router) users_urls.register_in(router) urlpatterns = [ url(r'^admin/', admin.site.urls), url(r'^admintools/', include('admin_tools.urls')), ] if settings.NODECONDUCTOR.get('EXTENSIONS_AUTOREGISTER'):
def ready(self): Instance = self.get_model('Instance') Cloud = self.get_model('Cloud') CloudProjectMembership = self.get_model('CloudProjectMembership') from nodeconductor.iaas import handlers, cost_tracking from nodeconductor.structure.serializers import CustomerSerializer, ProjectSerializer CostTrackingRegister.register(self.label, cost_tracking.IaaSCostTrackingBackend) pre_serializer_fields.connect( handlers.add_clouds_to_related_model, sender=CustomerSerializer, dispatch_uid='nodeconductor.iaas.handlers.add_clouds_to_customer', ) pre_serializer_fields.connect( handlers.add_clouds_to_related_model, sender=ProjectSerializer, dispatch_uid='nodeconductor.iaas.handlers.add_clouds_to_project', ) signals.post_save.connect( quotas_handlers.add_quotas_to_scope, sender=CloudProjectMembership, dispatch_uid= 'nodeconductor.iaas.handlers.add_quotas_to_cloud_project_membership', ) signals.post_save.connect( handlers.create_initial_security_groups, sender=CloudProjectMembership, dispatch_uid= 'nodeconductor.iaas.handlers.create_initial_security_groups', ) # protect against a deletion of the Instance with connected backups # TODO: introduces dependency of IaaS on Backups, should be reconsidered signals.pre_delete.connect( handlers.prevent_deletion_of_instances_with_connected_backups, sender=Instance, dispatch_uid= 'nodeconductor.iaas.handlers.prevent_deletion_of_instances_with_connected_backups', ) signals.pre_save.connect( core_handlers.preserve_fields_before_update, sender=Instance, dispatch_uid= 'nodeconductor.iaas.handlers.preserve_fields_before_update', ) # if instance name is updated, zabbix host visible name should be also updated signals.post_save.connect( handlers.check_instance_name_update, sender=Instance, dispatch_uid= 'nodeconductor.iaas.handlers.check_instance_name_update', ) signals.pre_save.connect( handlers.set_cpm_default_availability_zone, sender=CloudProjectMembership, dispatch_uid= 'nodeconductor.iaas.handlers.set_cpm_default_availability_zone', ) signals.post_save.connect( handlers.increase_quotas_usage_on_instance_creation, sender=Instance, dispatch_uid= 'nodeconductor.iaas.handlers.increase_quotas_usage_on_instance_creation', ) signals.post_delete.connect( handlers.decrease_quotas_usage_on_instances_deletion, sender=Instance, dispatch_uid= 'nodeconductor.iaas.handlers.decrease_quotas_usage_on_instances_deletion', ) signals.post_save.connect( handlers.change_customer_nc_service_quota, sender=Cloud, dispatch_uid= 'nodeconductor.iaas.handlers.increase_customer_nc_service_quota') signals.post_delete.connect( handlers.change_customer_nc_service_quota, sender=Cloud, dispatch_uid= 'nodeconductor.iaas.handlers.decrease_customer_nc_service_quota') signals.post_save.connect( handlers.check_project_name_update, sender=Project, dispatch_uid='nodeconductor.iaas.handlers.check_project_name_update' )
def setUp(self): CostTrackingRegister.register_strategy( factories.TestNewInstanceCostTrackingStrategy)
class Types(object): TYPE = 'type' FLAVOR = 'flavor' STORAGE = 'storage' @classmethod def get_consumable_items(cls): # flavor for flavor in models.Flavor.objects.all(): yield ConsumableItem(item_type=cls.Types.FLAVOR, key=flavor.name, name='Flavor: %s' % flavor.name) # type for v, _ in models.Deployment.Version.CHOICES: for t, _ in models.Deployment.Type.CHOICES: d = models.Deployment(db_type=t, db_version=v) key = slugify(d.db_version_type) yield ConsumableItem(item_type=cls.Types.TYPE, key=key) # storage yield ConsumableItem(item_type=cls.Types.STORAGE, key='1 GB') @classmethod def get_configuration(cls, deployment): return { ConsumableItem(item_type=cls.Types.TYPE, key=slugify(deployment.db_version_type)): 1, ConsumableItem(item_type=cls.Types.FLAVOR, key=deployment.flavor.name): 1, ConsumableItem(item_type=cls.Types.STORAGE, key='1 GB'): deployment.db_size, } CostTrackingRegister.register_strategy(DeploymentStrategy)