def import_openstack_instances_and_volumes(dry_run=False):
"""
Import OpenStack tenant resources as marketplace resources.
It is expected that offerings for OpenStack tenant service settings are imported before this command is ran.
"""
model_classes = {
INSTANCE_TYPE: openstack_tenant_models.Instance,
VOLUME_TYPE: openstack_tenant_models.Volume,
}
resources_counter = 0
for offering_type in (INSTANCE_TYPE, VOLUME_TYPE):
front_ids = set(
marketplace_models.Resource.objects.filter(
offering__type=offering_type).values_list('object_id',
flat=True))
model_class = model_classes[offering_type]
missing_resources = model_class.objects.exclude(id__in=front_ids)
if dry_run:
ids = format_list(missing_resources)
logger.warning(
'OpenStack resource with IDs would be imported to marketplace: %s.',
ids)
continue
offerings = {
offering.scope: offering
for offering in marketplace_models.Offering.objects.filter(
type=offering_type)
}
for resource in missing_resources:
offering = offerings.get(resource.service_settings)
if not offering:
logger.warning(
'Offering for service setting with ID %s is not imported yet.',
resource.service_settings.id,
)
continue
plan = get_plan_for_resource(resource, offering)
new_resource = marketplace_models.Resource.objects.create(
name=resource.name,
created=resource.created,
project=resource.project,
offering=offering,
plan=plan,
scope=resource,
state=get_resource_state(resource.state),
attributes=dict(
name=resource.name,
description=resource.description,
),
)
if isinstance(resource, openstack_tenant_models.Volume):
import_volume_metadata(new_resource)
if isinstance(resource, openstack_tenant_models.Instance):
import_instance_metadata(new_resource)
resources_counter += 1
return resources_counter
def import_openstack_tenant_service_settings(dry_run=False):
"""
Import OpenStack tenant service settings as marketplace offerings.
"""
offerings_counter = 0
plans_counter = 0
for offering_type in (INSTANCE_TYPE, VOLUME_TYPE):
marketplace_offerings = marketplace_models.Offering.objects.filter(
type=offering_type)
front_settings = set(
marketplace_offerings.exclude(object_id=None).values_list(
'object_id', flat=True))
missing_settings = structure_models.ServiceSettings.objects.filter(
type=openstack_tenant_apps.OpenStackTenantConfig.service_name
).exclude(id__in=front_settings)
if dry_run:
logger.warning(
'OpenStack tenant service settings would be imported to marketplace. '
'ID: %s.',
format_list(missing_settings),
)
continue
packages = package_models.OpenStackPackage.objects.filter(
service_settings__in=missing_settings)
settings_to_template = {
package.service_settings: package.template
for package in packages
}
for service_settings in missing_settings:
category, offering_name = get_category_and_name_for_offering_type(
offering_type, service_settings)
offering = marketplace_models.Offering.objects.create(
customer=service_settings.customer,
category=category,
name=offering_name,
scope=service_settings,
shared=service_settings.shared,
type=offering_type,
state=marketplace_models.Offering.States.ACTIVE,
billable=False,
)
create_offering_components(offering)
offerings_counter += 1
template = settings_to_template.get(service_settings)
if not template:
logger.warning(
'Billing for service setting is not imported because it does not have template. '
'Service setting ID: %s',
service_settings.id,
)
continue
try:
parent_plan = marketplace_models.Plan.objects.get(
scope=template, offering__type=PACKAGE_TYPE)
except marketplace_models.Plan.DoesNotExist:
logger.warning(
'Billing for template is not imported because it does not have plan. '
'Template ID: %s',
template.id,
)
continue
plan = marketplace_models.Plan.objects.create(
offering=offering,
name=parent_plan.name,
scope=parent_plan.scope)
copy_plan_components_from_template(plan, offering, template)
plans_counter += 1
return offerings_counter, plans_counter
def import_openstack_tenants(dry_run=False):
"""
Import OpenStack tenants as marketplace resources.
It is expected that offerings for OpenStack service settings are imported before this command is ran.
"""
front_ids = set(
marketplace_models.Resource.objects.filter(
offering__type=PACKAGE_TYPE).values_list('object_id', flat=True))
missing_resources = openstack_models.Tenant.objects.exclude(
id__in=front_ids)
if dry_run:
logger.warning(
'OpenStack tenants would be imported to marketplace. '
'ID: %s.',
format_list(missing_resources),
)
return 0
packages = package_models.OpenStackPackage.objects.filter(
tenant__in=missing_resources)
tenants_without_packages = missing_resources.exclude(
id__in=packages.values_list('tenant_id', flat=True))
def create_resource(offering, tenant, plan=None):
resource = marketplace_models.Resource.objects.create(
name=tenant.name,
created=tenant.created,
offering=offering,
plan=plan,
scope=tenant,
project=tenant.project,
state=get_resource_state(tenant.state),
attributes=dict(
name=tenant.name,
description=tenant.description,
user_username=tenant.user_username,
user_password=tenant.user_password,
),
)
if plan and tenant.backend_id:
marketplace_models.ResourcePlanPeriod.objects.create(
resource=resource,
plan=plan,
start=tenant.created,
)
import_resource_metadata(resource)
return resource
resource_counter = 0
for tenant in tenants_without_packages:
# It is expected that service setting has exactly one offering
# if it does not have package
try:
offering = marketplace_models.Offering.objects.get(
scope=tenant.service_settings)
except marketplace_models.Offering.DoesNotExist:
logger.warning(
'Offering for service setting is not imported yet. '
'Service setting ID: %s.',
tenant.service_settings.id,
)
continue
create_resource(offering, tenant)
resource_counter += 1
for package in packages:
tenant = package.tenant
try:
offering = marketplace_models.Offering.objects.get(
scope=tenant.service_settings)
plan = marketplace_models.Plan.objects.get(scope=package.template,
offering=offering)
except marketplace_models.Plan.DoesNotExist:
logger.warning(
'Plan for template is not imported yet. '
'Template ID: %s.',
package.template_id,
)
continue
create_resource(plan.offering, tenant, plan)
resource_counter += 1
return resource_counter
def import_openstack_service_settings(default_customer,
dry_run=False,
require_templates=False):
"""
Import OpenStack service settings as marketplace offerings.
"""
service_type = openstack_apps.OpenStackConfig.service_name
category = get_offering_category_for_tenant()
package_offerings = marketplace_models.Offering.objects.filter(
type=PACKAGE_TYPE)
front_settings = set(
package_offerings.exclude(object_id=None).values_list('object_id',
flat=True))
back_settings = structure_models.ServiceSettings.objects.filter(
type=service_type)
missing_settings = back_settings.exclude(id__in=front_settings)
if dry_run:
logger.warning(
'OpenStack service settings would be imported to marketplace. '
'ID: %s.',
format_list(missing_settings),
)
return 0, 0
missing_templates = package_models.PackageTemplate.objects.filter(
service_settings__in=missing_settings)
settings_without_templates = missing_settings.exclude(
id__in=missing_templates.values_list('service_settings_id', flat=True))
def create_offering(service_settings, state):
offering = marketplace_models.Offering.objects.create(
scope=service_settings,
type=PACKAGE_TYPE,
name=service_settings.name,
geolocations=service_settings.geolocations,
customer=service_settings.customer or default_customer,
category=category,
shared=service_settings.shared,
state=state,
)
create_offering_components(offering)
return offering
offerings_counter = 0
plans_counter = 0
if settings_without_templates.exists():
logger.warning(
'The following service settings do not have package template, '
'therefore they would be imported in DRAFT state: %s',
format_list(settings_without_templates),
)
if not require_templates:
for service_settings in settings_without_templates:
with transaction.atomic():
create_offering(service_settings,
marketplace_models.Offering.States.DRAFT)
offerings_counter += 1
for template in missing_templates:
with transaction.atomic():
service_settings = template.service_settings
try:
offering = marketplace_models.Offering.objects.get(
scope=service_settings)
except marketplace_models.Offering.DoesNotExist:
offering = create_offering(
service_settings,
marketplace_models.Offering.States.ACTIVE)
offerings_counter += 1
plan = marketplace_models.Plan.objects.create(
offering=offering,
name=template.name,
unit_price=template.price,
unit=marketplace_models.Plan.Units.PER_DAY,
product_code=template.product_code,
article_code=template.article_code,
scope=template,
)
plans_counter += 1
copy_plan_components_from_template(plan, offering, template)
return offerings_counter, plans_counter
def import_allocation(dry_run=False):
ct = ContentType.objects.get_for_model(slurm_models.Allocation)
exist_ids = marketplace_models.Resource.objects.filter(content_type=ct).values_list(
'object_id', flat=True
)
missing_allocations = slurm_models.Allocation.objects.exclude(id__in=exist_ids)
if dry_run:
logger.warning(
'Allocations would be imported to marketplace. '
'IDs: %s.' % format_list(missing_allocations)
)
return missing_allocations.count()
for allocation in missing_allocations:
offering = marketplace_models.Offering.objects.filter(
scope=allocation.service_settings
).first()
component_cpu = offering.components.get(type='cpu')
component_gpu = offering.components.get(type='gpu')
component_ram = offering.components.get(type='ram')
try:
plan = marketplace_models.Plan.objects.get(offering=offering)
except marketplace_models.Plan.DoesNotExist:
logger.warning(
'Resource has not been created. Because Plan is not found. '
'Offering UUID: %s.' % offering.uuid.hex
)
continue
state = (
marketplace_models.Resource.States.OK
if allocation.is_active
else marketplace_models.Resource.States.TERMINATED
)
resource = marketplace_models.Resource.objects.create(
content_type=ct,
object_id=allocation.id,
state=state,
project=allocation.service_project_link.project,
offering=offering,
created=allocation.created,
plan=plan,
limits={'deposit_limit': int(allocation.deposit_limit)},
)
marketplace_models.ComponentQuota.objects.create(
resource=resource, limit=allocation.cpu_limit, component=component_cpu
)
marketplace_models.ComponentQuota.objects.create(
resource=resource, limit=allocation.gpu_limit, component=component_gpu
)
marketplace_models.ComponentQuota.objects.create(
resource=resource, limit=allocation.ram_limit, component=component_ram
)
resource_plan_period = marketplace_models.ResourcePlanPeriod.objects.create(
resource=resource, plan=resource.plan, start=allocation.created, end=None,
)
for allocation_usage in allocation.allocationusage_set.all():
if (
allocation_usage.year == allocation.created.year
and allocation_usage.month == allocation.created.month
):
day = allocation.created.day
else:
day = 1
date = datetime.date(allocation_usage.year, allocation_usage.month, day)
marketplace_models.ComponentUsage.objects.create(
resource=resource,
component=component_cpu,
usage=allocation_usage.cpu_usage,
date=date,
billing_period=month_start(date),
plan_period=resource_plan_period,
)
marketplace_models.ComponentUsage.objects.create(
resource=resource,
component=component_gpu,
usage=allocation_usage.gpu_usage,
date=date,
billing_period=month_start(date),
plan_period=resource_plan_period,
)
marketplace_models.ComponentUsage.objects.create(
resource=resource,
component=component_ram,
usage=allocation_usage.ram_usage,
date=date,
billing_period=month_start(date),
plan_period=resource_plan_period,
)
return missing_allocations.count()
def import_slurm_service_settings(default_customer, dry_run=False):
"""
Import SLURM service settings as marketplace offerings.
"""
service_type = slurm_apps.SlurmConfig.service_name
category = marketplace_models.Category.objects.get(
uuid=settings.WALDUR_MARKETPLACE_SLURM['CATEGORY_UUID'])
slurm_offerings = marketplace_models.Offering.objects.filter(
type=PLUGIN_NAME)
front_settings = set(
slurm_offerings.exclude(object_id=None).values_list('object_id',
flat=True))
back_settings = structure_models.ServiceSettings.objects.filter(
type=service_type, state=structure_models.ServiceSettings.States.OK)
missing_settings = back_settings.exclude(id__in=front_settings)
if dry_run:
logger.warning(
'SLURM service settings would be imported to marketplace. '
'IDs: %s.' % format_list(missing_settings))
return missing_settings.count()
for service_settings in missing_settings:
offering = marketplace_models.Offering.objects.create(
scope=service_settings,
type=PLUGIN_NAME,
name=service_settings.name,
geolocations=service_settings.geolocations,
customer=service_settings.customer or default_customer,
category=category,
shared=service_settings.shared,
state=marketplace_models.Offering.States.ACTIVE,
)
components = plugins.manager.get_components(PLUGIN_NAME)
for component_data in components:
marketplace_models.OfferingComponent.objects.create(
offering=offering, **component_data._asdict())
try:
slurm_package = slurm_invoices_models.SlurmPackage.objects. \
get(service_settings=service_settings)
plan = marketplace_models.Plan.objects.create(
offering=offering,
scope=slurm_package,
name=slurm_package.name)
marketplace_models.PlanComponent.objects.create(
plan=plan,
component=offering.components.filter(type='cpu').get(),
price=slurm_package.cpu_price)
marketplace_models.PlanComponent.objects.create(
plan=plan,
component=offering.components.filter(type='gpu').get(),
price=slurm_package.gpu_price)
marketplace_models.PlanComponent.objects.create(
plan=plan,
component=offering.components.filter(type='ram').get(),
price=slurm_package.ram_price)
except slurm_invoices_models.SlurmPackage.DoesNotExist:
logger.warning(
'Plan has not been created. Because SlurmPackage is not found. '
'Service settings UUID: %s.' % service_settings.uuid.hex)
return missing_settings.count()
