def update_image_membership(account_driver, cloud_machine, db_machine):
"""
Given a cloud_machine and db_machine, create any relationships possible for ProviderMachineMembership and ApplicationVersionMembership
"""
image_visibility = cloud_machine.get('visibility','private')
if image_visibility.lower() == 'public':
return
image_owner = cloud_machine.get('application_owner','')
#TODO: In a future update to 'imaging' we might image 'as the user' rather than 'as the admin user', in this case we should just use 'owner' metadata
shared_group_names = [image_owner]
shared_projects = account_driver.shared_images_for(cloud_machine.id, None)
has_machine_request = db_machine.application_version.machinerequest_set.first()
if has_machine_request and has_machine_request.status.name == 'completed':
provider = has_machine_request.new_machine_provider
identifier = has_machine_request.new_machine.identifier
main_account_driver = get_account_driver(provider)
shared_projects_from_main = main_account_driver.shared_images_for(identifier, None)
shared_group_names.extend(p.name for p in shared_projects_from_main if p)
shared_group_names.extend(p.name for p in shared_projects if p)
groups = Group.objects.filter(name__in=shared_group_names)
if not groups:
return
for group in groups:
update_db_membership_for_group(db_machine, group)
def __init__(self, identity, network_driver, neutron):
self.account_driver = get_account_driver(identity.provider)
if not self.account_driver:
raise Exception(
"ConfigError: Cannot use ExternalRouter topology without an "
"AccountDriver. Please add an AccountProvider linked to an "
"Identity who has the 'admin' role on Openstack "
"_or_ use ExternalNetworkTopology to continue"
)
router_name = identity.get_credential('router_name')
if not router_name:
router_name = identity.provider.get_credential('router_name')
if not router_name:
router_name = identity.provider.select_router()
from core.models import Credential
Credential.objects.get_or_create(
identity=identity, key='router_name', value=router_name
)
if not router_name:
raise Exception(
"Unknown Router - Identity %s is missing 'router_name' " %
identity
)
self.external_router_name = router_name
return super(ExternalRouter,
self).__init__(identity, network_driver, neutron)
def redeploy_users(provider, users=[]):
accounts = get_account_driver(provider)
tenant_instances_map = accounts.tenant_instances_map(status_list=['deploy_error', 'networking','deploying','initializing'])
for tenant, instance_list in tenant_instances_map.iteritems():
username = tenant.name
if users and username not in users:
print "Found affected user:%s and Instances:%s - Skipping because they aren't in the list." % (username, instance_list)
continue
for instance in instance_list:
metadata = instance._node.extra.get('metadata',{})
instance_status = instance.extra.get('status')
tmp_status = metadata.get('tmp_status','')
print "Starting idempotent redeployment for %s - Instance: %s (%s - %s)" % (username, instance.id, instance_status, tmp_status)
ident = Identity.objects.get(provider=provider, created_by__username=username)
driver = get_esh_driver(ident)
try:
start_task = get_idempotent_deploy_chain(driver.__class__, driver.provider, driver.identity, instance, username)
print "Starting idempotent redeployment: %s ..." % (start_task),
start_task.apply_async()
except Identity.DoesNotExist:
print "Identity does not exist in this DB. SKIPPED."
continue
if DO_NOTHING:
continue
print " Sent"
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--provider",
type=int,
help="Atmosphere provider ID"
" to use when importing users.")
parser.add_argument("--provider-id",
type=int,
help="Atmosphere provider ID"
" to use when importing users."
" DEPRECATION WARNING -- THIS WILL BE REMOVED SOON!")
parser.add_argument("--provider-list",
action="store_true",
help="List of provider names and IDs")
parser.add_argument("--rebuild",
action="store_true",
help="Rebuild all accounts that are in the provider")
parser.add_argument("--users",
help="LDAP usernames to import. (comma separated)")
parser.add_argument("--admin",
action="store_true",
help="Users addded as admin and staff users.")
args = parser.parse_args()
if args.provider_list:
print "ID\tName"
for p in Provider.objects.all().order_by('id'):
print "%d\t%s" % (p.id, p.location)
return
users = None
if args.provider_id and not args.provider:
print "WARNING: --provider-id has been *DEPRECATED*! Use --provider instead!"
args.provider = args.provider_id
if args.provider:
provider = Provider.objects.get(id=args.provider)
else:
raise Exception(
"Missing required argument: --provider <id>. use --provider-list to get a list of provider ID+names"
)
print "Using Provider: %s" % provider
type_name = provider.type.name.lower()
try:
acct_driver = get_account_driver(provider, raise_exception=True)
except:
account_provider = provider.accountprovider_set.first()
print "Could not create the account Driver for this Provider."\
" Check the configuration of this identity:%s" % account_provider
raise
if not args.users:
if not args.rebuild:
print "Retrieving all 'atmo-user' members in LDAP."
users = get_members('atmo-user')
else:
print "Rebuilding all existing users."
users = get_usernames(provider)
else:
users = args.users.split(",")
return create_accounts(acct_driver, provider, users, args.rebuild,
args.admin)
def _get_instance_owner_map(provider, users=None):
"""
All keys == All identities
Values = List of identities / username
NOTE: This is KEYSTONE && NOVA specific. the 'instance owner' here is the
username // ex_tenant_name
"""
from service.driver import get_account_driver
admin_driver = get_cached_driver(provider=provider)
accounts = get_account_driver(provider=provider)
all_identities = _select_identities(provider, users)
acct_providers = AccountProvider.objects.filter(provider=provider)
if acct_providers:
account_identity = acct_providers[0].identity
provider = None
else:
account_identity = None
all_instances = get_cached_instances(provider=provider, identity=account_identity, force=True)
#all_tenants = admin_driver._connection._keystone_list_tenants()
all_tenants = accounts.list_projects()
# Convert instance.owner from tenant-id to tenant-name all at once
all_instances = _convert_tenant_id_to_names(all_instances, all_tenants)
# Make a mapping of owner-to-instance
instance_map = _make_instance_owner_map(all_instances, users=users)
logger.info("Instance owner map created")
identity_map = _include_all_idents(all_identities, instance_map)
logger.info("Identity map created")
return identity_map
def update_application_owner(application, identity):
from service.openstack import glance_update_machine_metadata
from service.driver import get_account_driver
old_identity = application.created_by_identity
tenant_name = _extract_tenant_name(identity)
old_tenant_name = _extract_tenant_name(old_identity)
# Prepare the application
application.created_by_identity = identity
application.created_by = identity.created_by
application.save()
# Update all the PMs
all_pms = application.providermachine_set.all()
print "Updating %s machines.." % len(all_pms)
for provider_machine in all_pms:
accounts = get_account_driver(provider_machine.provider)
image_id = provider_machine.instance_source.identifier
image = accounts.get_image(image_id)
if not image:
continue
tenant_id = accounts.get_project(tenant_name).id
glance_update_machine_metadata(provider_machine, {"owner": tenant_id, "application_owner": tenant_name})
print "App data saved for %s" % image_id
accounts.image_manager.share_image(image, tenant_name)
print "Shared access to %s with %s" % (image_id, tenant_name)
accounts.image_manager.unshare_image(image, old_tenant_name)
print "Removed access to %s for %s" % (image_id, old_tenant_name)
def _set_compute_quota(user_quota, identity):
# Use THESE values...
compute_values = {
'cores': user_quota.cpu,
'ram': user_quota.memory*1024, # NOTE: Value is stored in GB, Openstack (Liberty) expects MB
'floating_ips': user_quota.floating_ip_count,
'fixed_ips': user_quota.port_count,
'instances': user_quota.instance_count,
}
creds = identity.get_all_credentials()
if creds.get('ex_force_auth_version','2.0_password') == "2.0_password":
compute_values.pop('instances')
username = identity.created_by.username
logger.info("Updating quota for %s to %s" % (username, compute_values))
driver = get_cached_driver(identity=identity)
username = driver._connection.key
tenant_id = driver._connection._get_tenant_id()
tenant_name = identity.project_name()
ad = get_account_driver(identity.provider)
ks_user = ad.get_user(username)
admin_driver = ad.admin_driver
try:
result = admin_driver._connection.ex_update_quota_for_user(
tenant_id, ks_user.id, compute_values)
except Exception:
logger.exception("Could not set a user-quota, trying to set tenant-quota")
result = admin_driver._connection.ex_update_quota(tenant_id, compute_values)
logger.info("Updated quota for %s to %s" % (username, result))
return result
def create_new_account_for(provider, user):
from service.exceptions import AccountCreationConflict
from service.driver import get_account_driver
existing_user_list = provider.identity_set.values_list(
'created_by__username', flat=True
)
if user.username in existing_user_list:
logger.info(
"Accounts already exists on %s for %s" %
(provider.location, user.username)
)
return None
try:
accounts = get_account_driver(provider)
logger.info("Create NEW account for %s" % user.username)
default_quota = DefaultQuotaPluginManager.default_quota(
user=user, provider=provider
)
new_identity = accounts.create_account(
user.username, quota=default_quota
)
return new_identity
except AccountCreationConflict:
raise # TODO: Ideally, have sentry handle these events, rather than force an Unhandled 500 to bubble up.
except:
logger.exception(
"Could *NOT* Create NEW account for %s" % user.username
)
return None
def upload_privacy_data(machine_request, new_machine):
"""
ASSERT: The image in 'new_machine' SHOULD BE private
(Based on values in machine_request)
"""
prov = new_machine.provider
accounts = get_account_driver(prov)
if not accounts:
print "Aborting import: Could not retrieve Account Driver "\
"for Provider %s" % prov
return
img = accounts.get_image(new_machine.identifier)
if hasattr(img, 'visibility'): # Treated as an obj.
is_public = img.visibility == 'public'
elif hasattr(img, 'items'): # Treated as a dict.
is_public = img.get('visibility','N/A') == 'public'
if is_public:
print "Marking image %s private" % img.id
accounts.image_manager.update_image(img, visibility='private')
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
print "Aborting import: Could not retrieve admin_driver "\
"for Provider %s" % prov
return
img = accounts.get_image(new_machine.identifier)
tenant_list = machine_request.get_access_list()
# All in the list will be added as 'sharing' the OStack img
# All tenants already sharing the OStack img will be added to this list
return sync_machine_membership(accounts, img, new_machine, tenant_list)
def fix_requests(provider, requests=[]):
accounts = get_account_driver(provider)
for request_id in requests:
try:
machine_request = MachineRequest.objects.get(id=request_id)
except MachineRequest.DoesNotExist:
print "Warn: MachineRequest by this ID could not be found: %s" % request_id
continue
if machine_request.new_machine_provider != provider:
raise ValueError(
"MachineRequest ID:%s is for Provider:%s. Testing Provider is:%s" %
(request_id, machine_request.new_machine_provider, provider))
fixed = False
try:
new_machine = ProviderMachine.objects.get(
id=machine_request.new_machine_id)
except ProviderMachine.DoesNotExist as no_match:
print "OK: This MachineRequest has a BAD 'new_machine' (DoesNotExist)"
new_machine = None
if not fixed:
fixed = _fix_wrong_machine_on_request(
machine_request,
provider,
new_machine)
if not fixed and new_machine:
_fix_new_version_forked(machine_request, provider, new_machine)
def _fix_new_machine_forked(machine_request, provider, new_machine):
app_uuid = _generate_app_uuid(new_machine.identifier)
if not machine_request.new_version_forked:
return False
if Application.objects.filter(uuid=app_uuid).count():
return False
print "OK: This MachineRequest: %s has a BAD Application." \
"\tUUID should be %s." % (machine_request, app_uuid)
old_application = new_machine.application
current_application = _create_new_application(
machine_request,
new_machine.identifier)
remaining_machines = old_application._current_machines()
for machine in remaining_machines:
if machine.identifier == new_machine.identifier:
new_machine.application = current_application
new_machine.save()
current_application.save()
# Pass #2 - If remaining, unmatched ids:
remaining_machines = old_application._current_machines()
acct_provider = machine_request.new_machine_provider
accounts = get_account_driver(acct_provider)
if remaining_machines:
print "Warn: These machines likely point to the wrong application.:%s" % remaining_machines
for machine in remaining_machines:
glance_image = accounts.image_manager.get_image(machine.identifier)
if glance_image:
original_request = MachineRequest.objects.filter(
new_application_name=glance_image.name)
print "Hint: Image_ID:%s Named:%s MachineRequest:%s" % (glance_image.id, glance_image.name, original_request)
return True
def print_instances(provider, users=[], status_list=[]):
accounts = get_account_driver(provider)
tenant_instances_map = accounts.tenant_instances_map(
status_list=status_list, match_all=MATCH_ALL)
for tenant, instance_list in tenant_instances_map.iteritems():
username = tenant.name
if users and username not in users:
continue
for instance in instance_list:
instance_status = instance.extra.get('status')
task = instance.extra.get('task')
metadata = instance.extra.get('metadata', {})
tmp_status = metadata.get('tmp_status', '')
created = instance.extra.get('created', "N/A")
updated = instance.extra.get('updated', "N/A")
status_name = _get_status_name_for_provider(
provider, instance_status, task, tmp_status)
try:
last_history = Instance.objects.get(
provider_alias=instance.id).get_last_history()
except:
last_history = "N/A (Instance not in this DB)"
print "Tenant:%s Instance:%s Status: (%s - %s) Created:%s Updated:%s, Last History:%s" % (
username, instance.id, instance_status, tmp_status, created,
updated, last_history)
def update_cloud_membership_for_machine(provider_machine, group):
"""
Given a provider_machine and a group
* Loop through identities owned by group
* * If identity.provider == provider_machine.provider, allow identity to launch via cloud ACLs
"""
prov = provider_machine.instance_source.provider
accounts = get_account_driver(prov)
if not accounts:
raise NotImplemented("Account Driver could not be created for %s" % prov)
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
raise NotImplemented("Admin Driver could not be created for %s" % prov)
img = accounts.get_image(provider_machine.identifier)
if img.visibility == 'public':
return
approved_projects = accounts.shared_images_for(img.id)
for identity_membership in group.identitymembership_set.all():
if identity_membership.identity.provider != prov:
logger.debug("Skipped %s -- Wrong provider" % identity_membership.identity)
continue
# Get project name from the identity's credential-list
identity = identity_membership.identity
project_name = identity.get_credential('ex_project_name')
project = accounts.get_project(project_name)
if not project:
logger.debug("Unknown Project: %s -- Does not exist" % project)
continue
elif project in approved_projects:
logger.debug("Skipped Project: %s -- Already shared" % project)
continue
accounts.share_image_with_identity(img, identity)
def _get_instance_owner_map(provider, users=None):
"""
All keys == All identities
Values = List of identities / username
NOTE: This is KEYSTONE && NOVA specific. the 'instance owner' here is the
username // ex_tenant_name
"""
from service.driver import get_account_driver
accounts = get_account_driver(provider=provider, raise_exception=True)
all_identities = _select_identities(provider, users)
acct_providers = AccountProvider.objects.filter(provider=provider)
if acct_providers:
account_identity = acct_providers[0].identity
provider = None
else:
account_identity = None
all_instances = get_cached_instances(
provider=provider, identity=account_identity, force=True
)
#all_tenants = admin_driver._connection._keystone_list_tenants()
all_tenants = accounts.list_projects()
# Convert instance.owner from tenant-id to tenant-name all at once
all_instances = _convert_tenant_id_to_names(all_instances, all_tenants)
# Make a mapping of owner-to-instance
instance_map = _make_instance_owner_map(all_instances, users=users)
logger.info("Instance owner map created")
identity_map = _include_all_idents(all_identities, instance_map)
logger.info("Identity map created")
return identity_map
def delete_accounts(self, provider, username):
from service.driver import get_account_driver
account_driver = get_account_driver(provider)
if not account_driver:
raise ValueError(
"Provider %s produced an invalid account driver "\
"-- Use plugin after you create a core.Provider "\
"*AND* assign a core.Identity to be the core.AccountProvider."
% provider)
credentials_list = self.get_credentials_list(provider, username)
identity_list = []
for credentials in credentials_list:
try:
identities = self.find_accounts(**credentials)
if not identities:
continue
logger.debug(
"Removing account for %s with credentials - %s" %
(username, credentials)
)
for identity in identities:
removed_identity = account_driver.delete_account(
identity, **credentials
)
identity_list.append(removed_identity)
except:
logger.exception("Could *NOT* delete account for %s" % username)
return identity_list
def validate_account_driver(self, validated_data):
try:
provider = validated_data['provider']
acct_driver = get_account_driver(provider, raise_exception=True)
return acct_driver
except Exception as exc:
raise serializers.ValidationError("Attempting to create an account for provider %s failed. Message: %s" % (provider, exc.message))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--provider-list", action="store_true",
help="List of provider names and IDs")
parser.add_argument("--quota-list", action="store_true",
help="List of provider names and IDs")
parser.add_argument("--provider-id", type=int,
help="Atmosphere provider ID"
" to use when importing users.")
parser.add_argument("--quota-id",
help="Atmosphere Quota ID to assign (Optional)")
parser.add_argument("--groups",
help="LDAP groups to import. (comma separated)")
parser.add_argument("--dry-run", action="store_true",
help="A 'dry-run' so you know what will happen,"
" before it happens")
parser.add_argument("--users",
help="LDAP usernames to import. (comma separated)")
parser.add_argument("--admin", action="store_true",
help="ALL Users addded are treated as admin and staff "
"users. They also receive the maximum quota.")
args = parser.parse_args()
make_admins = args.admin
dry_run = args.dry_run
users = None
quota = None
if args.provider_list:
print "ID\tName"
for p in Provider.objects.all().order_by('id'):
print "%d\t%s" % (p.id, p.location)
return
elif args.quota_list:
print "ID\tSpecs"
for q in Quota.objects.all().order_by('id'):
print "%s\t%s" % (q.id, q)
return
#Debugging args
if dry_run:
print "Dry run initialized.."
#Optional args
if args.quota_id:
quota = Quota.objects.get(id=args.quota_id)
if not args.provider_id:
print "ERROR: provider-id is required. To get a list of providers use"\
" --provider-list"
provider = Provider.objects.get(id=args.provider_id)
print "Provider Selected:%s" % provider
acct_driver = get_account_driver(provider)
groups = args.groups.split(",") if args.groups else []
total_added = process_groups(acct_driver, groups, quota, make_admins, dry_run)
users = args.users.split(",") if args.users else []
total_added += process_users(acct_driver, users, quota, make_admins, dry_run)
print "Processing complete. %d users processed." % total_added
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--dry-run", action="store_true",
help="List of provider names and IDs")
args = parser.parse_args()
admin_owned_apps = Application.objects.filter(
created_by__username__contains='admin').filter(only_current_apps()).distinct()
account_drivers = {}
# FIXME: Change the provider_id if necessary.
for app in admin_owned_apps:
# Step 1 - See if MachineRequest can answer the question
machine = app._current_machines().filter(instance_source__provider__id=4).first()
if not machine:
continue
mr = MachineRequest.objects.filter(new_machine=machine).first()
if mr:
fix_application_owner(app, mr.created_by, args.dry_run)
continue
# Step 2 - See if glance can answer the question
provider = machine.provider
if account_drivers.get(provider):
accounts = account_drivers[provider]
else:
accounts = get_account_driver(provider)
account_drivers[provider] = accounts
img = accounts.get_image(machine.identifier)
if not img:
continue
project = accounts.get_project_by_id(img.owner)
if not project:
continue
user = AtmosphereUser.objects.filter(username=project.name).first()
if user:
fix_application_owner(app, user, args.dry_run)
def update_application_owner(application, identity):
from service.openstack import glance_update_machine_metadata
from service.driver import get_account_driver
old_identity = application.created_by_identity
tenant_name = _extract_tenant_name(identity)
old_tenant_name = _extract_tenant_name(old_identity)
# Prepare the application
application.created_by_identity = identity
application.created_by = identity.created_by
application.save()
# Update all the PMs
all_pms = application.providermachine_set.all()
print "Updating %s machines.." % len(all_pms)
for provider_machine in all_pms:
accounts = get_account_driver(provider_machine.provider)
image_id = provider_machine.instance_source.identifier
image = accounts.get_image(image_id)
if not image:
continue
tenant_id = accounts.get_project(tenant_name).id
glance_update_machine_metadata(
provider_machine,
{'owner': tenant_id,
'application_owner': tenant_name})
print "App data saved for %s" % image_id
accounts.image_manager.share_image(image, tenant_name)
print "Shared access to %s with %s" % (image_id, tenant_name)
accounts.image_manager.unshare_image(image, old_tenant_name)
print "Removed access to %s for %s" % (image_id, old_tenant_name)
def update_cloud_membership_for_machine(provider_machine, group):
"""
Given a provider_machine and a group
* Loop through identities owned by group
* * If identity.provider == provider_machine.provider, allow identity to launch via cloud ACLs
"""
prov = provider_machine.instance_source.provider
accounts = get_account_driver(prov)
if not accounts:
raise NotImplemented("Account Driver could not be created for %s" % prov)
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
raise NotImplemented("Admin Driver could not be created for %s" % prov)
img = accounts.get_image(provider_machine.identifier)
if img.get('visibility') != 'shared':
logger.debug("Skipped updates for image %s -- visibility (%s) is not 'shared'", img.id, img.get('visibility'))
return
approved_projects = accounts.get_image_members(img.id)
for identity_membership in group.identitymembership_set.all():
if identity_membership.identity.provider != prov:
logger.debug("Skipped %s -- Wrong provider" % identity_membership.identity)
continue
# Get project name from the identity's credential-list
identity = identity_membership.identity
project_name = identity.get_credential('ex_project_name')
project = accounts.get_project(project_name)
if not project:
logger.debug("Unknown Project: %s -- Does not exist" % project)
continue
elif project in approved_projects:
logger.debug("Skipped Project: %s -- Already shared" % project)
continue
accounts.share_image_with_identity(img, identity)
def post(self, request, cloud_admin_uuid):
"""
Passes in:
Username (To apply the identity to)
Credentials (Nested, will be applied to new identity)
"""
user = request.user
data = request.DATA
try:
provider_uuid = data['provider']
provider = Provider.objects.get(
cloudadministrator__user=user,
uuid=provider_uuid)
except KeyError:
return Response(
"Missing 'provider' key, Expected UUID. Received no value.",
status=status.HTTP_409_conflict)
except Exception:
return Response(
"Provider with UUID %s does not exist" % provider_uuid,
status=status.HTTP_409_conflict)
raise Exception
driver = get_account_driver(provider)
missing_args = driver.clean_credentials(data)
if missing_args:
raise Exception("Cannot create account. Missing credentials: %s"
% missing_args)
identity = driver.create_account(**data)
serializer = IdentitySerializer(identity)
# TODO: Account creation SHOULD return IdentityMembership NOT identity.
return Response(serializer.data)
def print_instances(provider, users=[], status_list=[]):
accounts = get_account_driver(provider)
tenant_instances_map = accounts.tenant_instances_map(
status_list=status_list,
match_all=MATCH_ALL)
for tenant, instance_list in tenant_instances_map.iteritems():
username = tenant.name
if users and username not in users:
continue
for instance in instance_list:
instance_status = instance.extra.get('status')
task = instance.extra.get('task')
metadata = instance.extra.get('metadata', {})
tmp_status = metadata.get('tmp_status', '')
created = instance.extra.get('created', "N/A")
updated = instance.extra.get('updated', "N/A")
status_name = _get_status_name_for_provider(
provider,
instance_status,
task,
tmp_status)
try:
last_history = Instance.objects.get(
provider_alias=instance.id).get_last_history()
except:
last_history = "N/A (Instance not in this DB)"
print "Tenant:%s Instance:%s Status: (%s - %s) Created:%s Updated:%s, Last History:%s" % (username, instance.id, instance_status, tmp_status, created, updated, last_history)
def _get_all_access_list(account_driver, db_machine, cloud_machine):
"""
Input: AccountDriver, ProviderMachine, glance_image
Output: A list of _all project names_ that should be included on `cloud_machine`
This list will include:
- Users who match the provider_machine's application.access_list
- Users who are already approved to use the `cloud_machine`
- The owner of the application/Creator of the MachineRequest
- If using settings.REPLICATION_PROVIDER:
- include all those approved on the replication provider's copy of the image
"""
#TODO: In a future update to 'imaging' we might image 'as the user' rather than 'as the admin user', in this case we should just use 'owner' metadata
image_owner = cloud_machine.get('application_owner')
# NOTE: This assumes that the 'owner' (atmosphere user) == 'project_name' (Openstack)
# Always include the original application owner
owner_set = set()
if image_owner:
owner_set.add(image_owner)
if hasattr(cloud_machine, 'id'):
image_id = cloud_machine.id
elif type(cloud_machine) == dict:
image_id = cloud_machine.get('id')
else:
raise ValueError("Unexpected cloud_machine: %s" % cloud_machine)
existing_members = account_driver.get_image_members(image_id, None)
# Extend to include based on projects already granted access to the image
cloud_shared_set = { p.name for p in existing_members }
# Deprecation warning: Now that we use a script to do replication,
# we should not need to account for shares on another provider.
# Remove this code any time during/after the v29 release
has_machine_request = MachineRequest.objects.filter(
new_machine__instance_source__identifier=cloud_machine.id,
status__name='completed').last()
machine_request_set = set()
machine_request_provider_set = set()
if has_machine_request:
access_list = has_machine_request.get_access_list()
# NOTE: This assumes that every name in
# accesslist (AtmosphereUser) == project_name(Openstack)
machine_request_set = { name.strip() for name in access_list }
request_provider = has_machine_request.new_machine_provider
request_identifier = has_machine_request.new_machine.instance_source.identifier
if request_provider != db_machine.provider:
main_account_driver = get_account_driver(request_provider)
# Extend to include based on information in the machine request
request_shared_projects = main_account_driver.get_image_members(request_identifier, None)
machine_request_provider_set = set(p.name for p in request_shared_projects)
# End deprecation warning
# Extend to include new names found by application pattern_match
parent_app = db_machine.application_version.application
access_list_set = set(parent_app.get_users_from_access_list().values_list('username', flat=True))
shared_project_names = list(owner_set | cloud_shared_set | machine_request_set | machine_request_provider_set | access_list_set)
return shared_project_names
def glance_image_owner(provider_uuid, identifier, glance_image=None):
try:
prov = Provider.objects.get(uuid=provider_uuid)
accounts = get_account_driver(prov)
if not glance_image:
accounts.clear_cache()
glance_image = accounts.get_image(identifier)
project = accounts.user_manager.get_project_by_id(
glance_image.get('owner')
)
except Exception as e:
logger.exception(e)
project = None
if not project:
return None
try:
image_owner = Identity.objects.get(
provider__uuid=provider_uuid, created_by__username=project.name
)
except Identity.DoesNotExist:
logger.warn(
"Could not find a username %s on Provider %s" %
(project.name, provider_uuid)
)
image_owner = None
return image_owner
def remove_empty_networks_for(provider_id):
provider = Provider.objects.get(id=provider_id)
os_driver = get_account_driver(provider)
all_instances = os_driver.admin_driver.list_all_instances()
project_map = os_driver.network_manager.project_network_map()
projects_with_networks = project_map.keys()
for project in projects_with_networks:
networks = project_map[project]['network']
if type(networks) != list:
networks = [networks]
for network in networks:
network_name = network['name']
logger.debug("Checking if network %s is in use" % network_name)
if running_instances(network_name, all_instances):
continue
#TODO: MUST change when not using 'usergroups' explicitly.
user = project
try:
logger.debug("Removing project network for User:%s, Project:%s"
% (user, project))
os_driver.network_manager.delete_project_network(user, project)
except NeutronClientException:
logger.exception("Neutron unable to remove project"
"network for %s-%s" % (user,project))
except NeutronException:
logger.exception("Neutron unable to remove project"
"network for %s-%s" % (user,project))
def upload_privacy_data(machine_request, new_machine):
"""
ASSERT: The image in 'new_machine' SHOULD BE private
(Based on values in machine_request)
"""
prov = new_machine.provider
accounts = get_account_driver(prov)
if not accounts:
print "Aborting import: Could not retrieve Account Driver "\
"for Provider %s" % prov
return
img = accounts.get_image(new_machine.identifier)
if hasattr(img, 'visibility'): # Treated as an obj.
is_public = img.visibility == 'public'
elif hasattr(img, 'items'): # Treated as a dict.
is_public = img.get('visibility','N/A') == 'public'
if is_public:
print "Marking image %s private" % img.id
accounts.image_manager.update_image(img, visibility='shared')
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
print "Aborting import: Could not retrieve admin_driver "\
"for Provider %s" % prov
return
img = accounts.get_image(new_machine.identifier)
tenant_list = machine_request.get_access_list()
# All in the list will be added as 'sharing' the OStack img
# All tenants already sharing the OStack img will be added to this list
return sync_machine_membership(accounts, img, new_machine, tenant_list)
def validate_new_image(image_id, machine_request_id):
machine_request = MachineRequest.objects.get(id=machine_request_id)
new_status, _ = StatusType.objects.get_or_create(name="validating")
machine_request.status = new_status
machine_request.old_status = 'validating'
machine_request.save()
accounts = get_account_driver(machine_request.new_machine.provider)
accounts.clear_cache()
from service.instance import launch_machine_instance
admin_driver = accounts.admin_driver
admin_ident = machine_request.new_admin_identity()
if not admin_driver:
celery_logger.warn(
"Need admin_driver functionality to auto-validate instance")
return False
if not admin_ident:
celery_logger.warn(
"Need to know the AccountProvider to auto-validate instance")
return False
# Attempt to launch using the admin_driver
admin_driver.identity.user = admin_ident.created_by
machine = admin_driver.get_machine(image_id)
small_size = admin_driver.list_sizes()[0]
instance = launch_machine_instance(
admin_driver, admin_ident,
machine, small_size,
'Automated Image Verification - %s' % image_id,
username='******',
using_admin=True)
return instance.id
def start_instance_maintenances(provider, action, users=[], sleep_time=None, dry_run=False):
accounts = get_account_driver(provider)
all_insts = accounts.list_all_instances()
all_tenants = accounts.list_projects()
all_insts = make_user_instances(all_insts, all_tenants, users)
hostname_map = _create_hostname_mapping(all_insts)
finished = False
while not finished:
#Iterate the list of hosts, complete
finished = True
for host in hostname_map.keys():
inst_list = hostname_map[host]
if len(inst_list) == 0:
continue
instance = inst_list.pop()
print "Instance %s - Hostname %s" % (instance.id, host)
status = instance.extra['status']
if status != 'active':
print "Skipping instance %s in state %s" % (instance.id, status)
continue
finished = False
identity = Identity.objects.get(
created_by__username=instance.username,
provider=provider)
print 'Performing Instance Maintenance - %s - %s' % (instance.id, host)
try:
_execute_action(identity, instance, action, dry_run)
except Exception, e:
print "Could not %s Instance %s - Error %s" % (action, instance.id, e)
continue
print "Waiting %s seconds" % sleep_time
if not dry_run:
time.sleep(sleep_time)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--provider", type=int,
help="Atmosphere provider ID"
" to use when importing users.")
parser.add_argument("--provider-id", type=int,
help="Atmosphere provider ID"
" to use when importing users."
" DEPRECATION WARNING -- THIS WILL BE REMOVED SOON!")
parser.add_argument("--provider-list",
action="store_true",
help="List of provider names and IDs")
parser.add_argument("--rebuild", action="store_true",
help="Rebuild all accounts that are in the provider")
parser.add_argument("--group",
help="LDAP group of usernames to import.")
parser.add_argument("--users",
help="usernames to add to Atmosphere. (comma separated list with no spaces)")
parser.add_argument("--admin", action="store_true",
help="Users addded as admin and staff users.")
args = parser.parse_args()
if args.provider_list:
print "ID\tName"
for p in Provider.objects.all().order_by('id'):
print "%d\t%s" % (p.id, p.location)
return
usernames = []
if args.provider_id and not args.provider:
print "WARNING: --provider-id has been *DEPRECATED*! Use --provider instead!"
args.provider = args.provider_id
if args.provider:
provider = Provider.objects.get(id=args.provider)
else:
raise Exception("Missing required argument: --provider <id>. use --provider-list to get a list of provider ID+names")
print "Using Provider: %s" % provider
try:
acct_driver = get_account_driver(provider, raise_exception=True)
except:
account_provider = provider.accountprovider_set.first()
print "Could not create the account Driver for this Provider."\
" Check the configuration of this identity:%s" % account_provider
raise
if args.group:
print "Retrieving all '%s' members in LDAP." % args.group
usernames = get_members(args.group)
elif args.users:
usernames = args.users.split(",")
else: # if not args.users
if not args.rebuild:
print "Retrieving all 'atmo-user' members in LDAP."
usernames = get_members('atmo-user')
else:
print "Rebuilding all existing users."
usernames = get_usernames(provider)
return run_create_accounts(acct_driver, provider, usernames,
args.rebuild, args.admin)
def prune_machines_for(
provider_id, print_logs=False, dry_run=False, forced_removal=False):
"""
Look at the list of machines (as seen by the AccountProvider)
if a machine cannot be found in the list, remove it.
NOTE: BEFORE CALLING THIS TASK you should ensure
that the AccountProvider can see ALL images.
Failure to do so will result in any image unseen by the admin
to be prematurely end-dated and removed from the API/UI.
"""
provider = Provider.objects.get(id=provider_id)
now = timezone.now()
if print_logs:
import logging
import sys
consolehandler = logging.StreamHandler(sys.stdout)
consolehandler.setLevel(logging.DEBUG)
celery_logger.addHandler(consolehandler)
celery_logger.info("Starting prune_machines for Provider %s @ %s"
% (provider, now))
if provider.is_active():
account_driver = get_account_driver(provider)
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
cloud_machines = account_driver.list_all_images()
else:
db_machines = ProviderMachine.objects.filter(
source_in_range(), # like 'only_current..' w/o active_provider
instance_source__provider=provider)
cloud_machines = []
# Don't do anything if cloud machines == [None,[]]
if not cloud_machines and not forced_removal:
return
# Loop 1 - End-date All machines in the DB that
# can NOT be found in the cloud.
mach_count = _end_date_missing_database_machines(
db_machines, cloud_machines, now=now, dry_run=dry_run)
# Loop 2 and 3 - Capture all (still-active) versions without machines,
# and all applications without versions.
# These are 'outliers' and mainly here for safety-check purposes.
ver_count = _remove_versions_without_machines(now=now)
app_count = _remove_applications_without_versions(now=now)
# Loop 4 - All 'Application' DB objects require
# >=1 Version with >=1 ProviderMachine (ACTIVE!)
# Apps that don't meet this criteria should be end-dated.
app_count += _update_improperly_enddated_applications(now)
celery_logger.info(
"prune_machines completed for Provider %s : "
"%s Applications, %s versions and %s machines pruned."
% (provider, app_count, ver_count, mach_count))
if print_logs:
celery_logger.removeHandler(consolehandler)
def glance_image_for(provider_uuid, identifier):
try:
prov = Provider.objects.get(uuid=provider_uuid)
accounts = get_account_driver(prov)
glance_image = accounts.get_image(identifier)
except Exception as e:
logger.exception(e)
glance_image = None
return glance_image
def memoized_driver(machine, account_drivers={}):
provider = machine.instance_source.provider
account_driver = account_drivers.get(provider)
if not account_driver:
account_driver = get_account_driver(provider)
if not account_driver:
raise Exception("Cannot instantiate an account driver for %s" % provider)
account_drivers[provider] = account_driver
return account_driver
def glance_image_for(provider_uuid, identifier):
try:
prov = Provider.objects.get(uuid=provider_uuid)
accounts = get_account_driver(prov)
glance_image = accounts.get_image(identifier)
except Exception as e:
logger.exception(e)
glance_image = None
return glance_image
def memoized_driver(machine, account_drivers={}):
provider = machine.instance_source.provider
account_driver = account_drivers.get(provider)
if not account_driver:
account_driver = get_account_driver(provider)
if not account_driver:
raise Exception("Cannot instantiate an account driver for %s" % provider)
account_drivers[provider] = account_driver
return account_driver
def prune_machines_for(provider_id,
print_logs=False,
dry_run=False,
forced_removal=False):
"""
Look at the list of machines (as seen by the AccountProvider)
if a machine cannot be found in the list, remove it.
NOTE: BEFORE CALLING THIS TASK you should ensure
that the AccountProvider can see ALL images.
Failure to do so will result in any image unseen by the admin
to be prematurely end-dated and removed from the API/UI.
"""
provider = Provider.objects.get(id=provider_id)
now = timezone.now()
if print_logs:
console_handler = _init_stdout_logging()
celery_logger.info("Starting prune_machines for Provider %s @ %s" %
(provider, now))
if provider.is_active():
account_driver = get_account_driver(provider)
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
cloud_machines = account_driver.list_all_images()
else:
db_machines = ProviderMachine.objects.filter(
source_in_range(), # like 'only_current..' w/o active_provider
instance_source__provider=provider)
cloud_machines = []
# Don't do anything if cloud machines == [None,[]]
if not cloud_machines and not forced_removal:
return
# Loop 1 - End-date All machines in the DB that
# can NOT be found in the cloud.
mach_count = _end_date_missing_database_machines(db_machines,
cloud_machines,
now=now,
dry_run=dry_run)
# Loop 2 and 3 - Capture all (still-active) versions without machines,
# and all applications without versions.
# These are 'outliers' and mainly here for safety-check purposes.
ver_count = _remove_versions_without_machines(now=now)
app_count = _remove_applications_without_versions(now=now)
# Loop 4 - All 'Application' DB objects require
# >=1 Version with >=1 ProviderMachine (ACTIVE!)
# Apps that don't meet this criteria should be end-dated.
app_count += _update_improperly_enddated_applications(now)
celery_logger.info("prune_machines completed for Provider %s : "
"%s Applications, %s versions and %s machines pruned." %
(provider, app_count, ver_count, mach_count))
if print_logs:
_exit_stdout_logging(console_handler)
def monitor_machines_for(provider_id,
limit_machines=[],
print_logs=False,
dry_run=False):
"""
Run the set of tasks related to monitoring machines for a provider.
Optionally, provide a list of usernames to monitor
While debugging, print_logs=True can be very helpful.
start_date and end_date allow you to search a 'non-standard' window of time.
NEW LOGIC:
"""
provider = Provider.objects.get(id=provider_id)
if print_logs:
console_handler = _init_stdout_logging()
account_driver = get_account_driver(provider)
#Bail out if account driver is invalid
if not account_driver:
_exit_stdout_logging(console_handler)
return []
cloud_machines = account_driver.list_all_images()
if limit_machines:
cloud_machines = [
cm for cm in cloud_machines if cm.id in limit_machines
]
db_machines = []
# ASSERT: All non-end-dated machines in the DB can be found in the cloud
# if you do not believe this is the case, you should call 'prune_machines_for'
for cloud_machine in cloud_machines:
if not machine_is_valid(cloud_machine, account_driver):
continue
owner_project = _get_owner(account_driver, cloud_machine)
#STEP 1: Get the application, version, and provider_machine registered in Atmosphere
(db_machine, created) = convert_glance_image(cloud_machine,
provider.uuid,
owner_project)
db_machines.append(db_machine)
#STEP 2: For any private cloud_machine, convert the 'shared users' as known by cloud
update_image_membership(account_driver, cloud_machine, db_machine)
# into DB relationships: ApplicationVersionMembership, ProviderMachineMembership
#STEP 3: if ENFORCING -- occasionally 're-distribute' any ACLs that are *listed on DB but not on cloud* -- removals should be done explicitly, outside of this function
if settings.ENFORCING:
distribute_image_membership(account_driver, cloud_machine,
provider)
# ASSERTIONS about this method:
# 1) We will never 'remove' membership,
# 2) We will never 'remove' a public or private flag as listed in application.
# 2b) Future: Individual versions/machines as described by relationships above dictate whats shown in the application.
if print_logs:
_exit_stdout_logging(console_handler)
return db_machines
def validate_new_image(image_id, machine_request_id):
if not getattr(settings, 'ENABLE_IMAGE_VALIDATION', True):
celery_logger.warn("Skip validation: ENABLE_IMAGE_VALIDATION is False")
return True
machine_request = MachineRequest.objects.get(id=machine_request_id)
new_status, _ = StatusType.objects.get_or_create(name="validating")
machine_request.status = new_status
machine_request.old_status = 'validating'
local_username = machine_request.created_by.username #NOTE: Change local_username accordingly when this assumption is no longer true.
machine_request.save()
accounts = get_account_driver(machine_request.new_machine.provider)
accounts.clear_cache()
from service.instance import launch_machine_instance
admin_driver = accounts.admin_driver
admin_ident = machine_request.new_admin_identity()
if not admin_driver:
celery_logger.warn(
"Need admin_driver functionality to auto-validate instance"
)
return False
if not admin_ident:
celery_logger.warn(
"Need to know the AccountProvider to auto-validate instance"
)
return False
# Attempt to launch using the admin_driver
user = admin_ident.created_by
admin_driver.identity.user = user
machine = admin_driver.get_machine(image_id)
sorted_sizes = admin_driver.list_sizes()
size_index = 0
while size_index < len(sorted_sizes):
selected_size = sorted_sizes[size_index]
size_index += 1
try:
instance = launch_machine_instance(
admin_driver,
user,
admin_ident,
machine,
selected_size,
'Automated Image Verification - %s' % image_id,
username=local_username,
using_admin=True
)
return instance.provider_alias
except BaseHTTPError as http_error:
if "Flavor's disk is too small for requested image" in http_error.message:
continue
logger.exception(http_error)
raise
except Exception as exc:
logger.exception(exc)
raise
# End of while loop
raise Exception("Validation of new Image %s has *FAILED*" % image_id)
def create_first_identity(username, provider=None):
from service.driver import get_account_driver
user = AtmosphereUser.objects.get(username=username)
if not provider:
provider = get_available_provider()
if not provider:
raise Exception("No currently active providers -- Could not create First identity")
accounts = get_account_driver(provider)
new_identity = accounts.create_account(user.username)
return new_identity
def add_os_project_network(core_identity, *args, **kwargs):
try:
logger.debug("add_os_project_network task started at %s." %
datetime.now())
account_driver = get_account_driver(core_identity.provider)
account_driver.create_network(core_identity)
logger.debug("add_os_project_network task finished at %s." %
datetime.now())
except Exception as exc:
add_os_project_network.retry(exc=exc)
def monitor_volumes_for(provider_id, print_logs=False):
"""
Run the set of tasks related to monitoring sizes for a provider.
Optionally, provide a list of usernames to monitor
While debugging, print_logs=True can be very helpful.
start_date and end_date allow you to search a 'non-standard' window of time.
"""
from service.driver import get_account_driver
from core.models import Identity
if print_logs:
console_handler = _init_stdout_logging()
provider = Provider.objects.get(id=provider_id)
account_driver = get_account_driver(provider)
# Non-End dated volumes on this provider
db_volumes = Volume.objects.filter(only_current_source(), instance_source__provider=provider)
all_volumes = account_driver.admin_driver.list_all_volumes(timeout=30)
seen_volumes = []
for cloud_volume in all_volumes:
try:
core_volume = convert_esh_volume(cloud_volume, provider_uuid=provider.uuid)
seen_volumes.append(core_volume)
except ObjectDoesNotExist:
tenant_id = cloud_volume.extra['object']['os-vol-tenant-attr:tenant_id']
tenant = account_driver.get_project_by_id(tenant_id)
tenant_name = tenant.name if tenant else tenant_id
try:
if not tenant:
celery_logger.warn("Warning: tenant_id %s found on volume %s, but did not exist from the account driver perspective.", tenant_id, cloud_volume)
raise ObjectDoesNotExist()
identity = Identity.objects.filter(
contains_credential('ex_project_name', tenant_name), provider=provider
).first()
if not identity:
raise ObjectDoesNotExist()
core_volume = convert_esh_volume(
cloud_volume,
provider.uuid, identity.uuid,
identity.created_by)
except ObjectDoesNotExist:
celery_logger.info("Skipping Volume %s - No Identity for: Provider:%s + Project Name:%s" % (cloud_volume.id, provider, tenant_name))
pass
now_time = timezone.now()
needs_end_date = [volume for volume in db_volumes if volume not in seen_volumes]
for volume in needs_end_date:
celery_logger.debug("End dating inactive volume: %s" % volume)
volume.end_date = now_time
volume.save()
if print_logs:
_exit_stdout_logging(console_handler)
for vol in seen_volumes:
vol.esh = None
return [vol.instance_source.identifier for vol in seen_volumes]
def monitor_volumes_for(provider_id, print_logs=False):
"""
Run the set of tasks related to monitoring sizes for a provider.
Optionally, provide a list of usernames to monitor
While debugging, print_logs=True can be very helpful.
start_date and end_date allow you to search a 'non-standard' window of time.
"""
from service.driver import get_account_driver
from core.models import Identity
if print_logs:
console_handler = _init_stdout_logging()
provider = Provider.objects.get(id=provider_id)
account_driver = get_account_driver(provider)
# Non-End dated volumes on this provider
db_volumes = Volume.objects.filter(only_current_source(), instance_source__provider=provider)
all_volumes = account_driver.admin_driver.list_all_volumes(timeout=30)
seen_volumes = []
for cloud_volume in all_volumes:
try:
core_volume = convert_esh_volume(cloud_volume, provider_uuid=provider.uuid)
seen_volumes.append(core_volume)
except ObjectDoesNotExist:
tenant_id = cloud_volume.extra['object']['os-vol-tenant-attr:tenant_id']
tenant = account_driver.get_project_by_id(tenant_id)
tenant_name = tenant.name if tenant else tenant_id
try:
if not tenant:
celery_logger.warn("Warning: tenant_id %s found on volume %s, but did not exist from the account driver perspective.", tenant_id, cloud_volume)
raise ObjectDoesNotExist()
identity = Identity.objects.filter(
contains_credential('ex_project_name', tenant_name), provider=provider
).first()
if not identity:
raise ObjectDoesNotExist()
core_volume = convert_esh_volume(
cloud_volume,
provider.uuid, identity.uuid,
identity.created_by)
except ObjectDoesNotExist:
celery_logger.info("Skipping Volume %s - No Identity for: Provider:%s + Project Name:%s" % (cloud_volume.id, provider, tenant_name))
pass
now_time = timezone.now()
needs_end_date = [volume for volume in db_volumes if volume not in seen_volumes]
for volume in needs_end_date:
celery_logger.debug("End dating inactive volume: %s" % volume)
volume.end_date = now_time
volume.save()
if print_logs:
_exit_stdout_logging(console_handler)
for vol in seen_volumes:
vol.esh = None
return [vol.instance_source.identifier for vol in seen_volumes]
def validate_new_provider(new_provider, new_identity):
acct_driver = get_account_driver(new_provider)
if not acct_driver:
print "Could not create an account driver for the new Provider"\
" %s - %s. Check your credentials and try again. "\
"If you believe you are receiving this message in error, "\
"AND you are able to use external CLI tools on this machine "\
"to contact your cloud, please report the issue to a developer!"\
% (new_provider, new_identity)
return False
return True
def validate_new_provider(new_provider, new_identity):
acct_driver = get_account_driver(new_provider)
if not acct_driver:
print "Could not create an account driver for the new Provider"\
" %s - %s. Check your credentials and try again. "\
"If you believe you are receiving this message in error, "\
"AND you are able to use external CLI tools on this machine "\
"to contact your cloud, please report the issue to a developer!"\
% (new_provider, new_identity)
return False
return True
def remove_membership(image_version, group, accounts=None):
"""
This function will remove *all* users in the group
to *all* providers/machines using this image_version
"""
for provider_machine in image_version.machines.filter(
only_current_source()):
prov = provider_machine.instance_source.provider
if not accounts:
accounts = get_account_driver(prov)
if not accounts:
raise NotImplemented("Account Driver could not be created for %s" %
prov)
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
raise NotImplemented("Admin Driver could not be created for %s" %
prov)
img = accounts.get_image(provider_machine.identifier)
approved_projects = accounts.get_image_members(img.id)
for identity_membership in group.identitymembership_set.order_by(
'identity__created_by__username'):
if identity_membership.identity.provider != prov:
continue
# Get project name from the identity's credential-list
project_name = identity_membership.identity.get_credential(
'ex_project_name')
project = accounts.get_project(project_name)
if project and project not in approved_projects:
continue
# Perform a *DATABASE* remove first.
application = provider_machine.application
application_version = provider_machine.application_version
models.ApplicationMembership.objects.filter(
group=group, application=application).delete()
logger.info("Removed ApplicationMembership: %s-%s" %
(application, group))
models.ApplicationVersionMembership.objects.filter(
group=group, image_version=application_version).delete()
logger.info("Removed ApplicationVersionMembership: %s-%s" %
(application_version, group))
models.ProviderMachineMembership.objects.filter(
group=group, provider_machine=provider_machine).delete()
logger.info("Removed ProviderMachineMembership: %s-%s" %
(provider_machine, group))
# Perform a *CLOUD* remove last.
try:
accounts.image_manager.unshare_image(img, project_name)
except Exception as exc:
logger.exception(
"Exception occurred while removing user from cloud: %s",
exc)
logger.info("Removed Cloud Access: %s-%s" % (img, project_name))
return
def _set_compute_quota(user_quota, identity):
# Use THESE values...
compute_values = {
'cores': user_quota.cpu,
'ram':
user_quota.memory *
1024, # NOTE: Value is stored in GB, Openstack (Liberty) expects MB
'floating_ips': user_quota.floating_ip_count,
'fixed_ips': user_quota.port_count,
'instances': user_quota.instance_count,
'force': True
}
creds = identity.get_all_credentials()
use_tenant_id = False
if creds.get('ex_force_auth_version', '2.0_password') == "2.0_password":
compute_values.pop('instances')
use_tenant_id = True
username = identity.created_by.username
logger.info("Updating quota for %s to %s" % (username, compute_values))
driver = get_cached_driver(identity=identity)
username = driver._connection.key
ad = get_account_driver(identity.provider, raise_exception=True)
tenant = ad.get_project(username)
ks_user = ad.get_user(username)
admin_driver = ad.admin_driver
creds = identity.get_all_credentials()
if creds.get('ex_force_auth_version', '2.0_password') != "2.0_password":
# FIXME: Remove 'use_tenant_id' when legacy clouds are no-longer in use.
try:
result = admin_driver._connection.ex_update_quota(
tenant.id, compute_values, use_tenant_id=use_tenant_id
)
except Exception:
logger.exception(
"Could not set a user-quota, trying to set tenant-quota"
)
raise
else:
# For CyVerse old clouds, run the top method. don't use try/except.
try:
result = admin_driver._connection.ex_update_quota_for_user(
tenant.id,
ks_user.id,
compute_values,
use_tenant_id=use_tenant_id
)
except Exception:
logger.exception(
"Could not set a user-quota, trying to set tenant-quota"
)
raise
logger.info("Updated quota for %s to %s" % (username, result))
return result
def glance_write_machine(provider_machine):
"""
Using the provider_machine in the DB, save information to the Cloud.
"""
update_method = ""
base_source = provider_machine.instance_source
provider = base_source.provider
base_app = provider_machine.application
version = provider_machine.application_version
identifier = base_source.identifier
accounts = get_account_driver(provider)
g_image = glance_image_for(provider.uuid, identifier)
app_version_bundle_name = provider_machine.generated_name()
if not g_image:
return
if hasattr(g_image, 'properties'):
props = g_image.properties
update_method = 'v2'
elif hasattr(g_image, 'items'):
props = dict(g_image.items())
update_method = 'v3'
else:
raise Exception(
"The method for 'introspecting an image' has changed!"
" Ask a programmer to fix this!")
# Do any updating that makes sense... Name. Metadata..
blacklist_metadata_key = getattr(settings, "BLACKLIST_METADATA_KEY", "atmo_image_exclude")
whitelist_metadata_key = getattr(settings, "WHITELIST_METADATA_KEY", "atmo_image_include")
overrides = {
"application_uuid": str(base_app.uuid),
"application_name": _make_safe(base_app.name),
"application_owner": base_app.created_by.username,
"application_tags": json.dumps(
[_make_safe(tag.name) for tag in base_app.tags.all()]),
"application_description": _make_safe(base_app.description),
"version_name": str(version.name),
"version_changelog": str(version.change_log),
whitelist_metadata_key: "true",
blacklist_metadata_key: "false"
}
if update_method == 'v2':
extras = {
'properties': overrides
}
extras['name'] = app_version_bundle_name
props.update(extras)
g_image.update(props)
else:
overrides['name'] = app_version_bundle_name
accounts.image_manager.glance.images.update(
g_image.id, **overrides)
return True
def create_accounts(self, provider, username, force=False):
from service.driver import get_account_driver
from core.models import Project, Identity
credentials_list = self.get_credentials_list(provider, username)
identities = Identity.objects.none()
for credentials in credentials_list:
try:
project_name = credentials['project_name']
created_identities = self.find_accounts(provider, **credentials)
if created_identities and not force:
# logger.debug(
# "Accounts already created for %s on provider %s", username, provider)
identities |= created_identities
continue
logger.debug(
"Creating new account for %s with credentials - %s" %
(username, credentials)
)
account_driver = get_account_driver(provider)
if not account_driver:
raise ValueError(
"Provider %s produced an invalid account driver "\
"-- Use plugin after you create a core.Provider "\
"*AND* assign a core.Identity to be the core.AccountProvider."
% provider)
new_identity = account_driver.create_account(**credentials)
identities |= Identity.objects.filter(id=new_identity.id)
memberships = new_identity.identity_memberships.filter(
member__memberships__is_leader=True
)
if not memberships:
memberships = new_identity.identity_memberships.all()
membership = memberships.first()
if not membership:
raise ValueError(
"Expected at least one member in identity %s" %
new_identity
)
group = membership.member
try:
Project.objects.get(name=project_name, owner=group)
except Project.DoesNotExist:
Project.objects.create(
name=project_name,
created_by=new_identity.created_by,
owner=group
)
except:
logger.exception(
"Could *NOT* Create NEW account for %s" % username
)
return identities
def remove_membership(image_version, group, accounts=None):
"""
This function will remove *all* users in the group
to *all* providers/machines using this image_version
"""
for provider_machine in image_version.machines.filter(only_current_source()):
prov = provider_machine.instance_source.provider
if not accounts:
accounts = get_account_driver(prov)
if not accounts:
raise NotImplemented("Account Driver could not be created for %s" % prov)
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
raise NotImplemented("Admin Driver could not be created for %s" % prov)
img = accounts.get_image(provider_machine.identifier)
approved_projects = accounts.get_image_members(img.id)
for identity_membership in group.identitymembership_set.order_by('identity__created_by__username'):
if identity_membership.identity.provider != prov:
continue
# Get project name from the identity's credential-list
project_name = identity_membership.identity.get_credential(
'ex_project_name')
project = accounts.get_project(project_name)
if project and project not in approved_projects:
continue
# Perform a *DATABASE* remove first.
application = provider_machine.application
application_version = provider_machine.application_version
models.ApplicationMembership.objects.filter(
group=group,
application=application).delete()
logger.info("Removed ApplicationMembership: %s-%s"
% (application, group))
models.ApplicationVersionMembership.objects.filter(
group=group,
image_version=application_version).delete()
logger.info("Removed ApplicationVersionMembership: %s-%s"
% (application_version, group))
models.ProviderMachineMembership.objects.filter(
group=group,
provider_machine=provider_machine).delete()
logger.info("Removed ProviderMachineMembership: %s-%s"
% (provider_machine, group))
# Perform a *CLOUD* remove last.
try:
accounts.image_manager.unshare_image(img, project_name)
except Exception as exc:
logger.exception("Exception occurred while removing user from cloud: %s", exc)
logger.info("Removed Cloud Access: %s-%s"
% (img, project_name))
return
def glance_image_owner(provider_uuid, identifier):
try:
prov = Provider.objects.get(uuid=provider_uuid)
accounts = get_account_driver(prov)
glance_image = accounts.get_image(identifier)
project = accounts.user_manager.get_project_by_id(glance_image.owner)
image_owner = Identity.objects.get(
provider__uuid=provider_uuid,
created_by__username=project.name)
except Exception as e:
logger.exception(e)
image_owner = None
return image_owner
def add_membership(image_version, group):
"""
This function will add *all* users in the group
to *all* providers/machines using this image_version
O(N^2)
"""
for provider_machine in image_version.machines.filter(
only_current_source()):
prov = provider_machine.instance_source.provider
accounts = get_account_driver(prov)
if not accounts:
raise NotImplemented("Account Driver could not be created for %s" %
prov)
accounts.clear_cache()
admin_driver = accounts.admin_driver # cache has been cleared
if not admin_driver:
raise NotImplemented("Admin Driver could not be created for %s" %
prov)
img = accounts.get_image(provider_machine.identifier)
projects = get_current_projects_for_image(accounts, img.id)
for identity_membership in group.identitymembership_set.all():
if identity_membership.identity.provider != prov:
continue
# Get project name from the identity's credential-list
project_name = identity_membership.identity.get_credential(
'ex_project_name')
project = accounts.get_project(project_name)
if project and project in projects:
continue
# Share with the *database* first!
obj, created = models.ApplicationMembership.objects.get_or_create(
group=group, application=provider_machine.application)
if created:
print "Created new ApplicationMembership: %s" \
% (obj,)
obj, created = models.ApplicationVersionMembership.objects.get_or_create(
group=group,
image_version=provider_machine.application_version)
if created:
print "Created new ApplicationVersionMembership: %s" \
% (obj,)
obj, created = models.ProviderMachineMembership.objects.get_or_create(
group=group, provider_machine=provider_machine)
if created:
print "Created new ProviderMachineMembership: %s" \
% (obj,)
# Share with the *cloud* last!
accounts.image_manager.share_image(img, project_name)
accounts.accept_shared_image(img, project_name)
logger.info("Added Cloud Access: %s-%s" % (img, project_name))
continue
def _get_hard_limits(identity):
"""
Lookup the OpenStack "Hard Limits" based on the account provider
"""
accounts = get_account_driver(identity.provider)
defaults = {"ram": 999, "cpu": 99} # Used when all else fails.
limits = {}
limits.update(defaults)
username = identity.get_credential('key')
project_name = identity.get_credential('ex_project_name')
user_limits = accounts.get_quota_limit(username, project_name)
if user_limits:
limits.update(user_limits)
return limits
def _set_volume_quota(user_quota, identity):
volume_values = {
'volumes': user_quota.storage_count,
'gigabytes': user_quota.storage,
'snapshots': user_quota.snapshot_count,
}
username = identity.created_by.username
logger.info("Updating quota for %s to %s" % (username, volume_values))
driver = get_cached_driver(identity=identity)
username = driver._connection._get_username()
ad = get_account_driver(identity.provider)
admin_driver = ad.admin_driver
admin_driver._connection._cinder_update_quota(username, volume_values)
return
def glance_update_machine_metadata(provider_machine, metadata={}):
update_method = ""
base_source = provider_machine.instance_source
provider = base_source.provider
base_app = provider_machine.application
version = provider_machine.application_version
identifier = base_source.identifier
accounts = get_account_driver(provider)
g_image = glance_image_for(base_source.provider.uuid, identifier)
if not g_image:
return False
if hasattr(g_image, 'properties'):
props = g_image.properties
update_method = 'v2'
elif hasattr(g_image, 'items'):
props = dict(g_image.items())
update_method = 'v3'
else:
raise Exception(
"The method for 'introspecting an image' has changed!"
" Ask a programmer to fix this!"
)
overrides = {
"application_version":
str(version.name),
"application_uuid":
str(base_app.uuid),
"application_name":
_make_safe(base_app.name),
"application_owner":
base_app.created_by.username,
"application_tags":
json.dumps([_make_safe(tag.name) for tag in base_app.tags.all()]),
"application_description":
_make_safe(base_app.description),
"version_name":
str(version.name),
"version_changelog":
str(version.change_log)
}
overrides.update(metadata)
if update_method == 'v2':
extras = {'properties': overrides}
props.update(extras)
g_image.update(name=base_app.name, properties=extras)
else:
accounts.image_manager.glance.images.update(g_image.id, **overrides)
return True
def _set_volume_quota(user_quota, identity):
volume_values = {
'volumes': user_quota.storage_count,
'gigabytes': user_quota.storage,
'snapshots': user_quota.snapshot_count,
}
username = identity.created_by.username
logger.info("Updating quota for %s to %s" % (username, volume_values))
ad = get_account_driver(identity.provider)
admin_driver = ad.admin_driver
tenant = ad.get_project(username)
result = admin_driver._connection._cinder_update_quota(
tenant.id, volume_values
)
logger.info("Updated quota for %s to %s" % (username, result))
return result
def create_new_account_for(provider, user):
from service.driver import get_account_driver
existing_user_list = provider.identity_set.values_list(
'created_by__username', flat=True)
if user.username in existing_user_list:
logger.info("Accounts already exists on %s for %s" %
(provider.location, user.username))
return None
try:
accounts = get_account_driver(provider)
logger.info("Create NEW account for %s" % user.username)
new_identity = accounts.create_account(user.username)
return new_identity
except:
logger.exception("Could *NOT* Create NEW account for %s" %
user.username)
return None
def remove_empty_networks_for(provider_id):
provider = Provider.objects.get(id=provider_id)
os_driver = get_account_driver(provider)
if not os_driver:
celery_logger.warn(
"Cannot remove_empty_networks_for provider %s -- Account Driver not created"
% provider)
return
all_instances = os_driver.admin_driver.list_all_instances()
project_map = os_driver.network_manager.project_network_map()
known_project_names = Credential.objects.filter(
key='ex_project_name').values_list('value', flat=True)
projects_with_networks = sorted(
[k for k in project_map.keys() if k in known_project_names])
for project in projects_with_networks:
networks = project_map[project]['network']
if not isinstance(networks, list):
networks = [networks]
for network in networks:
network_name = network['name']
celery_logger.debug("Checking if network %s is in use" %
network_name)
if running_instances(network_name, all_instances):
continue
user = project
identity = Identity.objects.filter(
provider_id=provider_id,
credential__key='ex_project_name',
credential__value=project).filter(
credential__key='key', credential__value=user).first()
if not identity:
celery_logger.warn(
"NOT Removing project network for User:%s, Project:%s -- No Valid Identity found!"
% (user, project))
continue
try:
celery_logger.debug(
"Removing project network for User:%s, Project:%s" %
(user, project))
os_driver.delete_user_network(identity)
except NeutronClientException:
celery_logger.exception("Neutron unable to remove project"
"network for %s-%s" % (user, project))
except NeutronException:
celery_logger.exception("Neutron unable to remove project"
"network for %s-%s" % (user, project))
def get_public_and_private_apps(provider):
"""
INPUT: Provider provider
OUTPUT: 2-tuple (
new_public_apps [],
private_apps(key) + super-set-membership(value) {})
"""
account_driver = get_account_driver(provider)
all_projects_map = tenant_id_to_name_map(account_driver)
cloud_machines = account_driver.list_all_images()
db_machines = ProviderMachine.objects.filter(
only_current_source(), instance_source__provider=provider)
new_public_apps = []
private_apps = {}
# ASSERT: All non-end-dated machines in the DB can be found in the cloud
# if you do not believe this is the case, you should call 'prune_machines_for'
for cloud_machine in cloud_machines:
#Filter out: ChromoSnapShot, eri-, eki-, ... (Or dont..)
if any(
cloud_machine.name.startswith(prefix)
for prefix in ['eri-', 'eki-', 'ChromoSnapShot']):
#celery_logger.debug("Skipping cloud machine %s" % cloud_machine)
continue
app_name, version_name = ProviderMachine._split_cloud_name(
cloud_machine.name)
db_machine = get_or_create_provider_machine(cloud_machine.id,
app_name,
provider.uuid,
version_name=version_name)
db_version = db_machine.application_version
db_application = db_version.application
if cloud_machine.get('visibility') == 'public':
if db_application.private and db_application not in new_public_apps:
new_public_apps.append(db_application) #Distinct list..
#Else the db app is public and no changes are necessary.
else:
# cloud machine is private
membership = get_shared_identities(account_driver, cloud_machine,
all_projects_map)
all_members = private_apps.get(db_application, [])
all_members.extend(membership)
#Distinct list..
private_apps[db_application] = all_members
return new_public_apps, private_apps
