def update_image_membership(account_driver, cloud_machine, db_machine): """ Given a cloud_machine and db_machine, create any relationships possible for ProviderMachineMembership and ApplicationVersionMembership """ image_visibility = cloud_machine.get('visibility','private') if image_visibility.lower() == 'public': return image_owner = cloud_machine.get('application_owner','') #TODO: In a future update to 'imaging' we might image 'as the user' rather than 'as the admin user', in this case we should just use 'owner' metadata shared_group_names = [image_owner] shared_projects = account_driver.shared_images_for(cloud_machine.id, None) has_machine_request = db_machine.application_version.machinerequest_set.first() if has_machine_request and has_machine_request.status.name == 'completed': provider = has_machine_request.new_machine_provider identifier = has_machine_request.new_machine.identifier main_account_driver = get_account_driver(provider) shared_projects_from_main = main_account_driver.shared_images_for(identifier, None) shared_group_names.extend(p.name for p in shared_projects_from_main if p) shared_group_names.extend(p.name for p in shared_projects if p) groups = Group.objects.filter(name__in=shared_group_names) if not groups: return for group in groups: update_db_membership_for_group(db_machine, group)
def __init__(self, identity, network_driver, neutron): self.account_driver = get_account_driver(identity.provider) if not self.account_driver: raise Exception( "ConfigError: Cannot use ExternalRouter topology without an " "AccountDriver. Please add an AccountProvider linked to an " "Identity who has the 'admin' role on Openstack " "_or_ use ExternalNetworkTopology to continue" ) router_name = identity.get_credential('router_name') if not router_name: router_name = identity.provider.get_credential('router_name') if not router_name: router_name = identity.provider.select_router() from core.models import Credential Credential.objects.get_or_create( identity=identity, key='router_name', value=router_name ) if not router_name: raise Exception( "Unknown Router - Identity %s is missing 'router_name' " % identity ) self.external_router_name = router_name return super(ExternalRouter, self).__init__(identity, network_driver, neutron)
def redeploy_users(provider, users=[]): accounts = get_account_driver(provider) tenant_instances_map = accounts.tenant_instances_map(status_list=['deploy_error', 'networking','deploying','initializing']) for tenant, instance_list in tenant_instances_map.iteritems(): username = tenant.name if users and username not in users: print "Found affected user:%s and Instances:%s - Skipping because they aren't in the list." % (username, instance_list) continue for instance in instance_list: metadata = instance._node.extra.get('metadata',{}) instance_status = instance.extra.get('status') tmp_status = metadata.get('tmp_status','') print "Starting idempotent redeployment for %s - Instance: %s (%s - %s)" % (username, instance.id, instance_status, tmp_status) ident = Identity.objects.get(provider=provider, created_by__username=username) driver = get_esh_driver(ident) try: start_task = get_idempotent_deploy_chain(driver.__class__, driver.provider, driver.identity, instance, username) print "Starting idempotent redeployment: %s ..." % (start_task), start_task.apply_async() except Identity.DoesNotExist: print "Identity does not exist in this DB. SKIPPED." continue if DO_NOTHING: continue print " Sent"
def main(): parser = argparse.ArgumentParser() parser.add_argument("--provider", type=int, help="Atmosphere provider ID" " to use when importing users.") parser.add_argument("--provider-id", type=int, help="Atmosphere provider ID" " to use when importing users." " DEPRECATION WARNING -- THIS WILL BE REMOVED SOON!") parser.add_argument("--provider-list", action="store_true", help="List of provider names and IDs") parser.add_argument("--rebuild", action="store_true", help="Rebuild all accounts that are in the provider") parser.add_argument("--users", help="LDAP usernames to import. (comma separated)") parser.add_argument("--admin", action="store_true", help="Users addded as admin and staff users.") args = parser.parse_args() if args.provider_list: print "ID\tName" for p in Provider.objects.all().order_by('id'): print "%d\t%s" % (p.id, p.location) return users = None if args.provider_id and not args.provider: print "WARNING: --provider-id has been *DEPRECATED*! Use --provider instead!" args.provider = args.provider_id if args.provider: provider = Provider.objects.get(id=args.provider) else: raise Exception( "Missing required argument: --provider <id>. use --provider-list to get a list of provider ID+names" ) print "Using Provider: %s" % provider type_name = provider.type.name.lower() try: acct_driver = get_account_driver(provider, raise_exception=True) except: account_provider = provider.accountprovider_set.first() print "Could not create the account Driver for this Provider."\ " Check the configuration of this identity:%s" % account_provider raise if not args.users: if not args.rebuild: print "Retrieving all 'atmo-user' members in LDAP." users = get_members('atmo-user') else: print "Rebuilding all existing users." users = get_usernames(provider) else: users = args.users.split(",") return create_accounts(acct_driver, provider, users, args.rebuild, args.admin)
def _get_instance_owner_map(provider, users=None): """ All keys == All identities Values = List of identities / username NOTE: This is KEYSTONE && NOVA specific. the 'instance owner' here is the username // ex_tenant_name """ from service.driver import get_account_driver admin_driver = get_cached_driver(provider=provider) accounts = get_account_driver(provider=provider) all_identities = _select_identities(provider, users) acct_providers = AccountProvider.objects.filter(provider=provider) if acct_providers: account_identity = acct_providers[0].identity provider = None else: account_identity = None all_instances = get_cached_instances(provider=provider, identity=account_identity, force=True) #all_tenants = admin_driver._connection._keystone_list_tenants() all_tenants = accounts.list_projects() # Convert instance.owner from tenant-id to tenant-name all at once all_instances = _convert_tenant_id_to_names(all_instances, all_tenants) # Make a mapping of owner-to-instance instance_map = _make_instance_owner_map(all_instances, users=users) logger.info("Instance owner map created") identity_map = _include_all_idents(all_identities, instance_map) logger.info("Identity map created") return identity_map
def update_application_owner(application, identity): from service.openstack import glance_update_machine_metadata from service.driver import get_account_driver old_identity = application.created_by_identity tenant_name = _extract_tenant_name(identity) old_tenant_name = _extract_tenant_name(old_identity) # Prepare the application application.created_by_identity = identity application.created_by = identity.created_by application.save() # Update all the PMs all_pms = application.providermachine_set.all() print "Updating %s machines.." % len(all_pms) for provider_machine in all_pms: accounts = get_account_driver(provider_machine.provider) image_id = provider_machine.instance_source.identifier image = accounts.get_image(image_id) if not image: continue tenant_id = accounts.get_project(tenant_name).id glance_update_machine_metadata(provider_machine, {"owner": tenant_id, "application_owner": tenant_name}) print "App data saved for %s" % image_id accounts.image_manager.share_image(image, tenant_name) print "Shared access to %s with %s" % (image_id, tenant_name) accounts.image_manager.unshare_image(image, old_tenant_name) print "Removed access to %s for %s" % (image_id, old_tenant_name)
def _set_compute_quota(user_quota, identity): # Use THESE values... compute_values = { 'cores': user_quota.cpu, 'ram': user_quota.memory*1024, # NOTE: Value is stored in GB, Openstack (Liberty) expects MB 'floating_ips': user_quota.floating_ip_count, 'fixed_ips': user_quota.port_count, 'instances': user_quota.instance_count, } creds = identity.get_all_credentials() if creds.get('ex_force_auth_version','2.0_password') == "2.0_password": compute_values.pop('instances') username = identity.created_by.username logger.info("Updating quota for %s to %s" % (username, compute_values)) driver = get_cached_driver(identity=identity) username = driver._connection.key tenant_id = driver._connection._get_tenant_id() tenant_name = identity.project_name() ad = get_account_driver(identity.provider) ks_user = ad.get_user(username) admin_driver = ad.admin_driver try: result = admin_driver._connection.ex_update_quota_for_user( tenant_id, ks_user.id, compute_values) except Exception: logger.exception("Could not set a user-quota, trying to set tenant-quota") result = admin_driver._connection.ex_update_quota(tenant_id, compute_values) logger.info("Updated quota for %s to %s" % (username, result)) return result
def create_new_account_for(provider, user): from service.exceptions import AccountCreationConflict from service.driver import get_account_driver existing_user_list = provider.identity_set.values_list( 'created_by__username', flat=True ) if user.username in existing_user_list: logger.info( "Accounts already exists on %s for %s" % (provider.location, user.username) ) return None try: accounts = get_account_driver(provider) logger.info("Create NEW account for %s" % user.username) default_quota = DefaultQuotaPluginManager.default_quota( user=user, provider=provider ) new_identity = accounts.create_account( user.username, quota=default_quota ) return new_identity except AccountCreationConflict: raise # TODO: Ideally, have sentry handle these events, rather than force an Unhandled 500 to bubble up. except: logger.exception( "Could *NOT* Create NEW account for %s" % user.username ) return None
def upload_privacy_data(machine_request, new_machine): """ ASSERT: The image in 'new_machine' SHOULD BE private (Based on values in machine_request) """ prov = new_machine.provider accounts = get_account_driver(prov) if not accounts: print "Aborting import: Could not retrieve Account Driver "\ "for Provider %s" % prov return img = accounts.get_image(new_machine.identifier) if hasattr(img, 'visibility'): # Treated as an obj. is_public = img.visibility == 'public' elif hasattr(img, 'items'): # Treated as a dict. is_public = img.get('visibility','N/A') == 'public' if is_public: print "Marking image %s private" % img.id accounts.image_manager.update_image(img, visibility='private') accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: print "Aborting import: Could not retrieve admin_driver "\ "for Provider %s" % prov return img = accounts.get_image(new_machine.identifier) tenant_list = machine_request.get_access_list() # All in the list will be added as 'sharing' the OStack img # All tenants already sharing the OStack img will be added to this list return sync_machine_membership(accounts, img, new_machine, tenant_list)
def fix_requests(provider, requests=[]): accounts = get_account_driver(provider) for request_id in requests: try: machine_request = MachineRequest.objects.get(id=request_id) except MachineRequest.DoesNotExist: print "Warn: MachineRequest by this ID could not be found: %s" % request_id continue if machine_request.new_machine_provider != provider: raise ValueError( "MachineRequest ID:%s is for Provider:%s. Testing Provider is:%s" % (request_id, machine_request.new_machine_provider, provider)) fixed = False try: new_machine = ProviderMachine.objects.get( id=machine_request.new_machine_id) except ProviderMachine.DoesNotExist as no_match: print "OK: This MachineRequest has a BAD 'new_machine' (DoesNotExist)" new_machine = None if not fixed: fixed = _fix_wrong_machine_on_request( machine_request, provider, new_machine) if not fixed and new_machine: _fix_new_version_forked(machine_request, provider, new_machine)
def _fix_new_machine_forked(machine_request, provider, new_machine): app_uuid = _generate_app_uuid(new_machine.identifier) if not machine_request.new_version_forked: return False if Application.objects.filter(uuid=app_uuid).count(): return False print "OK: This MachineRequest: %s has a BAD Application." \ "\tUUID should be %s." % (machine_request, app_uuid) old_application = new_machine.application current_application = _create_new_application( machine_request, new_machine.identifier) remaining_machines = old_application._current_machines() for machine in remaining_machines: if machine.identifier == new_machine.identifier: new_machine.application = current_application new_machine.save() current_application.save() # Pass #2 - If remaining, unmatched ids: remaining_machines = old_application._current_machines() acct_provider = machine_request.new_machine_provider accounts = get_account_driver(acct_provider) if remaining_machines: print "Warn: These machines likely point to the wrong application.:%s" % remaining_machines for machine in remaining_machines: glance_image = accounts.image_manager.get_image(machine.identifier) if glance_image: original_request = MachineRequest.objects.filter( new_application_name=glance_image.name) print "Hint: Image_ID:%s Named:%s MachineRequest:%s" % (glance_image.id, glance_image.name, original_request) return True
def print_instances(provider, users=[], status_list=[]): accounts = get_account_driver(provider) tenant_instances_map = accounts.tenant_instances_map( status_list=status_list, match_all=MATCH_ALL) for tenant, instance_list in tenant_instances_map.iteritems(): username = tenant.name if users and username not in users: continue for instance in instance_list: instance_status = instance.extra.get('status') task = instance.extra.get('task') metadata = instance.extra.get('metadata', {}) tmp_status = metadata.get('tmp_status', '') created = instance.extra.get('created', "N/A") updated = instance.extra.get('updated', "N/A") status_name = _get_status_name_for_provider( provider, instance_status, task, tmp_status) try: last_history = Instance.objects.get( provider_alias=instance.id).get_last_history() except: last_history = "N/A (Instance not in this DB)" print "Tenant:%s Instance:%s Status: (%s - %s) Created:%s Updated:%s, Last History:%s" % ( username, instance.id, instance_status, tmp_status, created, updated, last_history)
def update_cloud_membership_for_machine(provider_machine, group): """ Given a provider_machine and a group * Loop through identities owned by group * * If identity.provider == provider_machine.provider, allow identity to launch via cloud ACLs """ prov = provider_machine.instance_source.provider accounts = get_account_driver(prov) if not accounts: raise NotImplemented("Account Driver could not be created for %s" % prov) accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: raise NotImplemented("Admin Driver could not be created for %s" % prov) img = accounts.get_image(provider_machine.identifier) if img.visibility == 'public': return approved_projects = accounts.shared_images_for(img.id) for identity_membership in group.identitymembership_set.all(): if identity_membership.identity.provider != prov: logger.debug("Skipped %s -- Wrong provider" % identity_membership.identity) continue # Get project name from the identity's credential-list identity = identity_membership.identity project_name = identity.get_credential('ex_project_name') project = accounts.get_project(project_name) if not project: logger.debug("Unknown Project: %s -- Does not exist" % project) continue elif project in approved_projects: logger.debug("Skipped Project: %s -- Already shared" % project) continue accounts.share_image_with_identity(img, identity)
def _get_instance_owner_map(provider, users=None): """ All keys == All identities Values = List of identities / username NOTE: This is KEYSTONE && NOVA specific. the 'instance owner' here is the username // ex_tenant_name """ from service.driver import get_account_driver accounts = get_account_driver(provider=provider, raise_exception=True) all_identities = _select_identities(provider, users) acct_providers = AccountProvider.objects.filter(provider=provider) if acct_providers: account_identity = acct_providers[0].identity provider = None else: account_identity = None all_instances = get_cached_instances( provider=provider, identity=account_identity, force=True ) #all_tenants = admin_driver._connection._keystone_list_tenants() all_tenants = accounts.list_projects() # Convert instance.owner from tenant-id to tenant-name all at once all_instances = _convert_tenant_id_to_names(all_instances, all_tenants) # Make a mapping of owner-to-instance instance_map = _make_instance_owner_map(all_instances, users=users) logger.info("Instance owner map created") identity_map = _include_all_idents(all_identities, instance_map) logger.info("Identity map created") return identity_map
def delete_accounts(self, provider, username): from service.driver import get_account_driver account_driver = get_account_driver(provider) if not account_driver: raise ValueError( "Provider %s produced an invalid account driver "\ "-- Use plugin after you create a core.Provider "\ "*AND* assign a core.Identity to be the core.AccountProvider." % provider) credentials_list = self.get_credentials_list(provider, username) identity_list = [] for credentials in credentials_list: try: identities = self.find_accounts(**credentials) if not identities: continue logger.debug( "Removing account for %s with credentials - %s" % (username, credentials) ) for identity in identities: removed_identity = account_driver.delete_account( identity, **credentials ) identity_list.append(removed_identity) except: logger.exception("Could *NOT* delete account for %s" % username) return identity_list
def validate_account_driver(self, validated_data): try: provider = validated_data['provider'] acct_driver = get_account_driver(provider, raise_exception=True) return acct_driver except Exception as exc: raise serializers.ValidationError("Attempting to create an account for provider %s failed. Message: %s" % (provider, exc.message))
def main(): parser = argparse.ArgumentParser() parser.add_argument("--provider-list", action="store_true", help="List of provider names and IDs") parser.add_argument("--quota-list", action="store_true", help="List of provider names and IDs") parser.add_argument("--provider-id", type=int, help="Atmosphere provider ID" " to use when importing users.") parser.add_argument("--quota-id", help="Atmosphere Quota ID to assign (Optional)") parser.add_argument("--groups", help="LDAP groups to import. (comma separated)") parser.add_argument("--dry-run", action="store_true", help="A 'dry-run' so you know what will happen," " before it happens") parser.add_argument("--users", help="LDAP usernames to import. (comma separated)") parser.add_argument("--admin", action="store_true", help="ALL Users addded are treated as admin and staff " "users. They also receive the maximum quota.") args = parser.parse_args() make_admins = args.admin dry_run = args.dry_run users = None quota = None if args.provider_list: print "ID\tName" for p in Provider.objects.all().order_by('id'): print "%d\t%s" % (p.id, p.location) return elif args.quota_list: print "ID\tSpecs" for q in Quota.objects.all().order_by('id'): print "%s\t%s" % (q.id, q) return #Debugging args if dry_run: print "Dry run initialized.." #Optional args if args.quota_id: quota = Quota.objects.get(id=args.quota_id) if not args.provider_id: print "ERROR: provider-id is required. To get a list of providers use"\ " --provider-list" provider = Provider.objects.get(id=args.provider_id) print "Provider Selected:%s" % provider acct_driver = get_account_driver(provider) groups = args.groups.split(",") if args.groups else [] total_added = process_groups(acct_driver, groups, quota, make_admins, dry_run) users = args.users.split(",") if args.users else [] total_added += process_users(acct_driver, users, quota, make_admins, dry_run) print "Processing complete. %d users processed." % total_added
def main(): parser = argparse.ArgumentParser() parser.add_argument("--dry-run", action="store_true", help="List of provider names and IDs") args = parser.parse_args() admin_owned_apps = Application.objects.filter( created_by__username__contains='admin').filter(only_current_apps()).distinct() account_drivers = {} # FIXME: Change the provider_id if necessary. for app in admin_owned_apps: # Step 1 - See if MachineRequest can answer the question machine = app._current_machines().filter(instance_source__provider__id=4).first() if not machine: continue mr = MachineRequest.objects.filter(new_machine=machine).first() if mr: fix_application_owner(app, mr.created_by, args.dry_run) continue # Step 2 - See if glance can answer the question provider = machine.provider if account_drivers.get(provider): accounts = account_drivers[provider] else: accounts = get_account_driver(provider) account_drivers[provider] = accounts img = accounts.get_image(machine.identifier) if not img: continue project = accounts.get_project_by_id(img.owner) if not project: continue user = AtmosphereUser.objects.filter(username=project.name).first() if user: fix_application_owner(app, user, args.dry_run)
def update_application_owner(application, identity): from service.openstack import glance_update_machine_metadata from service.driver import get_account_driver old_identity = application.created_by_identity tenant_name = _extract_tenant_name(identity) old_tenant_name = _extract_tenant_name(old_identity) # Prepare the application application.created_by_identity = identity application.created_by = identity.created_by application.save() # Update all the PMs all_pms = application.providermachine_set.all() print "Updating %s machines.." % len(all_pms) for provider_machine in all_pms: accounts = get_account_driver(provider_machine.provider) image_id = provider_machine.instance_source.identifier image = accounts.get_image(image_id) if not image: continue tenant_id = accounts.get_project(tenant_name).id glance_update_machine_metadata( provider_machine, {'owner': tenant_id, 'application_owner': tenant_name}) print "App data saved for %s" % image_id accounts.image_manager.share_image(image, tenant_name) print "Shared access to %s with %s" % (image_id, tenant_name) accounts.image_manager.unshare_image(image, old_tenant_name) print "Removed access to %s for %s" % (image_id, old_tenant_name)
def update_cloud_membership_for_machine(provider_machine, group): """ Given a provider_machine and a group * Loop through identities owned by group * * If identity.provider == provider_machine.provider, allow identity to launch via cloud ACLs """ prov = provider_machine.instance_source.provider accounts = get_account_driver(prov) if not accounts: raise NotImplemented("Account Driver could not be created for %s" % prov) accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: raise NotImplemented("Admin Driver could not be created for %s" % prov) img = accounts.get_image(provider_machine.identifier) if img.get('visibility') != 'shared': logger.debug("Skipped updates for image %s -- visibility (%s) is not 'shared'", img.id, img.get('visibility')) return approved_projects = accounts.get_image_members(img.id) for identity_membership in group.identitymembership_set.all(): if identity_membership.identity.provider != prov: logger.debug("Skipped %s -- Wrong provider" % identity_membership.identity) continue # Get project name from the identity's credential-list identity = identity_membership.identity project_name = identity.get_credential('ex_project_name') project = accounts.get_project(project_name) if not project: logger.debug("Unknown Project: %s -- Does not exist" % project) continue elif project in approved_projects: logger.debug("Skipped Project: %s -- Already shared" % project) continue accounts.share_image_with_identity(img, identity)
def post(self, request, cloud_admin_uuid): """ Passes in: Username (To apply the identity to) Credentials (Nested, will be applied to new identity) """ user = request.user data = request.DATA try: provider_uuid = data['provider'] provider = Provider.objects.get( cloudadministrator__user=user, uuid=provider_uuid) except KeyError: return Response( "Missing 'provider' key, Expected UUID. Received no value.", status=status.HTTP_409_conflict) except Exception: return Response( "Provider with UUID %s does not exist" % provider_uuid, status=status.HTTP_409_conflict) raise Exception driver = get_account_driver(provider) missing_args = driver.clean_credentials(data) if missing_args: raise Exception("Cannot create account. Missing credentials: %s" % missing_args) identity = driver.create_account(**data) serializer = IdentitySerializer(identity) # TODO: Account creation SHOULD return IdentityMembership NOT identity. return Response(serializer.data)
def print_instances(provider, users=[], status_list=[]): accounts = get_account_driver(provider) tenant_instances_map = accounts.tenant_instances_map( status_list=status_list, match_all=MATCH_ALL) for tenant, instance_list in tenant_instances_map.iteritems(): username = tenant.name if users and username not in users: continue for instance in instance_list: instance_status = instance.extra.get('status') task = instance.extra.get('task') metadata = instance.extra.get('metadata', {}) tmp_status = metadata.get('tmp_status', '') created = instance.extra.get('created', "N/A") updated = instance.extra.get('updated', "N/A") status_name = _get_status_name_for_provider( provider, instance_status, task, tmp_status) try: last_history = Instance.objects.get( provider_alias=instance.id).get_last_history() except: last_history = "N/A (Instance not in this DB)" print "Tenant:%s Instance:%s Status: (%s - %s) Created:%s Updated:%s, Last History:%s" % (username, instance.id, instance_status, tmp_status, created, updated, last_history)
def _get_all_access_list(account_driver, db_machine, cloud_machine): """ Input: AccountDriver, ProviderMachine, glance_image Output: A list of _all project names_ that should be included on `cloud_machine` This list will include: - Users who match the provider_machine's application.access_list - Users who are already approved to use the `cloud_machine` - The owner of the application/Creator of the MachineRequest - If using settings.REPLICATION_PROVIDER: - include all those approved on the replication provider's copy of the image """ #TODO: In a future update to 'imaging' we might image 'as the user' rather than 'as the admin user', in this case we should just use 'owner' metadata image_owner = cloud_machine.get('application_owner') # NOTE: This assumes that the 'owner' (atmosphere user) == 'project_name' (Openstack) # Always include the original application owner owner_set = set() if image_owner: owner_set.add(image_owner) if hasattr(cloud_machine, 'id'): image_id = cloud_machine.id elif type(cloud_machine) == dict: image_id = cloud_machine.get('id') else: raise ValueError("Unexpected cloud_machine: %s" % cloud_machine) existing_members = account_driver.get_image_members(image_id, None) # Extend to include based on projects already granted access to the image cloud_shared_set = { p.name for p in existing_members } # Deprecation warning: Now that we use a script to do replication, # we should not need to account for shares on another provider. # Remove this code any time during/after the v29 release has_machine_request = MachineRequest.objects.filter( new_machine__instance_source__identifier=cloud_machine.id, status__name='completed').last() machine_request_set = set() machine_request_provider_set = set() if has_machine_request: access_list = has_machine_request.get_access_list() # NOTE: This assumes that every name in # accesslist (AtmosphereUser) == project_name(Openstack) machine_request_set = { name.strip() for name in access_list } request_provider = has_machine_request.new_machine_provider request_identifier = has_machine_request.new_machine.instance_source.identifier if request_provider != db_machine.provider: main_account_driver = get_account_driver(request_provider) # Extend to include based on information in the machine request request_shared_projects = main_account_driver.get_image_members(request_identifier, None) machine_request_provider_set = set(p.name for p in request_shared_projects) # End deprecation warning # Extend to include new names found by application pattern_match parent_app = db_machine.application_version.application access_list_set = set(parent_app.get_users_from_access_list().values_list('username', flat=True)) shared_project_names = list(owner_set | cloud_shared_set | machine_request_set | machine_request_provider_set | access_list_set) return shared_project_names
def glance_image_owner(provider_uuid, identifier, glance_image=None): try: prov = Provider.objects.get(uuid=provider_uuid) accounts = get_account_driver(prov) if not glance_image: accounts.clear_cache() glance_image = accounts.get_image(identifier) project = accounts.user_manager.get_project_by_id( glance_image.get('owner') ) except Exception as e: logger.exception(e) project = None if not project: return None try: image_owner = Identity.objects.get( provider__uuid=provider_uuid, created_by__username=project.name ) except Identity.DoesNotExist: logger.warn( "Could not find a username %s on Provider %s" % (project.name, provider_uuid) ) image_owner = None return image_owner
def remove_empty_networks_for(provider_id): provider = Provider.objects.get(id=provider_id) os_driver = get_account_driver(provider) all_instances = os_driver.admin_driver.list_all_instances() project_map = os_driver.network_manager.project_network_map() projects_with_networks = project_map.keys() for project in projects_with_networks: networks = project_map[project]['network'] if type(networks) != list: networks = [networks] for network in networks: network_name = network['name'] logger.debug("Checking if network %s is in use" % network_name) if running_instances(network_name, all_instances): continue #TODO: MUST change when not using 'usergroups' explicitly. user = project try: logger.debug("Removing project network for User:%s, Project:%s" % (user, project)) os_driver.network_manager.delete_project_network(user, project) except NeutronClientException: logger.exception("Neutron unable to remove project" "network for %s-%s" % (user,project)) except NeutronException: logger.exception("Neutron unable to remove project" "network for %s-%s" % (user,project))
def upload_privacy_data(machine_request, new_machine): """ ASSERT: The image in 'new_machine' SHOULD BE private (Based on values in machine_request) """ prov = new_machine.provider accounts = get_account_driver(prov) if not accounts: print "Aborting import: Could not retrieve Account Driver "\ "for Provider %s" % prov return img = accounts.get_image(new_machine.identifier) if hasattr(img, 'visibility'): # Treated as an obj. is_public = img.visibility == 'public' elif hasattr(img, 'items'): # Treated as a dict. is_public = img.get('visibility','N/A') == 'public' if is_public: print "Marking image %s private" % img.id accounts.image_manager.update_image(img, visibility='shared') accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: print "Aborting import: Could not retrieve admin_driver "\ "for Provider %s" % prov return img = accounts.get_image(new_machine.identifier) tenant_list = machine_request.get_access_list() # All in the list will be added as 'sharing' the OStack img # All tenants already sharing the OStack img will be added to this list return sync_machine_membership(accounts, img, new_machine, tenant_list)
def validate_new_image(image_id, machine_request_id): machine_request = MachineRequest.objects.get(id=machine_request_id) new_status, _ = StatusType.objects.get_or_create(name="validating") machine_request.status = new_status machine_request.old_status = 'validating' machine_request.save() accounts = get_account_driver(machine_request.new_machine.provider) accounts.clear_cache() from service.instance import launch_machine_instance admin_driver = accounts.admin_driver admin_ident = machine_request.new_admin_identity() if not admin_driver: celery_logger.warn( "Need admin_driver functionality to auto-validate instance") return False if not admin_ident: celery_logger.warn( "Need to know the AccountProvider to auto-validate instance") return False # Attempt to launch using the admin_driver admin_driver.identity.user = admin_ident.created_by machine = admin_driver.get_machine(image_id) small_size = admin_driver.list_sizes()[0] instance = launch_machine_instance( admin_driver, admin_ident, machine, small_size, 'Automated Image Verification - %s' % image_id, username='******', using_admin=True) return instance.id
def start_instance_maintenances(provider, action, users=[], sleep_time=None, dry_run=False): accounts = get_account_driver(provider) all_insts = accounts.list_all_instances() all_tenants = accounts.list_projects() all_insts = make_user_instances(all_insts, all_tenants, users) hostname_map = _create_hostname_mapping(all_insts) finished = False while not finished: #Iterate the list of hosts, complete finished = True for host in hostname_map.keys(): inst_list = hostname_map[host] if len(inst_list) == 0: continue instance = inst_list.pop() print "Instance %s - Hostname %s" % (instance.id, host) status = instance.extra['status'] if status != 'active': print "Skipping instance %s in state %s" % (instance.id, status) continue finished = False identity = Identity.objects.get( created_by__username=instance.username, provider=provider) print 'Performing Instance Maintenance - %s - %s' % (instance.id, host) try: _execute_action(identity, instance, action, dry_run) except Exception, e: print "Could not %s Instance %s - Error %s" % (action, instance.id, e) continue print "Waiting %s seconds" % sleep_time if not dry_run: time.sleep(sleep_time)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--provider", type=int, help="Atmosphere provider ID" " to use when importing users.") parser.add_argument("--provider-id", type=int, help="Atmosphere provider ID" " to use when importing users." " DEPRECATION WARNING -- THIS WILL BE REMOVED SOON!") parser.add_argument("--provider-list", action="store_true", help="List of provider names and IDs") parser.add_argument("--rebuild", action="store_true", help="Rebuild all accounts that are in the provider") parser.add_argument("--group", help="LDAP group of usernames to import.") parser.add_argument("--users", help="usernames to add to Atmosphere. (comma separated list with no spaces)") parser.add_argument("--admin", action="store_true", help="Users addded as admin and staff users.") args = parser.parse_args() if args.provider_list: print "ID\tName" for p in Provider.objects.all().order_by('id'): print "%d\t%s" % (p.id, p.location) return usernames = [] if args.provider_id and not args.provider: print "WARNING: --provider-id has been *DEPRECATED*! Use --provider instead!" args.provider = args.provider_id if args.provider: provider = Provider.objects.get(id=args.provider) else: raise Exception("Missing required argument: --provider <id>. use --provider-list to get a list of provider ID+names") print "Using Provider: %s" % provider try: acct_driver = get_account_driver(provider, raise_exception=True) except: account_provider = provider.accountprovider_set.first() print "Could not create the account Driver for this Provider."\ " Check the configuration of this identity:%s" % account_provider raise if args.group: print "Retrieving all '%s' members in LDAP." % args.group usernames = get_members(args.group) elif args.users: usernames = args.users.split(",") else: # if not args.users if not args.rebuild: print "Retrieving all 'atmo-user' members in LDAP." usernames = get_members('atmo-user') else: print "Rebuilding all existing users." usernames = get_usernames(provider) return run_create_accounts(acct_driver, provider, usernames, args.rebuild, args.admin)
def prune_machines_for( provider_id, print_logs=False, dry_run=False, forced_removal=False): """ Look at the list of machines (as seen by the AccountProvider) if a machine cannot be found in the list, remove it. NOTE: BEFORE CALLING THIS TASK you should ensure that the AccountProvider can see ALL images. Failure to do so will result in any image unseen by the admin to be prematurely end-dated and removed from the API/UI. """ provider = Provider.objects.get(id=provider_id) now = timezone.now() if print_logs: import logging import sys consolehandler = logging.StreamHandler(sys.stdout) consolehandler.setLevel(logging.DEBUG) celery_logger.addHandler(consolehandler) celery_logger.info("Starting prune_machines for Provider %s @ %s" % (provider, now)) if provider.is_active(): account_driver = get_account_driver(provider) db_machines = ProviderMachine.objects.filter( only_current_source(), instance_source__provider=provider) cloud_machines = account_driver.list_all_images() else: db_machines = ProviderMachine.objects.filter( source_in_range(), # like 'only_current..' w/o active_provider instance_source__provider=provider) cloud_machines = [] # Don't do anything if cloud machines == [None,[]] if not cloud_machines and not forced_removal: return # Loop 1 - End-date All machines in the DB that # can NOT be found in the cloud. mach_count = _end_date_missing_database_machines( db_machines, cloud_machines, now=now, dry_run=dry_run) # Loop 2 and 3 - Capture all (still-active) versions without machines, # and all applications without versions. # These are 'outliers' and mainly here for safety-check purposes. ver_count = _remove_versions_without_machines(now=now) app_count = _remove_applications_without_versions(now=now) # Loop 4 - All 'Application' DB objects require # >=1 Version with >=1 ProviderMachine (ACTIVE!) # Apps that don't meet this criteria should be end-dated. app_count += _update_improperly_enddated_applications(now) celery_logger.info( "prune_machines completed for Provider %s : " "%s Applications, %s versions and %s machines pruned." % (provider, app_count, ver_count, mach_count)) if print_logs: celery_logger.removeHandler(consolehandler)
def glance_image_for(provider_uuid, identifier): try: prov = Provider.objects.get(uuid=provider_uuid) accounts = get_account_driver(prov) glance_image = accounts.get_image(identifier) except Exception as e: logger.exception(e) glance_image = None return glance_image
def memoized_driver(machine, account_drivers={}): provider = machine.instance_source.provider account_driver = account_drivers.get(provider) if not account_driver: account_driver = get_account_driver(provider) if not account_driver: raise Exception("Cannot instantiate an account driver for %s" % provider) account_drivers[provider] = account_driver return account_driver
def prune_machines_for(provider_id, print_logs=False, dry_run=False, forced_removal=False): """ Look at the list of machines (as seen by the AccountProvider) if a machine cannot be found in the list, remove it. NOTE: BEFORE CALLING THIS TASK you should ensure that the AccountProvider can see ALL images. Failure to do so will result in any image unseen by the admin to be prematurely end-dated and removed from the API/UI. """ provider = Provider.objects.get(id=provider_id) now = timezone.now() if print_logs: console_handler = _init_stdout_logging() celery_logger.info("Starting prune_machines for Provider %s @ %s" % (provider, now)) if provider.is_active(): account_driver = get_account_driver(provider) db_machines = ProviderMachine.objects.filter( only_current_source(), instance_source__provider=provider) cloud_machines = account_driver.list_all_images() else: db_machines = ProviderMachine.objects.filter( source_in_range(), # like 'only_current..' w/o active_provider instance_source__provider=provider) cloud_machines = [] # Don't do anything if cloud machines == [None,[]] if not cloud_machines and not forced_removal: return # Loop 1 - End-date All machines in the DB that # can NOT be found in the cloud. mach_count = _end_date_missing_database_machines(db_machines, cloud_machines, now=now, dry_run=dry_run) # Loop 2 and 3 - Capture all (still-active) versions without machines, # and all applications without versions. # These are 'outliers' and mainly here for safety-check purposes. ver_count = _remove_versions_without_machines(now=now) app_count = _remove_applications_without_versions(now=now) # Loop 4 - All 'Application' DB objects require # >=1 Version with >=1 ProviderMachine (ACTIVE!) # Apps that don't meet this criteria should be end-dated. app_count += _update_improperly_enddated_applications(now) celery_logger.info("prune_machines completed for Provider %s : " "%s Applications, %s versions and %s machines pruned." % (provider, app_count, ver_count, mach_count)) if print_logs: _exit_stdout_logging(console_handler)
def monitor_machines_for(provider_id, limit_machines=[], print_logs=False, dry_run=False): """ Run the set of tasks related to monitoring machines for a provider. Optionally, provide a list of usernames to monitor While debugging, print_logs=True can be very helpful. start_date and end_date allow you to search a 'non-standard' window of time. NEW LOGIC: """ provider = Provider.objects.get(id=provider_id) if print_logs: console_handler = _init_stdout_logging() account_driver = get_account_driver(provider) #Bail out if account driver is invalid if not account_driver: _exit_stdout_logging(console_handler) return [] cloud_machines = account_driver.list_all_images() if limit_machines: cloud_machines = [ cm for cm in cloud_machines if cm.id in limit_machines ] db_machines = [] # ASSERT: All non-end-dated machines in the DB can be found in the cloud # if you do not believe this is the case, you should call 'prune_machines_for' for cloud_machine in cloud_machines: if not machine_is_valid(cloud_machine, account_driver): continue owner_project = _get_owner(account_driver, cloud_machine) #STEP 1: Get the application, version, and provider_machine registered in Atmosphere (db_machine, created) = convert_glance_image(cloud_machine, provider.uuid, owner_project) db_machines.append(db_machine) #STEP 2: For any private cloud_machine, convert the 'shared users' as known by cloud update_image_membership(account_driver, cloud_machine, db_machine) # into DB relationships: ApplicationVersionMembership, ProviderMachineMembership #STEP 3: if ENFORCING -- occasionally 're-distribute' any ACLs that are *listed on DB but not on cloud* -- removals should be done explicitly, outside of this function if settings.ENFORCING: distribute_image_membership(account_driver, cloud_machine, provider) # ASSERTIONS about this method: # 1) We will never 'remove' membership, # 2) We will never 'remove' a public or private flag as listed in application. # 2b) Future: Individual versions/machines as described by relationships above dictate whats shown in the application. if print_logs: _exit_stdout_logging(console_handler) return db_machines
def validate_new_image(image_id, machine_request_id): if not getattr(settings, 'ENABLE_IMAGE_VALIDATION', True): celery_logger.warn("Skip validation: ENABLE_IMAGE_VALIDATION is False") return True machine_request = MachineRequest.objects.get(id=machine_request_id) new_status, _ = StatusType.objects.get_or_create(name="validating") machine_request.status = new_status machine_request.old_status = 'validating' local_username = machine_request.created_by.username #NOTE: Change local_username accordingly when this assumption is no longer true. machine_request.save() accounts = get_account_driver(machine_request.new_machine.provider) accounts.clear_cache() from service.instance import launch_machine_instance admin_driver = accounts.admin_driver admin_ident = machine_request.new_admin_identity() if not admin_driver: celery_logger.warn( "Need admin_driver functionality to auto-validate instance" ) return False if not admin_ident: celery_logger.warn( "Need to know the AccountProvider to auto-validate instance" ) return False # Attempt to launch using the admin_driver user = admin_ident.created_by admin_driver.identity.user = user machine = admin_driver.get_machine(image_id) sorted_sizes = admin_driver.list_sizes() size_index = 0 while size_index < len(sorted_sizes): selected_size = sorted_sizes[size_index] size_index += 1 try: instance = launch_machine_instance( admin_driver, user, admin_ident, machine, selected_size, 'Automated Image Verification - %s' % image_id, username=local_username, using_admin=True ) return instance.provider_alias except BaseHTTPError as http_error: if "Flavor's disk is too small for requested image" in http_error.message: continue logger.exception(http_error) raise except Exception as exc: logger.exception(exc) raise # End of while loop raise Exception("Validation of new Image %s has *FAILED*" % image_id)
def create_first_identity(username, provider=None): from service.driver import get_account_driver user = AtmosphereUser.objects.get(username=username) if not provider: provider = get_available_provider() if not provider: raise Exception("No currently active providers -- Could not create First identity") accounts = get_account_driver(provider) new_identity = accounts.create_account(user.username) return new_identity
def add_os_project_network(core_identity, *args, **kwargs): try: logger.debug("add_os_project_network task started at %s." % datetime.now()) account_driver = get_account_driver(core_identity.provider) account_driver.create_network(core_identity) logger.debug("add_os_project_network task finished at %s." % datetime.now()) except Exception as exc: add_os_project_network.retry(exc=exc)
def monitor_volumes_for(provider_id, print_logs=False): """ Run the set of tasks related to monitoring sizes for a provider. Optionally, provide a list of usernames to monitor While debugging, print_logs=True can be very helpful. start_date and end_date allow you to search a 'non-standard' window of time. """ from service.driver import get_account_driver from core.models import Identity if print_logs: console_handler = _init_stdout_logging() provider = Provider.objects.get(id=provider_id) account_driver = get_account_driver(provider) # Non-End dated volumes on this provider db_volumes = Volume.objects.filter(only_current_source(), instance_source__provider=provider) all_volumes = account_driver.admin_driver.list_all_volumes(timeout=30) seen_volumes = [] for cloud_volume in all_volumes: try: core_volume = convert_esh_volume(cloud_volume, provider_uuid=provider.uuid) seen_volumes.append(core_volume) except ObjectDoesNotExist: tenant_id = cloud_volume.extra['object']['os-vol-tenant-attr:tenant_id'] tenant = account_driver.get_project_by_id(tenant_id) tenant_name = tenant.name if tenant else tenant_id try: if not tenant: celery_logger.warn("Warning: tenant_id %s found on volume %s, but did not exist from the account driver perspective.", tenant_id, cloud_volume) raise ObjectDoesNotExist() identity = Identity.objects.filter( contains_credential('ex_project_name', tenant_name), provider=provider ).first() if not identity: raise ObjectDoesNotExist() core_volume = convert_esh_volume( cloud_volume, provider.uuid, identity.uuid, identity.created_by) except ObjectDoesNotExist: celery_logger.info("Skipping Volume %s - No Identity for: Provider:%s + Project Name:%s" % (cloud_volume.id, provider, tenant_name)) pass now_time = timezone.now() needs_end_date = [volume for volume in db_volumes if volume not in seen_volumes] for volume in needs_end_date: celery_logger.debug("End dating inactive volume: %s" % volume) volume.end_date = now_time volume.save() if print_logs: _exit_stdout_logging(console_handler) for vol in seen_volumes: vol.esh = None return [vol.instance_source.identifier for vol in seen_volumes]
def validate_new_provider(new_provider, new_identity): acct_driver = get_account_driver(new_provider) if not acct_driver: print "Could not create an account driver for the new Provider"\ " %s - %s. Check your credentials and try again. "\ "If you believe you are receiving this message in error, "\ "AND you are able to use external CLI tools on this machine "\ "to contact your cloud, please report the issue to a developer!"\ % (new_provider, new_identity) return False return True
def remove_membership(image_version, group, accounts=None): """ This function will remove *all* users in the group to *all* providers/machines using this image_version """ for provider_machine in image_version.machines.filter( only_current_source()): prov = provider_machine.instance_source.provider if not accounts: accounts = get_account_driver(prov) if not accounts: raise NotImplemented("Account Driver could not be created for %s" % prov) accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: raise NotImplemented("Admin Driver could not be created for %s" % prov) img = accounts.get_image(provider_machine.identifier) approved_projects = accounts.get_image_members(img.id) for identity_membership in group.identitymembership_set.order_by( 'identity__created_by__username'): if identity_membership.identity.provider != prov: continue # Get project name from the identity's credential-list project_name = identity_membership.identity.get_credential( 'ex_project_name') project = accounts.get_project(project_name) if project and project not in approved_projects: continue # Perform a *DATABASE* remove first. application = provider_machine.application application_version = provider_machine.application_version models.ApplicationMembership.objects.filter( group=group, application=application).delete() logger.info("Removed ApplicationMembership: %s-%s" % (application, group)) models.ApplicationVersionMembership.objects.filter( group=group, image_version=application_version).delete() logger.info("Removed ApplicationVersionMembership: %s-%s" % (application_version, group)) models.ProviderMachineMembership.objects.filter( group=group, provider_machine=provider_machine).delete() logger.info("Removed ProviderMachineMembership: %s-%s" % (provider_machine, group)) # Perform a *CLOUD* remove last. try: accounts.image_manager.unshare_image(img, project_name) except Exception as exc: logger.exception( "Exception occurred while removing user from cloud: %s", exc) logger.info("Removed Cloud Access: %s-%s" % (img, project_name)) return
def _set_compute_quota(user_quota, identity): # Use THESE values... compute_values = { 'cores': user_quota.cpu, 'ram': user_quota.memory * 1024, # NOTE: Value is stored in GB, Openstack (Liberty) expects MB 'floating_ips': user_quota.floating_ip_count, 'fixed_ips': user_quota.port_count, 'instances': user_quota.instance_count, 'force': True } creds = identity.get_all_credentials() use_tenant_id = False if creds.get('ex_force_auth_version', '2.0_password') == "2.0_password": compute_values.pop('instances') use_tenant_id = True username = identity.created_by.username logger.info("Updating quota for %s to %s" % (username, compute_values)) driver = get_cached_driver(identity=identity) username = driver._connection.key ad = get_account_driver(identity.provider, raise_exception=True) tenant = ad.get_project(username) ks_user = ad.get_user(username) admin_driver = ad.admin_driver creds = identity.get_all_credentials() if creds.get('ex_force_auth_version', '2.0_password') != "2.0_password": # FIXME: Remove 'use_tenant_id' when legacy clouds are no-longer in use. try: result = admin_driver._connection.ex_update_quota( tenant.id, compute_values, use_tenant_id=use_tenant_id ) except Exception: logger.exception( "Could not set a user-quota, trying to set tenant-quota" ) raise else: # For CyVerse old clouds, run the top method. don't use try/except. try: result = admin_driver._connection.ex_update_quota_for_user( tenant.id, ks_user.id, compute_values, use_tenant_id=use_tenant_id ) except Exception: logger.exception( "Could not set a user-quota, trying to set tenant-quota" ) raise logger.info("Updated quota for %s to %s" % (username, result)) return result
def glance_write_machine(provider_machine): """ Using the provider_machine in the DB, save information to the Cloud. """ update_method = "" base_source = provider_machine.instance_source provider = base_source.provider base_app = provider_machine.application version = provider_machine.application_version identifier = base_source.identifier accounts = get_account_driver(provider) g_image = glance_image_for(provider.uuid, identifier) app_version_bundle_name = provider_machine.generated_name() if not g_image: return if hasattr(g_image, 'properties'): props = g_image.properties update_method = 'v2' elif hasattr(g_image, 'items'): props = dict(g_image.items()) update_method = 'v3' else: raise Exception( "The method for 'introspecting an image' has changed!" " Ask a programmer to fix this!") # Do any updating that makes sense... Name. Metadata.. blacklist_metadata_key = getattr(settings, "BLACKLIST_METADATA_KEY", "atmo_image_exclude") whitelist_metadata_key = getattr(settings, "WHITELIST_METADATA_KEY", "atmo_image_include") overrides = { "application_uuid": str(base_app.uuid), "application_name": _make_safe(base_app.name), "application_owner": base_app.created_by.username, "application_tags": json.dumps( [_make_safe(tag.name) for tag in base_app.tags.all()]), "application_description": _make_safe(base_app.description), "version_name": str(version.name), "version_changelog": str(version.change_log), whitelist_metadata_key: "true", blacklist_metadata_key: "false" } if update_method == 'v2': extras = { 'properties': overrides } extras['name'] = app_version_bundle_name props.update(extras) g_image.update(props) else: overrides['name'] = app_version_bundle_name accounts.image_manager.glance.images.update( g_image.id, **overrides) return True
def create_accounts(self, provider, username, force=False): from service.driver import get_account_driver from core.models import Project, Identity credentials_list = self.get_credentials_list(provider, username) identities = Identity.objects.none() for credentials in credentials_list: try: project_name = credentials['project_name'] created_identities = self.find_accounts(provider, **credentials) if created_identities and not force: # logger.debug( # "Accounts already created for %s on provider %s", username, provider) identities |= created_identities continue logger.debug( "Creating new account for %s with credentials - %s" % (username, credentials) ) account_driver = get_account_driver(provider) if not account_driver: raise ValueError( "Provider %s produced an invalid account driver "\ "-- Use plugin after you create a core.Provider "\ "*AND* assign a core.Identity to be the core.AccountProvider." % provider) new_identity = account_driver.create_account(**credentials) identities |= Identity.objects.filter(id=new_identity.id) memberships = new_identity.identity_memberships.filter( member__memberships__is_leader=True ) if not memberships: memberships = new_identity.identity_memberships.all() membership = memberships.first() if not membership: raise ValueError( "Expected at least one member in identity %s" % new_identity ) group = membership.member try: Project.objects.get(name=project_name, owner=group) except Project.DoesNotExist: Project.objects.create( name=project_name, created_by=new_identity.created_by, owner=group ) except: logger.exception( "Could *NOT* Create NEW account for %s" % username ) return identities
def remove_membership(image_version, group, accounts=None): """ This function will remove *all* users in the group to *all* providers/machines using this image_version """ for provider_machine in image_version.machines.filter(only_current_source()): prov = provider_machine.instance_source.provider if not accounts: accounts = get_account_driver(prov) if not accounts: raise NotImplemented("Account Driver could not be created for %s" % prov) accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: raise NotImplemented("Admin Driver could not be created for %s" % prov) img = accounts.get_image(provider_machine.identifier) approved_projects = accounts.get_image_members(img.id) for identity_membership in group.identitymembership_set.order_by('identity__created_by__username'): if identity_membership.identity.provider != prov: continue # Get project name from the identity's credential-list project_name = identity_membership.identity.get_credential( 'ex_project_name') project = accounts.get_project(project_name) if project and project not in approved_projects: continue # Perform a *DATABASE* remove first. application = provider_machine.application application_version = provider_machine.application_version models.ApplicationMembership.objects.filter( group=group, application=application).delete() logger.info("Removed ApplicationMembership: %s-%s" % (application, group)) models.ApplicationVersionMembership.objects.filter( group=group, image_version=application_version).delete() logger.info("Removed ApplicationVersionMembership: %s-%s" % (application_version, group)) models.ProviderMachineMembership.objects.filter( group=group, provider_machine=provider_machine).delete() logger.info("Removed ProviderMachineMembership: %s-%s" % (provider_machine, group)) # Perform a *CLOUD* remove last. try: accounts.image_manager.unshare_image(img, project_name) except Exception as exc: logger.exception("Exception occurred while removing user from cloud: %s", exc) logger.info("Removed Cloud Access: %s-%s" % (img, project_name)) return
def glance_image_owner(provider_uuid, identifier): try: prov = Provider.objects.get(uuid=provider_uuid) accounts = get_account_driver(prov) glance_image = accounts.get_image(identifier) project = accounts.user_manager.get_project_by_id(glance_image.owner) image_owner = Identity.objects.get( provider__uuid=provider_uuid, created_by__username=project.name) except Exception as e: logger.exception(e) image_owner = None return image_owner
def add_membership(image_version, group): """ This function will add *all* users in the group to *all* providers/machines using this image_version O(N^2) """ for provider_machine in image_version.machines.filter( only_current_source()): prov = provider_machine.instance_source.provider accounts = get_account_driver(prov) if not accounts: raise NotImplemented("Account Driver could not be created for %s" % prov) accounts.clear_cache() admin_driver = accounts.admin_driver # cache has been cleared if not admin_driver: raise NotImplemented("Admin Driver could not be created for %s" % prov) img = accounts.get_image(provider_machine.identifier) projects = get_current_projects_for_image(accounts, img.id) for identity_membership in group.identitymembership_set.all(): if identity_membership.identity.provider != prov: continue # Get project name from the identity's credential-list project_name = identity_membership.identity.get_credential( 'ex_project_name') project = accounts.get_project(project_name) if project and project in projects: continue # Share with the *database* first! obj, created = models.ApplicationMembership.objects.get_or_create( group=group, application=provider_machine.application) if created: print "Created new ApplicationMembership: %s" \ % (obj,) obj, created = models.ApplicationVersionMembership.objects.get_or_create( group=group, image_version=provider_machine.application_version) if created: print "Created new ApplicationVersionMembership: %s" \ % (obj,) obj, created = models.ProviderMachineMembership.objects.get_or_create( group=group, provider_machine=provider_machine) if created: print "Created new ProviderMachineMembership: %s" \ % (obj,) # Share with the *cloud* last! accounts.image_manager.share_image(img, project_name) accounts.accept_shared_image(img, project_name) logger.info("Added Cloud Access: %s-%s" % (img, project_name)) continue
def _get_hard_limits(identity): """ Lookup the OpenStack "Hard Limits" based on the account provider """ accounts = get_account_driver(identity.provider) defaults = {"ram": 999, "cpu": 99} # Used when all else fails. limits = {} limits.update(defaults) username = identity.get_credential('key') project_name = identity.get_credential('ex_project_name') user_limits = accounts.get_quota_limit(username, project_name) if user_limits: limits.update(user_limits) return limits
def _set_volume_quota(user_quota, identity): volume_values = { 'volumes': user_quota.storage_count, 'gigabytes': user_quota.storage, 'snapshots': user_quota.snapshot_count, } username = identity.created_by.username logger.info("Updating quota for %s to %s" % (username, volume_values)) driver = get_cached_driver(identity=identity) username = driver._connection._get_username() ad = get_account_driver(identity.provider) admin_driver = ad.admin_driver admin_driver._connection._cinder_update_quota(username, volume_values) return
def glance_update_machine_metadata(provider_machine, metadata={}): update_method = "" base_source = provider_machine.instance_source provider = base_source.provider base_app = provider_machine.application version = provider_machine.application_version identifier = base_source.identifier accounts = get_account_driver(provider) g_image = glance_image_for(base_source.provider.uuid, identifier) if not g_image: return False if hasattr(g_image, 'properties'): props = g_image.properties update_method = 'v2' elif hasattr(g_image, 'items'): props = dict(g_image.items()) update_method = 'v3' else: raise Exception( "The method for 'introspecting an image' has changed!" " Ask a programmer to fix this!" ) overrides = { "application_version": str(version.name), "application_uuid": str(base_app.uuid), "application_name": _make_safe(base_app.name), "application_owner": base_app.created_by.username, "application_tags": json.dumps([_make_safe(tag.name) for tag in base_app.tags.all()]), "application_description": _make_safe(base_app.description), "version_name": str(version.name), "version_changelog": str(version.change_log) } overrides.update(metadata) if update_method == 'v2': extras = {'properties': overrides} props.update(extras) g_image.update(name=base_app.name, properties=extras) else: accounts.image_manager.glance.images.update(g_image.id, **overrides) return True
def _set_volume_quota(user_quota, identity): volume_values = { 'volumes': user_quota.storage_count, 'gigabytes': user_quota.storage, 'snapshots': user_quota.snapshot_count, } username = identity.created_by.username logger.info("Updating quota for %s to %s" % (username, volume_values)) ad = get_account_driver(identity.provider) admin_driver = ad.admin_driver tenant = ad.get_project(username) result = admin_driver._connection._cinder_update_quota( tenant.id, volume_values ) logger.info("Updated quota for %s to %s" % (username, result)) return result
def create_new_account_for(provider, user): from service.driver import get_account_driver existing_user_list = provider.identity_set.values_list( 'created_by__username', flat=True) if user.username in existing_user_list: logger.info("Accounts already exists on %s for %s" % (provider.location, user.username)) return None try: accounts = get_account_driver(provider) logger.info("Create NEW account for %s" % user.username) new_identity = accounts.create_account(user.username) return new_identity except: logger.exception("Could *NOT* Create NEW account for %s" % user.username) return None
def remove_empty_networks_for(provider_id): provider = Provider.objects.get(id=provider_id) os_driver = get_account_driver(provider) if not os_driver: celery_logger.warn( "Cannot remove_empty_networks_for provider %s -- Account Driver not created" % provider) return all_instances = os_driver.admin_driver.list_all_instances() project_map = os_driver.network_manager.project_network_map() known_project_names = Credential.objects.filter( key='ex_project_name').values_list('value', flat=True) projects_with_networks = sorted( [k for k in project_map.keys() if k in known_project_names]) for project in projects_with_networks: networks = project_map[project]['network'] if not isinstance(networks, list): networks = [networks] for network in networks: network_name = network['name'] celery_logger.debug("Checking if network %s is in use" % network_name) if running_instances(network_name, all_instances): continue user = project identity = Identity.objects.filter( provider_id=provider_id, credential__key='ex_project_name', credential__value=project).filter( credential__key='key', credential__value=user).first() if not identity: celery_logger.warn( "NOT Removing project network for User:%s, Project:%s -- No Valid Identity found!" % (user, project)) continue try: celery_logger.debug( "Removing project network for User:%s, Project:%s" % (user, project)) os_driver.delete_user_network(identity) except NeutronClientException: celery_logger.exception("Neutron unable to remove project" "network for %s-%s" % (user, project)) except NeutronException: celery_logger.exception("Neutron unable to remove project" "network for %s-%s" % (user, project))
def get_public_and_private_apps(provider): """ INPUT: Provider provider OUTPUT: 2-tuple ( new_public_apps [], private_apps(key) + super-set-membership(value) {}) """ account_driver = get_account_driver(provider) all_projects_map = tenant_id_to_name_map(account_driver) cloud_machines = account_driver.list_all_images() db_machines = ProviderMachine.objects.filter( only_current_source(), instance_source__provider=provider) new_public_apps = [] private_apps = {} # ASSERT: All non-end-dated machines in the DB can be found in the cloud # if you do not believe this is the case, you should call 'prune_machines_for' for cloud_machine in cloud_machines: #Filter out: ChromoSnapShot, eri-, eki-, ... (Or dont..) if any( cloud_machine.name.startswith(prefix) for prefix in ['eri-', 'eki-', 'ChromoSnapShot']): #celery_logger.debug("Skipping cloud machine %s" % cloud_machine) continue app_name, version_name = ProviderMachine._split_cloud_name( cloud_machine.name) db_machine = get_or_create_provider_machine(cloud_machine.id, app_name, provider.uuid, version_name=version_name) db_version = db_machine.application_version db_application = db_version.application if cloud_machine.get('visibility') == 'public': if db_application.private and db_application not in new_public_apps: new_public_apps.append(db_application) #Distinct list.. #Else the db app is public and no changes are necessary. else: # cloud machine is private membership = get_shared_identities(account_driver, cloud_machine, all_projects_map) all_members = private_apps.get(db_application, []) all_members.extend(membership) #Distinct list.. private_apps[db_application] = all_members return new_public_apps, private_apps