def main():
"""
TODO: Add argparse, --delete : Deletes existing users in openstack (Never use in PROD)
"""
openstack = Provider.objects.filter(
type__name__iexact="openstack").order_by("id")
if not openstack:
raise Provider.DoesNotExist("No OpenStack Provider Found")
openstack = openstack[0]
os_driver = OSAccountDriver(openstack)
found = 0
create = 0
usernames = os_driver.list_usergroup_names()
quota_dict = {'cpu': 10, 'memory': 20, 'storage': 10, 'storage_count': 10}
higher_quota = Quota.objects.get_or_create(**quota_dict)[0]
for user in usernames:
# Openstack account exists, but we need the identity.
ident = os_driver.create_account(user)
if is_staff(ident):
im = ident.identity_membership.all()[0]
#Disable time allocation
im.allocation = None
#Raise everybody's quota
im.quota = higher_quota
im.save()
print "Total users added to atmosphere:%s" % len(usernames)
def main():
"""
TODO: Add argparse, --delete : Deletes existing users in eucalyptus (Never use in PROD)
"""
euca = Provider.objects.get(location='Eucalyptus (retiring March 4)')
euca_driver = EucaAccountDriver(euca)
openstack = Provider.objects.get(location='iPlant Cloud - Tucson')
os_driver = OSAccountDriver(openstack)
all_users = euca_driver.list_users()
#Sort by users
all_values = sorted(all_users.values(), key=lambda user: user['username'])
total = 0
for user_dict in all_values:
id_exists = Identity.objects.filter(
created_by__username=user_dict['username'],
provider=euca)
if not id_exists:
euca_driver.create_account(user_dict)
total += 1
print "Added to Eucalyptus: %s" % user_dict['username']
print "Total users added:%s" % total
if include_openstack:
print "Adding all eucalyptus users to openstack"
total = 0
for user_dict in all_values:
id_exists = Identity.objects.filter(
created_by__username=user_dict['username'],
provider=openstack)
if not id_exists:
os_driver.create_account(user_dict['username'])
total += 1
print "Added to Openstack: %s" % user_dict['username']
print "Total users added to openstack:%s" % total
def get_os_account_driver(provider):
from service.accounts.openstack import AccountDriver as OSAccountDriver
if provider not in Provider.get_active(type_name='openstack'):
raise Exception("An active openstack provider is required to"
" update image owner")
accounts = OSAccountDriver(provider)
return accounts
def main():
"""
Using the keyname and public_key defined in settings
Ensure that the keypair has been distributed to every identity on the
provider.
It is essential that all users carry the same keypair to allow Deployment
access
"""
keyname = settings.ATMOSPHERE_KEYPAIR_NAME
with open(settings.ATMOSPHERE_KEYPAIR_FILE, 'r') as pub_key_file:
public_key = pub_key_file.read()
print "Adding keypair: %s Contents: %s" % (keyname, public_key)
os_providers = Provider.objects.filter(type__name="OpenStack")
for prov in os_providers:
count = 0
identities = Identity.objects.filter(provider=prov)
os_accounts = OSAccountDriver(prov)
for ident in identities:
creds = os_accounts.parse_identity(ident)
try:
(keypair, created) = os_accounts.get_or_create_keypair(
creds['username'], creds['password'], creds['tenant_name'],
keyname, public_key)
except Unauthorized, exc:
print "Could not create keypair for %s. Error message: %s"\
% (creds['username'], exc.message)
if created:
print "Created keypair %s for user %s"\
% (keypair, creds['username'])
count += 1
print 'Keypairs added for %s accounts on %s' % (count, prov)
def remove_empty_networks():
try:
logger.debug("remove_empty_networks task started at %s." %
datetime.now())
for provider in Provider.get_active(type_name='openstack'):
os_driver = OSAccountDriver(provider)
all_instances = os_driver.admin_driver.list_all_instances()
project_map = os_driver.network_manager.project_network_map()
projects_with_networks = project_map.keys()
for project in projects_with_networks:
network_name = project_map[project]['network']['name']
logger.debug("Checking if network %s is in use" % network_name)
if running_instances(network_name, all_instances):
continue
#TODO: Will change when not using 'usergroups' explicitly.
user = project
try:
logger.debug(
"Removing project network for User:%s, Project:%s" %
(user, project))
os_driver.network_manager.delete_project_network(
user, project)
except NeutronClientException:
logger.exception("Neutron unable to remove project"
"network for %s-%s" % (user, project))
except NeutronException:
logger.exception("Neutron unable to remove project"
"network for %s-%s" % (user, project))
except Exception as exc:
logger.exception("Failed to run remove_empty_networks")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--provider",
type=int,
help="Atmosphere provider ID"
" to use.")
parser.add_argument("image_ids",
help="Image ID(s) to be repaired. (Comma-Separated)")
args = parser.parse_args()
if not args.provider:
provider = Provider.objects.get(location='iPlant Cloud - Tucson')
else:
provider = Provider.objects.get(id=args.provider)
images = args.image_ids.split(",")
accounts = OSAccountDriver(provider)
for image_id in images:
mr = MachineRequest.objects.get(new_machine__identifier=image_id)
glance_image = accounts.get_image(image_id)
if not glance_image.properties.has_key('kernel_id')\
or not glance_image.properties.has_key('ramdisk_id'):
print "Image %s (%s) is missing kernel and/or ramdisk ..." % (
image_id, glance_image.name),
fix_image(accounts, glance_image, mr)
def security_group_init(core_identity):
os_driver = OSAccountDriver(core_identity.provider)
creds = core_identity.get_credentials()
security_group = os_driver.init_security_group(
creds['key'], creds['secret'],
creds['ex_tenant_name'], creds['ex_tenant_name'],
os_driver.MASTER_RULES_LIST)
return security_group
def network_init(core_identity):
provider_creds = core_identity.provider.get_credentials()
if 'router_name' not in provider_creds.keys():
logger.warn("ProviderCredential 'router_name' missing:"
"cannot create virtual network")
return
os_driver = OSAccountDriver(core_identity.provider)
(network, subnet) = os_driver.create_network(core_identity)
return (network, subnet)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--fixed-ip",
help="Fixed IP address to use "
" (This overrides any attempt to 'guess' "
"the next IP address to use.")
parser.add_argument("--port-id",
help="Atmosphere port ID (Override)"
" to use.")
parser.add_argument("--provider",
type=int,
help="Atmosphere provider ID"
" to use.")
parser.add_argument("instance", help="Instance to repair")
parser.add_argument("--admin",
action="store_true",
help="Users addded as admin and staff users.")
parser.add_argument("--suspend-loop",
action="store_true",
help="Repair an instance that is in suspended loop")
parser.add_argument("--suspend-release",
action="store_true",
help="Release the bridge-port for this instance")
args = parser.parse_args()
users = None
added = 0
provider_id = args.provider
instance_id = args.instance
new_fixed_ip = args.fixed_ip
if not provider_id:
provider_id = 4
if not instance_id:
raise Exception("Instance ID is required")
provider = Provider.objects.get(id=provider_id)
accounts = OSAccountDriver(Provider.objects.get(id=provider_id))
admin = accounts.admin_driver
instance = admin.get_instance(instance_id)
if not instance:
raise Exception(
"Instance %s does not exist on provider %s" % instance_id,
provider_id)
if args.suspend_release:
suspended_release_instance(accounts, admin, instance, provider,
args.port_id)
elif args.suspend_loop:
suspended_repair_instance(accounts, admin, instance, provider)
print 'Resuming instance: %s' % instance.id
admin.resume_instance(instance)
print 'Waiting 5 minutes to allow instance to resume (Ctrl+C to cancel): %s' % instance.id
time.sleep(5 * 60)
print 'Rebuilding instance network and adding port: %s' % instance.id
repair_instance(accounts, admin, instance, provider, new_fixed_ip)
else:
repair_instance(accounts, admin, instance, provider, new_fixed_ip)
def keypair_init(core_identity):
os_driver = OSAccountDriver(core_identity.provider)
creds = core_identity.get_credentials()
with open(settings.ATMOSPHERE_KEYPAIR_FILE, 'r') as pub_key_file:
public_key = pub_key_file.read()
keypair, created = os_driver.get_or_create_keypair(
creds['key'], creds['secret'], creds['ex_tenant_name'],
settings.ATMOSPHERE_KEYPAIR_NAME, public_key)
if created:
logger.info("Created keypair for %s" % creds['key'])
return keypair
def add_os_project_network(core_identity, *args, **kwargs):
try:
logger.debug("add_os_project_network task started at %s." %
datetime.now())
from rtwo.accounts.openstack import AccountDriver as OSAccountDriver
account_driver = OSAccountDriver(core_identity.provider)
account_driver.create_network(core_identity)
logger.debug("add_os_project_network task finished at %s." %
datetime.now())
except Exception as exc:
add_os_project_network.retry(exc=exc)
def main():
"""
TODO: Add argparse, --delete : Deletes existing users in openstack (Never use in PROD)
"""
openstack = Provider.objects.get(location='iPlant Cloud - Tucson')
os_driver = OSAccountDriver(openstack)
found = 0
create = 0
quota_dict = {'cpu': 16, 'memory': 128, 'storage': 10, 'storage_count': 10}
higher_quota = Quota.objects.get_or_create(**quota_dict)[0]
usernames = os_driver.list_usergroup_names()
staff = get_staff_users()
staff_users = sorted(list(set(staff) & set(usernames)))
non_staff = sorted(list(set(usernames) - set(staff)))
for user in non_staff:
#Raise everybody's quota
#try:
im_list = IdentityMembership.objects.filter(
identity__created_by__username=user, identity__provider=openstack)
if not im_list:
print "Missing user:%s" % user
continue
im = im_list[0]
if not im.allocation:
print "User missing Allocation: %s" % user
im.allocation = Allocation.default_allocation()
im.save()
#Ignore the quota set if you are above it..
if im.quota.cpu >= quota_dict["cpu"] \
or im.quota.memory >= quota_dict["memory"]:
continue
print "Existing Quota CPU:%s should be %s" % (im.quota.cpu,
quota_dict["cpu"])
im.quota = higher_quota
im.save()
print 'Found non-staff user:%s -- Update quota and add allocation' % user
#for user in staff_users:
# # Openstack account exists, but we need the identity.
# im = IdentityMembership.objects.filter(identity__created_by__username=user, identity__provider=openstack)
# if not im:
# print "Missing user:%s" % user
# continue
# im = im[0]
# if im.quota.cpu == quota_dict["cpu"]:
# continue
# #Disable time allocation
# im.allocation = None
# im.quota = higher_quota
# im.save()
# print 'Found staff user:%s -- Update quota and no allocation' % user
print "Total users added to atmosphere:%s" % len(usernames)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--provider",
type=int,
help="Atmosphere provider ID"
" to use when importing users.")
parser.add_argument("users",
help="LDAP usernames to import. (comma separated)")
parser.add_argument("--admin",
action="store_true",
help="Users addded as admin and staff users.")
args = parser.parse_args()
users = None
added = 0
if args.provider:
os_driver = OSAccountDriver(Provider.objects.get(id=args.provider))
else:
os_driver = OSAccountDriver(
Provider.objects.get(location='iPlant Workshop Cloud - Tucson'))
users = args.users.split(",")
for user in users:
# Then add the Openstack Identity
try:
if is_atmo_user(user):
os_driver.create_account(user, max_quota=args.admin)
added += 1
else:
print "%s is not in the ldap atmosphere group (atmo-user)." % (
user)
continue
if args.admin:
make_admin(user)
print "%s added as admin." % (user)
else:
print "%s added." % (user)
except Exception as e:
print "Problem adding %s." % (user)
print e.message
print "Total users added:%s" % (added)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--provider",
type=int,
help="Atmosphere provider ID"
" to use when importing users.")
parser.add_argument("--users",
help="LDAP usernames to import. (comma separated)")
parser.add_argument("--admin",
action="store_true",
help="Users addded as admin and staff users.")
args = parser.parse_args()
users = None
added = 0
if args.provider:
provider = Provider.objects.get(id=args.provider)
else:
provider = Provider.objects.get(location='iPlant Cloud - Tucson')
print "Using Provider: %s" % provider
type_name = provider.type.name.lower()
if type_name == 'openstack':
acct_driver = OSAccountDriver(provider)
elif type_name == 'eucalyptus':
acct_driver = EucaAccountDriver(provider)
else:
raise Exception("Could not find an account driver for Provider with"
" type:%s" % type_name)
if not args.users:
print "Retrieving all 'atmo-user' members in LDAP."
users = get_members('atmo-user')
else:
users = args.users.split(",")
for user in users:
# Then add the Openstack Identity
try:
id_exists = Identity.objects.filter(
created_by__username__iexact=user, provider=provider)
if id_exists:
continue
acct_driver.create_account(user, max_quota=args.admin)
added += 1
if args.admin:
make_admin(user)
print "%s added as admin." % (user)
else:
print "%s added." % (user)
except Exception as e:
print "Problem adding %s." % (user)
print e.message
print "Total users added:%s" % (added)
def _create_and_attach_port(provider, driver, instance, core_identity):
accounts = OSAccountDriver(core_identity.provider)
tenant_id = instance.extra['tenantId']
network_resources = accounts.network_manager.find_tenant_resources(tenant_id)
network = network_resources['networks']
if not network:
network, subnet = accounts.create_network(core_identity)
else:
network = network[0]
subnet = network_resources['subnets'][0]
#new_fixed_ip = _get_next_fixed_ip(network_resources['ports'])
#admin = accounts.admin_driver
#port = accounts.network_manager.create_port(
# instance.id, network['id'], subnet['id'], new_fixed_ip, tenant_id)
attached_intf = driver._connection.ex_attach_interface(instance.id, network['id'])
return attached_intf
def main():
"""
TODO: Add argparse, --delete : Deletes existing users in openstack
(Never use in PROD)
"""
euca_driver = EucaAccountDriver(
Provider.objects.get(location='EUCALYPTUS'))
os_driver = OSAccountDriver(Provider.objects.get(location='iPlant Cloud - Tucson'))
found = 0
create = 0
core_services = ['estevetest03', ] # get_core_services()
for user in core_services:
euca_driver.create_account(user, max_quota=True)
# Then add the Openstack Identity
os_driver.create_account(user, max_quota=True)
make_admin(user)
print "Total core-service/admins added:%s" % len(core_services)
def remove_empty_network(
driverCls, provider, identity,
core_identity_id,
*args, **kwargs):
try:
#For testing ONLY.. Test cases ignore countdown..
if app.conf.CELERY_ALWAYS_EAGER:
logger.debug("Eager task waiting 1 minute")
time.sleep(60)
logger.debug("remove_empty_network task started at %s." %
datetime.now())
logger.debug("CoreIdentity(id=%s)" % core_identity_id)
core_identity = Identity.objects.get(id=core_identity_id)
driver = get_driver(driverCls, provider, identity)
instances = driver.list_instances()
active_instances = False
for instance in instances:
if driver._is_active_instance(instance):
active_instances = True
break
if not active_instances:
inactive_instances = all(driver._is_inactive_instance(
instance) for instance in instances)
#Inactive instances, True: Remove network, False
remove_network = not inactive_instances
#Check for project network
from service.accounts.openstack import AccountDriver as\
OSAccountDriver
os_acct_driver = OSAccountDriver(core_identity.provider)
logger.info("No active instances. Removing project network"
"from %s" % core_identity)
os_acct_driver.delete_network(core_identity,
remove_network=remove_network)
if remove_network:
#Sec. group can't be deleted if instances are suspended
# when instances are suspended we pass remove_network=False
os_acct_driver.delete_security_group(core_identity)
return True
logger.debug("remove_empty_network task finished at %s." %
datetime.now())
return False
except Exception as exc:
logger.exception("Failed to check if project network is empty")
remove_empty_network.retry(exc=exc)
def security_group_init(core_identity, max_attempts = 3):
os_driver = OSAccountDriver(core_identity.provider)
creds = core_identity.get_credentials()
#TODO: Remove kludge when openstack connections can be
#Deemed reliable. Otherwise generalize this pattern so it
#can be arbitrarilly applied to any call that is deemed 'unstable'.
# -Steve
attempt = 0
while attempt < max_attempts:
attempt += 1
security_group = os_driver.init_security_group(
creds['key'], creds['secret'],
creds['ex_tenant_name'], creds['ex_tenant_name'],
os_driver.MASTER_RULES_LIST)
if security_group:
return security_group
time.sleep(2**attempt)
raise SecurityGroupNotCreated()
def get_account_driver(provider):
"""
Create an account driver for a given provider.
"""
try:
type_name = provider.get_type_name().lower()
if 'openstack' in type_name:
from service.accounts.openstack import AccountDriver as\
OSAccountDriver
return OSAccountDriver(provider)
elif 'eucalyptus' in type_name:
from service.accounts.eucalyptus import AccountDriver as\
EucaAccountDriver
return EucaAccountDriver(provider)
except:
logger.exception("Account driver for provider %s not found." %
(provider.location))
return None
def get_account_driver(provider_id):
try:
provider = Provider.objects.get(id=provider_id)
except CoreProvider.DoesNotExist:
return Response('No provider matching id %s' % provider_id,
status=status.HTTP_404_NOT_FOUND)
#TODO: We need better logic here. maybe use provider name?
provider_name = provider.location.lower()
#TODO: How we select args will change..
if 'openstack' in provider_name:
driver = OSAccountDriver(provider)
elif 'eucalyptus' in provider_name:
driver = EucaAccountDriver(provider)
#elif 'aws' in provider_name:
# driver = AWSAccountDriver(provider)
else:
raise Exception("Could not find a driver for provider %s" %
provider_name)
return driver
def start(images):
print 'Initializing account drivers'
euca_accounts = EucaAccountDriver(Provider.objects.get(id=1))
euca_img_class = euca_accounts.image_manager.__class__
euca_img_creds = euca_accounts.image_creds
os_accounts = OSAccountDriver(Provider.objects.get(id=4))
os_img_class = os_accounts.image_manager.__class__
os_img_creds = os_accounts.image_creds
migrate_args = {
'download_dir': "/Storage",
'image_id': None,
'xen_to_kvm': True,
}
print 'Account drivers initialized'
for mach_to_migrate in images:
migrate_args['image_id'] = mach_to_migrate
pm = ProviderMachine.objects.get(identifier=mach_to_migrate)
migrate_args['image_name'] = pm.application.name
print 'Migrating %s..' % mach_to_migrate
# Lookup machine, set nme
migrate_image(euca_img_class, euca_img_creds, os_img_class,
os_img_creds, **migrate_args)
def clear_empty_ips():
logger.debug("clear_empty_ips task started at %s." % datetime.now())
from service import instance as instance_service
from rtwo.driver import OSDriver
from api import get_esh_driver
from service.accounts.openstack import AccountDriver as\
OSAccountDriver
identities = Identity.objects.filter(
provider__type__name__iexact='openstack',
provider__active=True)
typename = ident.provider.type.name
username = ident.created_by.username
key_sorter = lambda ident: attrgetter(typename, username)
identities = sorted(
identities,
key=key_sorter)
os_acct_driver = None
total = len(identities)
for idx, core_identity in enumerate(identities):
try:
#Initialize the drivers
driver = get_esh_driver(core_identity)
if not isinstance(driver, OSDriver):
continue
if not os_acct_driver or\
os_acct_driver.core_provider != core_identity.provider:
os_acct_driver = OSAccountDriver(core_identity.provider)
logger.info("Initialized account driver")
# Get useful info
creds = core_identity.get_credentials()
tenant_name = creds['ex_tenant_name']
logger.info("Checking Identity %s/%s - %s"
% (idx+1, total, tenant_name))
# Attempt to clean floating IPs
num_ips_removed = driver._clean_floating_ip()
if num_ips_removed:
logger.debug("Removed %s ips from OpenStack Tenant %s"
% (num_ips_removed, tenant_name))
#Test for active/inactive instances
instances = driver.list_instances()
active = any(driver._is_active_instance(inst)
for inst in instances)
inactive = all(driver._is_inactive_instance(inst)
for inst in instances)
if active and not inactive:
#User has >1 active instances AND not all instances inactive
pass
elif os_acct_driver.network_manager.get_network_id(
os_acct_driver.network_manager.neutron,
'%s-net' % tenant_name):
#User has 0 active instances OR all instances are inactive
#Network exists, attempt to dismantle as much as possible
remove_network = not inactive
logger.info("Removing project network %s for %s"
% (remove_network, tenant_name))
if remove_network:
#Sec. group can't be deleted if instances are suspended
# when instances are suspended we pass remove_network=False
os_acct_driver.delete_security_group(core_identity)
os_acct_driver.delete_network(
core_identity,
remove_network=remove_network)
else:
#logger.info("No Network found. Skipping %s" % tenant_name)
pass
except Exception as exc:
logger.exception(exc)
logger.debug("clear_empty_ips task finished at %s." % datetime.now())