def action_move():
hostname = nc.get_fqdn(options.host)
instances = nc.get_instances(host=hostname)
if instances:
himutils.sys_error('Host %s not empty. Remove instances first' % hostname)
if nc.move_host_aggregate(hostname=hostname, aggregate=options.aggregate):
print "Host %s moved to aggregate %s" % (hostname, options.aggregate)
def action_update():
dry_run_txt = 'DRY-RUN: ' if options.dry_run else ''
defaults = himutils.load_config('config/quotas/%s' % options.quota_config, logger)
if not defaults:
himutils.sys_error('No default quotas found in config/quota/%s' % options.quota_config)
for region in regions:
novaclient = Nova(options.config, debug=options.debug, log=logger, region=region)
cinderclient = Cinder(options.config, debug=options.debug, log=logger, region=region)
components = {'nova': novaclient, 'cinder': cinderclient}
for comp, client in components.iteritems():
if options.service != 'all' and comp != options.service:
continue
if comp not in defaults:
logger.debug('=> could not find quota for %s in config' % comp)
continue
if hasattr(client, 'get_quota_class'):
current = getattr(client, 'get_quota_class')()
else:
logger.debug('=> function get_quota_class not found for %s' % comp)
continue
if not isinstance(current, dict):
current = current.to_dict()
updates = dict()
for k, v in defaults[comp].iteritems():
if k in current and current[k] != v:
logger.debug("=> %sUpdated %s: from %s to %s in %s" %
(dry_run_txt, k, current[k], v, region))
updates[k] = v
if updates and not options.dry_run:
result = getattr(client, 'update_quota_class')(updates=updates)
logger.debug('=> %s' % result)
elif not updates:
logger.debug('=> no need to update default quota for %s in %s' % (comp, region))
def action_compute():
stats = dict()
for region in regions:
metrics = ['vcpus', 'vcpus_used', 'running_vms', 'memory_mb_used',
'memory_mb', 'local_gb', 'local_gb_used']
nc = Nova(options.config, debug=options.debug, region=region, log=logger)
azs = nc.get_availability_zones()
zone_hosts = dict()
# Availablity zones
for az in azs:
if ('iaas-team-only' in az.zoneName or
region not in az.zoneName or not az.hosts):
continue
for host in az.hosts.iterkeys():
zone_hosts[host] = az.zoneName
stats[az.zoneName] = dict()
for metric in metrics:
stats[az.zoneName][metric] = 0
# Hypervisor hosts
hosts = nc.get_hosts()
for host in hosts:
if not host.hypervisor_hostname in zone_hosts:
himutils.sys_error('host %s not enabled or in valid az'
% host.hypervisor_hostname, 0)
continue
az = zone_hosts[host.hypervisor_hostname]
for metric in metrics:
logger.debug('=> %s %s=%s', host.hypervisor_hostname, metric, getattr(host, metric))
stats[az][metric] = int(getattr(host, metric) + stats[az][metric])
statsd.gauge_dict('compute', stats)
if not options.quiet:
for name, stat in stats.iteritems():
printer.output_dict({'header': name})
printer.output_dict(stat)
def action_disable():
host = nc.get_host(hostname=nc.get_fqdn(options.host), detailed=True)
if not host:
himutils.sys_error('Could not find valid host %s' % options.host)
if host.status != 'disabled' and not options.dry_run:
nc.disable_host(host.hypervisor_hostname)
print 'Host %s disabled' % host.hypervisor_hostname
def action_rename():
if not ksclient.is_valid_user(email=options.old):
himutils.sys_error('User %s not found as a valid user.' % options.old)
personal = ksclient.get_project_name(options.new, prefix='PRIVATE')
new_demo_project = ksclient.get_project_name(options.new)
old_demo_project = ksclient.get_project_name(options.old)
print "\nYou are about to rename user with email %s to %s" % (options.old, options.new)
print "\nWhen a user changes affilation we need to change the following:"
# new
print " * Delete %s-group if it exists" % options.new
print " * Delete %s api user if it exists" % options.new.lower()
print " * Delete %s dataporten user if it exists" % options.new.lower()
print " * Delete %s demo project and instances if exists" % new_demo_project
print " * Delete %s personal project and instances if exist" % personal
# old
print " * Rename group from %s-group to %s-group" % (options.old, options.new)
print " * Rename api user from %s to %s" % (options.old.lower(), options.new.lower())
print " * Rename demo project from %s to %s" % (old_demo_project, new_demo_project)
print " * Delete old dataporten user %s" % (options.old)
question = "\nAre you sure you will continue"
if not himutils.confirm_action(question):
return
print 'Please wait...'
ksclient.user_cleanup(email=options.new)
ksclient.rename_user(new_email=options.new,
old_email=options.old)
def action_update():
public = flavors['public'] if 'public' in flavors else False
properties = flavors['properties'] if 'properties' in flavors else None
if not properties.get('aggregate_instance_extra_specs:type', None):
properties['aggregate_instance_extra_specs:type'] = 's== standard'
if (options.flavor not in flavors or
not isinstance(flavors[options.flavor], dict)):
himutils.sys_error('%s hash not found in config' % options.flavor)
for region in regions:
nc = Nova(options.config, debug=options.debug, log=logger, region=region)
nc.set_dry_run(options.dry_run)
for name, spec in sorted(flavors[options.flavor].iteritems()):
nc.update_flavor(name=name, spec=spec,
properties=properties, public=public)
# Update access
access = nc.get_flavor_access(filters=options.flavor)
all_projects = set()
for name, projects in access.iteritems():
for project_id in projects:
all_projects.add(project_id.tenant_id)
for project in all_projects:
nc.update_flavor_access(filters=options.flavor,
project_id=project,
action='grant')
def action_show():
project = ksclient.get_project_by_name(project_name=options.project)
if not project:
himutils.sys_error('No project found with name %s' % options.project)
output_project = project.to_dict()
output_project['header'] = "Show information for %s" % project.name
printer.output_dict(output_project)
def action_instances():
host = nc.get_host(nc.get_fqdn(options.host))
if not host:
himutils.sys_error('Could not find valid host %s' % options.host)
search_opts = dict(all_tenants=1, host=host.hypervisor_hostname)
instances = nc.get_all_instances(search_opts=search_opts)
printer.output_dict({'header': 'Instance list (id, name, status, updated)'})
status = dict({'total': 0})
for i in instances:
# Filter for project type
if options.type:
project = kc.get_by_id('project', i.tenant_id)
if hasattr(project, 'type') and project.type != options.type:
status['type'] = options.type
continue
output = {
'1': i.id,
'3': i.name,
'4': i.status,
#'2': i.updated,
#'6'': getattr(i, 'OS-EXT-SRV-ATTR:instance_name'),
'5': i.flavor['original_name']
}
printer.output_dict(output, sort=True, one_line=True)
status['total'] += 1
status[str(i.status).lower()] = status.get(str(i.status).lower(), 0) + 1
printer.output_dict({'header': 'Counts'})
printer.output_dict(status)
def action_whales():
start = himutils.get_date(options.start, date.today() - timedelta(days=1))
stop = himutils.get_date(options.end, date.today() + timedelta(days=1))
if start > stop:
himutils.sys_error('start %s must be fore stop %s' % (start, stop))
logger.debug('=> start date = %s', start)
logger.debug('=> stop date = %s', stop)
for region in regions:
nc = Nova(options.config, debug=options.debug, log=logger, region=region)
cc = Cinder(options.config, debug=options.debug, log=logger, region=region)
project_usage = nc.get_usage(start=start, end=stop)
logger.debug('=> threshold for whales filter %s', options.threshold)
print_header = True
for usage in project_usage:
project = kc.get_by_id(obj_type='project', obj_id=usage.tenant_id)
if not project:
logger.debug('=> project with id %s not found',usage.tenant_id)
continue
if len(usage.server_usages) < options.threshold:
continue
cinderusage = cc.get_usage(usage.tenant_id)
admin = project.admin if hasattr(project, 'admin') else 'unknown!'
output = OrderedDict()
output['instances'] = len(usage.server_usages)
output['volume_gb'] = cinderusage.gigabytes['in_use']
output['name'] = project.name
output['admin'] = admin
if print_header:
output['header'] = 'project usage %s (instances, volume (GB), name, id)' % region
print_header = False
printer.output_dict(objects=output, sort=False, one_line=True)
def action_show_access():
project = ksclient.get_project_by_name(project_name=options.project)
if not project:
himutils.sys_error('No project found with name %s' % options.project)
roles = ksclient.list_roles(project_name=options.project)
printer.output_dict({'header': 'Roles in project %s' % options.project})
for role in roles:
printer.output_dict(role, sort=True, one_line=True)
def action_delete():
if not ksclient.is_valid_user(email=options.user):
himutils.sys_error('User %s not found as a valid user.' % options.user)
if not himutils.confirm_action('Delete user and all instances for %s' % options.user):
return
print "We are now deleting user, group, project and instances for %s" % options.user
print 'Please wait...'
ksclient.user_cleanup(email=options.user)
print 'Delete successful'
def update_access(nc, action):
public = flavors['public'] if 'public' in flavors else False
if 'public' in flavors and flavors['public']:
himutils.sys_error('grant or revoke will not work on public flavors!')
project = kc.get_project_by_name(options.project)
if not project:
himutils.sys_error('project not found %s' % project)
nc.update_flavor_access(filters=options.flavor,
project_id=project.id,
action=action)
def action_revoke():
project = ksclient.get_project_by_name(project_name=options.project)
if not project:
himutils.sys_error('No project found with name %s' % options.project)
# Get all users from a project
users = ksclient.get_users(domain=options.domain, project=project.id)
emails = list()
for user in users:
emails.append(user.email)
ksclient.revoke_role(emails=emails, project_name=project.name, role_name='object')
def action_important():
important_msg = msg
if options.link:
important_msg += " For live updates visit https://status.uh-iaas.no"
confirm_publish(important_msg)
if not twitter.twitter_length(important_msg):
himutils.sys_error("Message cannot contain more than 280 characters")
slack.publish_slack(important_msg)
twitter.publish_twitter(important_msg)
status.publish(important_msg, msg_type='important')
def action_create():
if not ksclient.is_valid_user(email=options.admin, domain=options.domain):
himutils.sys_error('%s is not a valid user. Admin must be a valid user' % options.admin, 1)
if options.enddate:
try:
enddate = datetime.strptime(options.enddate, '%d.%m.%y').date()
except ValueError:
himutils.sys_error('date format DD.MM.YY not valid for %s' % options.enddate, 1)
else:
enddate = None
email = options.email if options.email else options.user
ksclient.create_user(name=options.user,
email=email,
password=options.password,
admin=options.admin,
description=options.description,
enddate=str(enddate))
def get_valid_users():
whitelist = himutils.load_file('whitelist_users.txt', logger)
if not whitelist:
himutils.sys_error('Could not find whitelist_users.txt!')
orgs = himutils.load_config('config/ldap.yaml', logger).keys()
ldap = dict()
for org in orgs:
ldap[org] = LdapClient(options.config, debug=options.debug, log=logger)
ldap[org].bind(org)
users = ksclient.list_users(domain=options.domain)
deactive = list()
active = dict()
unknown = list()
count = 0
for user in users:
if options.limit and count >= int(options.limit):
break
count += 1
if user in whitelist:
ksclient.debug_log('user %s in whitelist' % user)
# Only add a user to deactive if user also enabled in OS
os_user = ksclient.get_user_by_email(user, 'api')
if not os_user.enabled:
continue
org_found = False
for org in orgs:
if not org in user:
continue
org_found = True
if not ldap[org].get_user(user) and user not in whitelist:
deactive.append(user)
else:
active[org] = active.setdefault(org, 0) + 1
break
if not org_found:
#print "%s org not found" % user
if '@' in user:
org = user.split("@")[1]
if org not in unknown:
unknown.append(org)
time.sleep(2)
total = 0
for k,v in active.iteritems():
total += v
active['total'] = total
return (active, deactive, unknown)
def action_list_access():
for region in regions:
nc = Nova(options.config, debug=options.debug, log=logger, region=region)
nc.set_dry_run(options.dry_run)
access = nc.get_flavor_access(filters=options.flavor)
header = 'access to %s flavor in %s' % (options.flavor, region)
printer.output_dict({'header': header})
output = dict()
for name, projects in access.iteritems():
output[name] = list()
for project_id in projects:
project = kc.get_by_id('project', project_id.tenant_id)
if project:
output[name].append(project.name)
else:
himutils.sys_error('project not found %s' % project_id.tenant_id, 0)
continue
printer.output_dict(output)
def action_instances():
project = ksclient.get_project_by_name(project_name=options.project)
for region in regions:
novaclient = Nova(options.config, debug=options.debug, log=logger)
instances = novaclient.get_project_instances(project_id=project.id)
if not instances:
himutils.sys_error('No instances found for the project %s' % options.project)
printer.output_dict({'header': 'Instances list (id, name, region)'})
count = 0
for i in instances:
output = {
'id': i.id,
'name': i.name,
'region': region,
}
count += 1
printer.output_dict(output, sort=True, one_line=True)
printer.output_dict({'header': 'Total instances in this project', 'count': count})
def action_flavors():
project = kc.get_project_by_name(options.project)
start = himutils.get_date(options.start, date.today() - timedelta(days=1))
stop = himutils.get_date(options.end, date.today() + timedelta(days=1))
if start > stop:
himutils.sys_error('start %s must be fore stop %s' % (start, stop))
logger.debug('=> start date = %s', start)
logger.debug('=> stop date = %s', stop)
flavors = dict()
for region in regions:
nc = Nova(options.config, debug=options.debug, log=logger, region=region)
usage = nc.get_usage(project_id=project.id, start=start, end=stop)
if not hasattr(usage, 'server_usages'):
continue
for server in usage.server_usages:
flavors[server['flavor']] = flavors.get(server['flavor'], 0) + 1
flavors['header'] = 'flavor usage for %s in all regions' % project.name
printer.output_dict(flavors)
def action_grant():
print "WARNING: this will break with the upgrade to ocata"
print "https://trello.com/c/vjRI4EKC/"
ksclient = Keystone(options.config, debug=options.debug, log=logger)
project = ksclient.get_project(project=options.project, domain=options.domain)
if not project:
himutils.sys_error('Unknown project %s in domain %s' %
(options.project, options.domain))
if options.name:
tags.append(options.name)
filters = {'status': 'active', 'tag': tags, 'visibility': 'private'}
logger.debug('=> filter: %s' % filters)
images = glclient.get_images(filters=filters)
for image in images:
log_msg = 'grant access to %s from %s' % (image.name, project.name)
if not options.dry_run:
glclient.set_access(image.id, project.id)
else:
log_msg = 'DRY-RUN: %s' % log_msg
logger.debug('=> %s' % log_msg)
def action_migrate():
# Find enabled aggregate
aggregates = novaclient.get_aggregates()
active_aggregate = 'unknown'
for aggregate in aggregates:
if aggregate not in legacy_aggregate:
continue
info = novaclient.get_aggregate(aggregate)
if 'enabled' in info.metadata:
active_aggregate = aggregate
break
if active_aggregate == 'unknown':
himutils.sys_error('Could not find enabled aggregate to migrate to. Make sure to activate aggregate first!')
q = 'Migrate all instances from %s to %s' % (options.aggregate, active_aggregate)
if not himutils.confirm_action(q):
return
instances = novaclient.get_instances(options.aggregate, host)
count = 0
target_host = next(iter(novaclient.get_aggregate_hosts(active_aggregate) or []), None)
if not target_host:
himutils.sys_error('Could not find valid host in aggregate %s' % active_aggregate)
for instance in instances:
count += 1
if options.dry_run:
logger.debug('=> DRY-RUN: migrate instance %s' % unicode(instance.name))
else:
logger.debug('=> migrate instance %s' % unicode(instance.name))
try:
instance.migrate(host=target_host.hypervisor_hostname)
time.sleep(2)
if count%options.limit == 0 and (options.hard_limit and count < options.limit):
logger.debug('=> sleep for %s seconds', options.sleep)
time.sleep(options.sleep)
except novaexc.BadRequest as e:
sys.stderr.write("%s\n" % e)
sys.stderr.write("Error found. Cancel migration!\n")
break
if options.hard_limit and count >= options.limit:
logger.debug('=> use of hard limit and exit after %s instances', options.limit)
break
def action_migrate():
target = nc.get_fqdn(options.target)
if not nc.get_host(target):
himutils.sys_error('Could not find target host %s' % target)
q = 'Migrate all instances from %s to %s' % (source, target)
if not himutils.confirm_action(q):
return
# Disable source host unless no-disable param is used
if not options.dry_run and not options.no_disable:
nc.disable_host(source)
dry_run_txt = 'DRY_RUN: ' if options.dry_run else ''
instances = nc.get_all_instances(search_opts=search_opts)
count = 0
for i in instances:
state = getattr(i, 'OS-EXT-STS:vm_state')
state_task = getattr(i, 'OS-EXT-STS:task_state')
if state_task:
logger.debug('=> instance running task %s, dropping migrate', state_task)
continue
logger.debug('=> %smigrate %s to %s', dry_run_txt, i.name, target)
if (state == 'active' or state == 'paused') and not options.dry_run:
i.live_migrate(host=target)
time.sleep(options.sleep)
elif state == 'stopped' and not options.dry_run:
i.migrate(host=target)
time.sleep(options.sleep)
# elif state == 'suspended' and not options.dry_run:
# i.resume()
# time.sleep(2)
# i.pause()
# time.sleep(5)
# i.live_migrate(host=target)
# time.sleep(options.sleep)
elif not options.dry_run:
logger.debug('=> dropping migrate of %s unknown state %s', i.name, state)
count += 1
if options.limit and count >= options.limit:
logger.debug('=> number of instances reached limit %s', options.limit)
break
def action_file():
if options.template:
content = options.template
email_content = open(content, 'r')
body_content = email_content.read()
if options.dry_run:
print body_content
else:
mail = Mail(options.config, debug=options.debug)
msg = MIMEText(body_content)
msg['subject'] = subject
print msg
with open(emails_file, 'r') as emails:
for toaddr in emails.readlines():
try:
logger.debug('=> Sending email ...')
mail.send_mail(toaddr, msg, fromaddr='*****@*****.**')
except ValueError:
himutils.sys_error('Not able to send the email.')
emails.close()
email_content.close()
mail.close()
def action_evacuate():
source_host = nc.get_host(source)
if source_host.state != 'down':
himutils.sys_error('Evacuate failed. Source host need to be down! Use migrate')
# Check that there are other valid hosts in the same aggregate
hosts = nc.get_aggregate_hosts(options.aggregate)
found_enabled = list()
for host in hosts:
if host.hypervisor_hostname == source:
continue
if host.status == 'enabled' and host.state == 'up':
found_enabled.append(host.hypervisor_hostname)
if not found_enabled:
himutils.sys_error('Evacuate failed. No valid host in aggregate %s'
% options.aggregate)
logger.debug('=> valid host found %s', ", ".join(found_enabled))
# Interactive question
q = 'Evacuate all instances from %s to other hosts' % source
if not himutils.confirm_action(q):
return
instances = nc.get_all_instances(search_opts=search_opts)
dry_run_txt = 'DRY_RUN: ' if options.dry_run else ''
count = 0
for i in instances:
state = getattr(i, 'OS-EXT-STS:vm_state')
logger.debug('=> %sevacuate %s', dry_run_txt, i.name)
if state == 'active' and not options.dry_run:
i.evacuate()
time.sleep(options.sleep)
elif state == 'stopped' and not options.dry_run:
i.evacuate()
time.sleep(options.sleep)
elif not options.dry_run:
logger.debug('=> dropping evacuate of %s unknown state %s', i.name, state)
count += 1
if options.limit and count > options.limit:
logger.debug('=> number of instances reached limit %s', options.limit)
break
def action_grant():
for user in options.users:
if not ksclient.is_valid_user(email=user, domain=options.domain):
himutils.sys_error('User %s not found as a valid user.' % user)
project = ksclient.get_project_by_name(project_name=options.project)
if not project:
himutils.sys_error('No project found with name "%s"' %
options.project)
if hasattr(project, 'type') and (project.type == 'demo'
or project.type == 'personal'):
himutils.sys_error('Project are %s. User access not allowed!' %
project.type)
role = ksclient.grant_role(project_name=options.project, email=user)
if role:
output = role.to_dict() if not isinstance(role, dict) else role
output['header'] = "Roles for %s" % options.project
printer.output_dict(output)
if options.mail:
mail = Mail(options.config, debug=options.debug)
mail.set_dry_run(options.dry_run)
if options.rt is None:
himutils.sys_error('--rt parameter is missing.')
else:
rt_mapping = dict(users='\n'.join(options.users))
rt_subject = 'NREC: Access granted to users in %s' % options.project
rt_body_content = himutils.load_template(
inputfile=access_granted_msg_file, mapping=rt_mapping)
rt_mime = mail.rt_mail(options.rt, rt_subject, rt_body_content)
mail.send_mail('*****@*****.**', rt_mime)
for user in options.users:
mapping = dict(project_name=options.project, admin=project.admin)
body_content = himutils.load_template(
inputfile=access_granted_user_msg_file, mapping=mapping)
msg = MIMEText(body_content, 'plain')
msg['subject'] = 'NREC: You have been given access to project %s' % options.project
mail.send_mail(user, msg, fromaddr='*****@*****.**')
def action_grant():
for user in options.users:
if not ksclient.is_valid_user(email=user, domain=options.domain):
himutils.sys_error('User %s not found as a valid user.' % user)
project = ksclient.get_project_by_name(project_name=options.project)
if not project:
himutils.sys_error('No project found with name "%s"' % options.project)
if hasattr(project, 'type') and (project.type == 'demo' or project.type == 'personal'):
himutils.sys_error('Project are %s. User access not allowed!' % project.type)
role = ksclient.grant_role(project_name=options.project,
email=user)
if role:
output = role.to_dict() if not isinstance(role, dict) else role
output['header'] = "Roles for %s" % options.project
printer.output_dict(output)
if options.mail:
mail = Mail(options.config, debug=options.debug)
mail.set_dry_run(options.dry_run)
if options.rt is None:
himutils.sys_error('--rt parameter is missing.')
else:
rt_mapping = dict(users='\n'.join(options.users))
rt_subject = 'UH-IaaS: Access granted to users in %s' % options.project
rt_body_content = himutils.load_template(inputfile=access_msg_file,
mapping=rt_mapping)
rt_mime = mail.rt_mail(options.rt, rt_subject, rt_body_content)
mail.send_mail('*****@*****.**', rt_mime)
for user in options.users:
mapping = dict(project_name=options.project, admin=project.admin)
body_content = himutils.load_template(inputfile=access_user_msg_file,
mapping=mapping)
msg = MIMEText(body_content, 'plain')
msg['subject'] = 'UH-IaaS: You have been given access to project %s' % options.project
mail.send_mail(user, msg, fromaddr='*****@*****.**')
def action_grant():
if not ksclient.is_valid_user(email=options.user, domain=options.domain):
himutils.sys_error('User %s not found as a valid user.' % options.user)
project = ksclient.get_project_by_name(project_name=options.project,
domain=options.domain)
if not project:
himutils.sys_error('No project found with name %s' % options.project)
if hasattr(project, 'type') and (project.type == 'demo'
or project.type == 'personal'):
himutils.sys_error('Project are %s. User access not allowed!' %
project.type)
role = ksclient.grant_role(project_name=options.project,
email=options.user,
domain=options.domain)
if role:
output = role.to_dict() if not isinstance(role, dict) else role
output['header'] = "Roles for %s" % options.project
printer.output_dict(output)
def update_access(nc, access_action, region):
flavors = get_flavor_config(region)
if options.flavor in flavors and not flavors[options.flavor]:
return False
if 'public' in flavors and flavors['public']:
himutils.sys_error('grant or revoke will not work on public flavors!')
project = kc.get_project_by_name(options.project)
if not project:
himutils.sys_error('project not found %s' % project)
result = nc.update_flavor_access(class_filter=options.flavor,
project_id=project.id,
action=access_action)
if not result:
himutils.sys_error(
'Update access for {} failed. Check debug log'.format(
options.flavor), 0)
return False
return True
def action_show():
host = nc.get_host(hostname=nc.get_fqdn(options.host), detailed=True)
if not host:
himutils.sys_error('Could not find valid host %s' % options.host)
printer.output_dict(host.to_dict())
def action_show():
host = nc.get_host(hostname=nc.get_fqdn(options.host), detailed=True)
if not host:
himutils.sys_error('Could not find valid host %s' % options.host)
printer.output_dict(host.to_dict())
parser.set_autocomplete(True)
options = parser.parse_args()
printer = Printer(options.format)
ksclient = Keystone(options.config, debug=options.debug)
ksclient.set_dry_run(options.dry_run)
ksclient.set_domain(options.domain)
logger = ksclient.get_logger()
if hasattr(options, 'region'):
regions = ksclient.find_regions(region_name=options.region)
else:
regions = ksclient.find_regions()
if not regions:
utils.sys_error('no regions found with this name!')
#---------------------------------------------------------------------
# Main functions
#---------------------------------------------------------------------
def action_show():
project = ksclient.get_project_by_name(project_name=options.project)
if not project:
utils.sys_error('No project found with name %s' % options.project)
sys.stdout.write(
Printer.prettyprint_project_metadata(project, options, logger,
regions))
if options.detail:
sys.stdout.write(
Printer.prettyprint_project_zones(project, options, logger))
#!/usr/bin/env python
import sys
import utils
import pprint
from himlarcli.keystone import Keystone
from himlarcli.nova import Nova
from himlarcli import utils as himutils
himutils.is_virtual_env()
himutils.sys_error('code review needed!!!!')
# Input args
desc = 'Mange student course project'
actions = ['create', 'list', 'delete']
opt_args = {
'-n': {
'dest': 'name',
'help': 'course name (mandatory)',
'required': True,
'metavar': 'name'
},
'-u': {
'dest': 'user',
'help': 'email of teacher',
'metavar': 'user'
},
'-q': {
'dest': 'quota',
'help': 'project quota',
'default': 'course',
hosts = foreman.get_hosts('*')
hostlist = dict()
for host in hosts['results']:
var = None
hostname = host['name'].split('.')[0]
check = hostname.count('-')
if check == 2:
loc, role, num = hostname.split('-')
#Example if test01-nat-linux-01
if check == 3:
loc, role, var, num = hostname.split('-')
try:
if not loc or not role:
pass
except:
himutils.sys_error('Broken hostname %s! Please fix!' % hostname, 0)
continue
group = "%s-%s" % (loc, role)
if not group in hostlist:
hostlist[group] = []
hostlist[group].append(hostname)
# Add group for var
if var:
var_group = group = "%s-%s-%s" % (loc, role, var)
if not var_group in hostlist:
hostlist[var_group] = []
hostlist[var_group].append(hostname)
children = "%s:children" % loc
logger.debug('=> stop date = %s', stop)
output = dict({'vcpu': 0, 'ram':0})
for region in regions:
# instances
nc = Nova(options.config, debug=options.debug, log=logger, region=region)
gc = Gnocchi(options.config, debug=options.debug, log=logger, region=region)
deleted = nc.get_project_instances(project_id=project.id, deleted=True)
running = nc.get_project_instances(project_id=project.id)
for i in deleted + running:
resource = gc.get_resource(resource_type='instance', resource_id=i.id)
if not resource:
continue
metrics = dict()
metrics['vcpu'] = gc.get_client().metric.get('vcpus', i.id)
metrics['ram'] = gc.get_client().metric.get('memory', i.id)
for key, value in metrics.iteritems():
measurement = gc.get_client().metric.get_measures(metric=value['id'],
aggregation='max',
start=start,
stop=stop)
if measurement:
output[key] += measurement[0][2]
printer.output_dict({'header': 'resources used by %s in all regions' % project.name})
printer.output_dict(output)
# Run local function with the same name as the action
action = locals().get('action_' + options.action)
if not action:
himutils.sys_error("Function action_%s() not implemented" % options.action)
action()
parser = Parser()
options = parser.parse_args()
printer = Printer(options.format)
ksclient = Keystone(options.config, debug=options.debug)
ksclient.set_dry_run(options.dry_run)
logger = ksclient.get_logger()
if hasattr(options, 'region'):
regions = ksclient.find_regions(region_name=options.region)
else:
regions = ksclient.find_regions()
if not regions:
himutils.sys_error('no valid regions found!')
def action_volume():
projects = ksclient.get_projects(domain=options.domain)
for region in regions:
cinderclient = Cinder(options.config, debug=options.debug, log=logger, region=region)
# Quotas
quotas = dict({'in_use': 0, 'quota': 0})
for project in projects:
if not hasattr(project, "type"): # unknown project type
logger.debug('=> unknow project type %s', project.name)
continue
# Filter demo
if not options.demo and project.type == 'demo':
continue
quota = cinderclient.list_quota(project_id=project.id, usage=True)
def action_password():
if not ksclient.is_valid_user(email=options.user, domain=options.domain):
himutils.sys_error("%s is not a valid user." % options.user, 1)
print ksclient.reset_password(email=options.user)
def action_test():
novaclient = Nova(options.config, debug=options.debug, log=logger, region=options.region)
neutronclient = Neutron(options.config, debug=options.debug, log=logger, region=options.region)
filters = {'status': 'active', 'visibility': options.visibility, 'tag': tags}
logger.debug('=> filter: %s' % filters)
images = glclient.get_images(filters=filters)
flavors = novaclient.get_flavors('m1')
networks = neutronclient.list_networks()
secgroup_name = 'image_test-' + str(int(time.time()))
secgroup = neutronclient.create_security_port_group(secgroup_name, 22)
tests = dict({'passed': 0, 'failed': 0, 'dropped': 0})
for image in images:
if options.name and options.name not in image.tags:
logger.debug('=> dropped: image name %s not in tags %s', options.name,
', '.join(image.tags))
continue
for network in networks:
if network['name'] == 'imagebuilder':
continue
try:
starttime = int(time.time())
print '* Create instance from %s with network %s' % (image.name, network['name'])
flavor = glclient.find_optimal_flavor(image, flavors)
if not flavor:
print '* Could not optimal find flavor for %s' % image.name
print '-------------------------------------------------------------'
continue
logger.debug('=> use %s flavor' % flavor.name)
nics = list()
nics.append({'net-id': network['id']})
server = novaclient.create_server(name='image_test'+ str(int(time.time())),
flavor=flavor,
image_id=image.id,
security_groups=[secgroup['id']],
nics=nics)
timeout = 300 # 5 min timeout
if not server:
print '-------------------------------------------------------------'
continue
server = novaclient.get_instance(server.id)
while timeout > 0 and server.status == 'BUILD':
time.sleep(2)
timeout -= 2
server = novaclient.get_instance(server.id)
if timeout <= 0:
print ('* Could not start instance from image %s in %s seconds' %
(image.name, timeout))
if server.status == 'ERROR':
print '* Instance started with error'
print server.fault
else:
used_time = int(time.time()) - starttime
print '* Instance started after %s sec' % used_time
if server.addresses:
for net in server.addresses[network['name']]:
starttime = int(time.time())
ip = IP(net['addr'])
print ('* Instance started with IPv%s %s (%s)' %
(net['version'], ip, ip.iptype()))
if ip.iptype() == 'ALLOCATED RIPE NCC':
print '* Drop connection check for IPv6 for now'
tests['dropped'] += 1
continue
elif ip.iptype() == 'PRIVATE':
print '* Drop connection check for rfc1918 address for now'
tests['dropped'] += 1
continue
timeout = 90
port = False
while timeout > 0 and not port:
start = int(time.time())
port = himutils.check_port(address=str(ip), port=22, timeout=2, log=logger)
time.sleep(3)
timeout -= (int(time.time()) - start)
used_time = int(time.time()) - starttime
if port:
print '* Port 22 open on %s (%s)' % (ip, ip.iptype())
tests['passed'] += 1
else:
print ('* Unable to reach port 22 on %s after %s sec (%s)'
% (ip, used_time, ip.iptype()))
tests['failed'] += 1
else:
print '* No IP found for instances %s' % server.name
try:
server.delete()
time.sleep(3)
print '* Instance deleted'
except:
himutils.sys_error('error!!!')
print '-------------------------------------------------------------'
except KeyboardInterrupt:
if server:
server.delete()
time.sleep(5)
print '* Instance deleted'
print '* Delete security group'
neutronclient.delete_security_group(secgroup['id'])
printer.output_dict({'header': 'Result'})
printer.output_dict(tests)
def action_create():
quota = himutils.load_config('config/quotas/%s.yaml' % options.quota)
if options.quota and not quota:
himutils.sys_error('Could not find quota in config/quotas/%s.yaml' %
options.quota)
test = 1 if options.type == 'test' else 0
if options.enddate:
try:
enddate = datetime.strptime(options.enddate, '%d.%m.%y').date()
except ValueError:
himutils.sys_error(
'date format DD.MM.YY not valid for %s' % options.enddate, 1)
else:
enddate = None
createdate = datetime.today()
project = ksclient.create_project(domain=options.domain,
project_name=options.project,
admin=options.admin.lower(),
test=test,
type=options.type,
description=options.desc,
enddate=str(enddate),
createdate=createdate.isoformat(),
quota=options.quota)
if project:
output = project.to_dict() if not isinstance(project,
dict) else project
output['header'] = "Show information for %s" % options.project
printer.output_dict(output)
# Quotas
for region in regions:
novaclient = Nova(options.config,
debug=options.debug,
log=logger,
region=region)
cinderclient = Cinder(options.config,
debug=options.debug,
log=logger,
region=region)
neutronclient = Neutron(options.config,
debug=options.debug,
log=logger,
region=region)
cinderclient.set_dry_run(options.dry_run)
novaclient.set_dry_run(options.dry_run)
neutronclient.set_dry_run(options.dry_run)
if project and not isinstance(project, dict):
project_id = project.id
elif project and isinstance(project, dict) and 'id' in project:
project_id = project['id']
else:
project_id = None
if quota and 'cinder' in quota and project:
cinderclient.update_quota(project_id=project_id,
updates=quota['cinder'])
if quota and 'nova' in quota and project:
novaclient.update_quota(project_id=project_id,
updates=quota['nova'])
if quota and 'neutron' in quota and project:
neutronclient.update_quota(project_id=project_id,
updates=quota['neutron'])
def action_info():
if not twitter.twitter_length(msg):
himutils.sys_error("Message cannot contain more than 280 characters")
confirm_publish(msg)
twitter.publish_twitter(msg)
status.publish(msg)
def action_info():
if not twitter.twitter_length(msg):
himutils.sys_error("Message cannot contain more than 280 characters")
confirm_publish(msg)
twitter.publish_twitter(msg)
status.publish(msg)
def action_event():
confirm_publish(msg)
status.publish(msg, msg_type='event')
if options.message:
msg = options.message
elif options.template:
msg = parse_template()
else:
himutils.sys_error("No template or message given.")
slack.set_dry_run(options.dry_run)
twitter.set_dry_run(options.dry_run)
status.set_dry_run(options.dry_run)
# Run local function with the same name as the action
action = locals().get('action_' + options.action)
if not action:
himutils.sys_error("Function action_%s() not implemented" % options.action)
action()
def action_notify():
question = 'Send mail to all users about demo projects'
if options.dry_run:
question = 'DRY-RUN: %s' % question
if not himutils.confirm_action(question):
return
projects = ksclient.get_projects(domain=options.domain)
count = 0
for project in projects:
if hasattr(project, 'notify') and project.notify == 'converted':
himutils.sys_error(
'personal project %s converted' % (project.name), 0)
continue
found = False
if hasattr(project, 'type') and project.type == 'personal':
print "%s (new personal project)" % project.name
count += 1
found = True
elif '@' in project.name and not hasattr(project, 'type'):
print "%s (old personal project)" % project.name
count += 1
found = True
if found:
if '@' not in project.name:
himutils.sys_error(
'unable to find email for project %s' % project.name, 0)
continue
user = ksclient.get_user_by_email(project.name, 'api',
options.domain)
if not user:
himutils.sys_error(
'unable to find user for project %s' % project.name, 0)
continue
search_filter = dict({'type': 'demo'})
demo_project = ksclient.get_user_projects(email=user.email.lower(),
domain=options.domain,
**search_filter)
demo_project = demo_project[0] if demo_project else None
if not demo_project:
himutils.sys_error(
'unable to find demo project for %s' % user.name, 0)
continue
delete_date = '2017-10-31'
sent_email = himutils.load_file('temp_email.txt',
log=ksclient.get_logger())
if user.email in sent_email:
himutils.sys_error('%s email sent, dropping' % user.email, 0)
continue
#ksclient.update_project(project_id=project.id, notify=delete_date)
#mapping = dict(region=region.upper(), project=project.name)
mapping = {
'personal': project.name,
'date': delete_date,
'demo': demo_project.name
}
body_content = himutils.load_template(
inputfile='misc/notify_demo2.txt', mapping=mapping, log=logger)
subject = ('[UH-IaaS] Your personal project will be deleted')
notify = Notify(options.config, debug=False, log=logger)
notify.set_dry_run(options.dry_run)
notify.mail_user(body_content, subject, user.email)
notify.close()
time.sleep(3)
printer.output_dict({'Personal projects': count})
parser = Parser()
options = parser.parse_args()
printer = Printer(options.format)
kc = Keystone(options.config, debug=options.debug)
kc.set_dry_run(options.dry_run)
logger = kc.get_logger()
nc = Nova(options.config, debug=options.debug, log=logger)
nc.set_dry_run(options.dry_run)
source = nc.get_fqdn(options.source)
search_opts = dict(all_tenants=1, host=source)
if not nc.get_host(source):
himutils.sys_error('Could not find source host %s' % source)
def action_list():
instances = nc.get_all_instances(search_opts=search_opts)
printer.output_dict({'header': 'Instance list (id, name, state, task)'})
for i in instances:
output = {
'id': i.id,
'name': i.name,
'state': getattr(i, 'OS-EXT-STS:vm_state'),
'state_task': getattr(i, 'OS-EXT-STS:task_state')
}
printer.output_dict(output, sort=True, one_line=True)
def action_migrate():
target = nc.get_fqdn(options.target)
if old is not None:
logger.debug('=> update owner for ip %s', owner['ip'])
old.update(owner)
else:
logger.debug('=> create owner for ip %s', owner['ip'])
session.add(Owner(**owner))
session.commit()
session.close()
def action_purge():
engine = create_engine(kc.get_config('db', 'database_uri'))
Base.metadata.bind = engine
DBSession = sessionmaker(bind=engine)
session = DBSession()
for region in regions:
nc = Nova(options.config, debug=options.debug, log=logger, region=region)
instances = nc.get_all_instances({'all_tenants': 1, 'deleted': 1})
for i in instances:
old = session.query(Owner).filter(Owner.instance_id == i.id).first()
if old is not None:
logger.debug('=> purge owner for instance %s', old.ip)
session.delete(old)
session.commit()
session.close()
# Run local function with the same name as the action
action = locals().get('action_' + options.action)
if not action:
himutils.sys_error("Function action_%s() not implemented" % options.action)
action()
def action_enable():
if not ksclient.is_valid_user(email=options.user):
himutils.sys_error('User %s not found as a valid user.' % options.user)
user = ksclient.get_user_by_email(options.user, 'api')
ksclient.update_user(user_id=user.id, enabled=True, disabled='None')
print "User %s enabled" % user.name
def action_test():
novaclient = Nova(options.config,
debug=options.debug,
log=logger,
region=options.region)
neutronclient = Neutron(options.config,
debug=options.debug,
log=logger,
region=options.region)
filters = {
'status': 'active',
'visibility': options.visibility,
'tag': tags
}
logger.debug('=> filter: %s' % filters)
images = glclient.get_images(filters=filters)
flavors = novaclient.get_flavors('m1')
networks = neutronclient.list_networks()
secgroup_name = 'image_test-' + str(int(time.time()))
secgroup = neutronclient.create_security_port_group(secgroup_name, 22)
for image in images:
if options.name and options.name not in image.tags:
logger.debug('=> dropped: image name %s not in tags %s',
options.name, ', '.join(image.tags))
continue
for network in networks:
if network['name'] == 'imagebuilder':
continue
try:
starttime = int(time.time())
print '* Create instance from %s with network %s' % (
image.name, network['name'])
flavor = glclient.find_optimal_flavor(image, flavors)
if not flavor:
print '* Could not optimal find flavor for %s' % image.name
print '-------------------------------------------------------------'
continue
logger.debug('=> use %s flavor' % flavor.name)
nics = list()
nics.append({'net-id': network['id']})
server = novaclient.create_server(
name='image_test' + str(int(time.time())),
flavor=flavor,
image_id=image.id,
security_groups=[secgroup['id']],
nics=nics)
timeout = 300 # 5 min timeout
if not server:
print '-------------------------------------------------------------'
continue
server = novaclient.get_instance(server.id)
while timeout > 0 and server.status == 'BUILD':
time.sleep(2)
timeout -= 2
server = novaclient.get_instance(server.id)
if timeout <= 0:
print(
'* Could not start instance from image %s in %s seconds'
% (image.name, timeout))
if server.status == 'ERROR':
print '* Instance started with error'
print server.fault
else:
used_time = int(time.time()) - starttime
print '* Instance started after %s sec' % used_time
if server.addresses:
for net in server.addresses[network['name']]:
starttime = int(time.time())
ip = IP(net['addr'])
print('* Instance started with IPv%s %s (%s)' %
(net['version'], ip, ip.iptype()))
if ip.iptype() == 'ALLOCATED RIPE NCC':
print '* Drop connection check for IPv6 for now'
continue
elif ip.iptype() == 'PRIVATE':
print '* Drop connection check for rfc1918 address for now'
continue
timeout = 90
port = False
while timeout > 0 and not port:
start = int(time.time())
port = himutils.check_port(address=str(ip),
port=22,
timeout=2,
log=logger)
time.sleep(3)
timeout -= (int(time.time()) - start)
used_time = int(time.time()) - starttime
if port:
print '* Port 22 open on %s (%s)' % (ip,
ip.iptype())
else:
print(
'* Unable to reach port 22 on %s after %s sec (%s)'
% (ip, used_time, ip.iptype()))
else:
print '* No IP found for instances %s' % server.name
try:
server.delete()
time.sleep(3)
print '* Instance deleted'
except:
himutils.sys_error('error!!!')
print '-------------------------------------------------------------'
except KeyboardInterrupt:
if server:
server.delete()
time.sleep(5)
print '* Instance deleted'
print '* Delete security group'
neutronclient.delete_security_group(secgroup['id'])
def action_create():
if not ksclient.is_valid_user(
options.admin, options.domain) and options.type == 'personal':
himutils.sys_error('not valid user', 1)
quota = himutils.load_config('config/quotas/%s.yaml' % options.quota)
if options.quota and not quota:
himutils.sys_error('Could not find quota in config/quotas/%s.yaml' %
options.quota)
test = 1 if options.type == 'test' else 0
project_msg = project_msg_file
enddate = himutils.get_date(options.enddate, None, '%d.%m.%Y')
if options.type == 'hpc':
project_msg = project_hpc_msg_file
if not enddate:
himutils.sys_error('HPC projects must have an enddate', 1)
createdate = datetime.today()
# Parse the "contact" option, setting to None if not used
# Exit with error if contact is not a valid email address
contact = None
if options.contact is not None:
contact = options.contact.lower()
if not ksclient._Keystone__validate_email(contact):
errmsg = "%s is not a valid email address." % contact
himutils.sys_error(errmsg, 1)
if not options.force:
print 'Project name: %s\nDescription: %s\nAdmin: %s\nContact: %s\nOrganization: %s\nType: %s\nEnd date: %s\nQuota: %s\nRT: %s' \
% (options.project,
ksclient.convert_ascii(options.desc),
options.admin.lower(),
contact,
options.org,
options.type,
str(enddate),
options.quota,
options.rt)
if not himutils.confirm_action(
'Are you sure you want to create this project?'):
himutils.sys_error('Aborted', 1)
project = ksclient.create_project(project_name=options.project,
admin=options.admin.lower(),
contact=contact,
org=options.org,
test=test,
type=options.type,
description=options.desc,
enddate=str(enddate),
createdate=createdate.isoformat(),
quota=options.quota,
rt=options.rt)
if not ksclient.is_valid_user(options.admin, options.domain):
himutils.sys_error(
'WARNING: "%s" is not a valid user.' % options.admin, 0)
if not project:
himutils.sys_error('Failed creating %s' % options.project, 1)
else:
output = Keystone.get_dict(project)
output['header'] = "Show information for %s" % options.project
printer.output_dict(output)
# Do stuff for regions
for region in regions:
# Get objects
novaclient = himutils.get_client(Nova, options, logger, region)
cinderclient = himutils.get_client(Cinder, options, logger, region)
neutronclient = himutils.get_client(Neutron, options, logger, region)
glanceclient = himutils.get_client(Glance, options, logger, region)
# Find the project ID
project_id = Keystone.get_attr(project, 'id')
# Update quotas for Cinder, Nova, Neutron
if quota and 'cinder' in quota and project:
cinderclient.update_quota(project_id=project_id,
updates=quota['cinder'])
if quota and 'nova' in quota and project:
novaclient.update_quota(project_id=project_id,
updates=quota['nova'])
if quota and 'neutron' in quota and project:
neutronclient.update_quota(project_id=project_id,
updates=quota['neutron'])
# Grant UiO Managed images if shared UiO project
if options.org == 'uio' and options.type not in ['personal', 'demo']:
tags = ['uio']
filters = {'status': 'active', 'tag': tags, 'visibility': 'shared'}
images = glanceclient.get_images(filters=filters)
for image in images:
glanceclient.set_image_access(image_id=image.id,
project_id=project.id,
action='grant')
printer.output_msg(
'GRANT access to image {} for project {}'.format(
image.name, project.name))
if options.mail:
mail = Mail(options.config, debug=options.debug)
mail.set_dry_run(options.dry_run)
if options.rt is None:
himutils.sys_error('--rt parameter is missing.')
else:
mapping = dict(project_name=options.project,
admin=options.admin.lower(),
quota=options.quota,
end_date=str(enddate))
subject = 'NREC: Project %s has been created' % options.project
body_content = himutils.load_template(inputfile=project_msg,
mapping=mapping)
if not body_content:
himutils.sys_error('ERROR! Could not find and parse mail body in \
%s' % options.msg)
mime = mail.rt_mail(options.rt, subject, body_content)
mail.send_mail('*****@*****.**', mime)
from himlarcli.parser import Parser
from himlarcli.printer import Printer
from himlarcli import utils as himutils
himutils.is_virtual_env()
parser = Parser()
options = parser.parse_args()
printer = Printer(options.format)
ksclient = Keystone(options.config, debug=options.debug)
logger = ksclient.get_logger()
regions = ksclient.find_regions(region_name=options.region)
if not regions:
himutils.sys_error('no regions found!')
def action_show():
for region in regions:
novaclient = Nova(options.config,
debug=options.debug,
log=logger,
region=region)
cinderclient = Cinder(options.config,
debug=options.debug,
log=logger,
region=region)
components = {'nova': novaclient, 'cinder': cinderclient}
for comp, client in components.iteritems():
if options.service != 'all' and comp != options.service:
parser = Parser()
options = parser.parse_args()
printer = Printer(options.format)
kc = Keystone(options.config, debug=options.debug)
kc.set_dry_run(options.dry_run)
logger = kc.get_logger()
nc = Nova(options.config, debug=options.debug, log=logger)
nc.set_dry_run(options.dry_run)
source = nc.get_fqdn(options.source)
search_opts = dict(all_tenants=1, host=source)
if not nc.get_host(source):
himutils.sys_error('Could not find source host %s' % source)
def action_list():
instances = nc.get_all_instances(search_opts=search_opts)
printer.output_dict({'header': 'Instance list (id, name, state, task)'})
for i in instances:
output = {
'id': i.id,
'name': i.name,
'state': getattr(i, 'OS-EXT-STS:vm_state'),
'state_task': getattr(i, 'OS-EXT-STS:task_state')
}
printer.output_dict(output, sort=True, one_line=True)
