def action_whales():
start = himutils.get_date(options.start, date.today() - timedelta(days=1))
stop = himutils.get_date(options.end, date.today() + timedelta(days=1))
if start > stop:
himutils.sys_error('start %s must be fore stop %s' % (start, stop))
logger.debug('=> start date = %s', start)
logger.debug('=> stop date = %s', stop)
for region in regions:
nc = Nova(options.config, debug=options.debug, log=logger, region=region)
cc = Cinder(options.config, debug=options.debug, log=logger, region=region)
project_usage = nc.get_usage(start=start, end=stop)
logger.debug('=> threshold for whales filter %s', options.threshold)
print_header = True
for usage in project_usage:
project = kc.get_by_id(obj_type='project', obj_id=usage.tenant_id)
if not project:
logger.debug('=> project with id %s not found',usage.tenant_id)
continue
if len(usage.server_usages) < options.threshold:
continue
cinderusage = cc.get_usage(usage.tenant_id)
admin = project.admin if hasattr(project, 'admin') else 'unknown!'
output = OrderedDict()
output['instances'] = len(usage.server_usages)
output['volume_gb'] = cinderusage.gigabytes['in_use']
output['name'] = project.name
output['admin'] = admin
if print_header:
output['header'] = 'project usage %s (instances, volume (GB), name, id)' % region
print_header = False
printer.output_dict(objects=output, sort=False, one_line=True)
def action_resources():
project = kc.get_project_by_name(options.project)
start = himutils.get_date(options.start, date.today() - timedelta(days=1))
stop = himutils.get_date(options.end, date.today() + timedelta(days=1))
logger.debug('=> start date = %s', start)
logger.debug('=> stop date = %s', stop)
output = dict({'vcpu': 0, 'ram':0})
for region in regions:
# instances
nc = Nova(options.config, debug=options.debug, log=logger, region=region)
gc = Gnocchi(options.config, debug=options.debug, log=logger, region=region)
deleted = nc.get_project_instances(project_id=project.id, deleted=True)
running = nc.get_project_instances(project_id=project.id)
for i in deleted + running:
resource = gc.get_resource(resource_type='instance', resource_id=i.id)
if not resource:
continue
metrics = dict()
metrics['vcpu'] = gc.get_client().metric.get('vcpus', i.id)
metrics['ram'] = gc.get_client().metric.get('memory', i.id)
for key, value in metrics.iteritems():
measurement = gc.get_client().metric.get_measures(metric=value['id'],
aggregation='max',
start=start,
stop=stop)
if measurement:
output[key] += measurement[0][2]
printer.output_dict({'header': 'resources used by %s in all regions' % project.name})
printer.output_dict(output)
def action_purge():
active = ksclient.get_users(domain=options.domain, enabled=True)
users = ksclient.get_users(domain=options.domain, enabled=False)
count = 0
disabled = list()
for user in users:
if not hasattr(user, 'disabled'):
himutils.sys_error(
"user %s is disabled but missing disabled date" % user.name)
continue
# Allow 30 days gracetime before we delete
disabled_date = himutils.get_date(user.disabled, None, '%Y-%m-%d')
gracetime = timedelta(30)
if date.today() - disabled_date < gracetime:
continue
if options.org != 'all':
org = ksclient.get_user_org(user.name)
if org and org != options.org:
continue
if options.limit and count >= int(options.limit):
break
count += 1
disabled.append(user)
q = 'This will delete %s disabled users (total active users %s)' \
% (len(disabled), len(active))
if not himutils.confirm_action(q):
return
for user in disabled:
ksclient.user_cleanup(email=user.name)
print "%s deleted" % user.name
def action_count():
projects = kc.get_projects(type='demo')
count_all = count_60 = 0
for project in projects:
for region in regions:
nc = utils.get_client(Nova, options, logger, region)
instances = nc.get_project_instances(project_id=project.id)
if not instances:
continue
for i in instances:
count_all += 1
created_at = utils.get_date(i.created, None,
'%Y-%m-%dT%H:%M:%SZ')
if (date.today() - created_at) >= timedelta(60):
count_60 += 1
with open('/opt/himlarcli/logs/dryrun-logs/dryrun-logs.log'
) as f:
if i.id in f.read():
printer.output_dict({
'instance name': i.name,
'instance id': i.id,
'notify?': 'yes'
})
f.close()
else:
print created_at
printer.output_dict({'header': 'count', 'all': count_all, '>60': count_60})
def action_notify():
users = dict()
instances = novaclient.get_instances(options.aggregate)
# update metadata
if not options.dry_run:
metadata = {'mail': options.date}
novaclient.update_aggregate(options.aggregate, metadata=metadata)
# Generate instance list per user
for i in instances:
email = None
user = ksclient.get_by_id('user', i.user_id)
if not user:
project = ksclient.get_by_id('project', i.tenant_id)
if hasattr(project, 'admin'):
email = project.admin
else:
continue
if not email:
if not user.name:
continue
if "@" not in user.name:
continue
email = user.name.lower()
if email not in users:
users[email] = dict()
users[email][i.name] = {'status': i.status}
if users:
mail = Mail(options.config, debug=options.debug)
# Email each users
for user, instances in users.iteritems():
user_instances = ""
for server, info in instances.iteritems():
user_instances += "%s (current status %s)\n" % (server, info['status'])
action_date = himutils.get_date(options.date, date.today(), '%Y-%m-%d')
mapping = dict(region=ksclient.region.upper(),
date=action_date.strftime("%d %B %Y"),
region_lower=ksclient.region.lower(),
instances=user_instances)
body_content = himutils.load_template(inputfile=options.template,
mapping=mapping,
log=ksclient.get_logger())
if not body_content:
print 'ERROR! Could not find and parse mail body in \
%s' % options.msg
sys.exit(1)
msg = MIMEText(body_content, 'plain', 'utf-8')
msg['Subject'] = ('[UH-IaaS]: Your legacy instances will be terminated on %s (%s)'
% (options.date, ksclient.region))
if not options.dry_run:
mail.send_mail(user, msg)
print "Sending email to user %s" % user
else:
print "Dry-run: Mail would be sendt to user %s" % user
pp = pprint.PrettyPrinter(indent=1)
print "\nComplete list of users and instances:"
print "====================================="
pp.pprint(users)
def action_flavors():
project = kc.get_project_by_name(options.project)
start = himutils.get_date(options.start, date.today() - timedelta(days=1))
stop = himutils.get_date(options.end, date.today() + timedelta(days=1))
if start > stop:
himutils.sys_error('start %s must be fore stop %s' % (start, stop))
logger.debug('=> start date = %s', start)
logger.debug('=> stop date = %s', stop)
flavors = dict()
for region in regions:
nc = Nova(options.config, debug=options.debug, log=logger, region=region)
usage = nc.get_usage(project_id=project.id, start=start, end=stop)
if not hasattr(usage, 'server_usages'):
continue
for server in usage.server_usages:
flavors[server['flavor']] = flavors.get(server['flavor'], 0) + 1
flavors['header'] = 'flavor usage for %s in all regions' % project.name
printer.output_dict(flavors)
def action_instance():
start = himutils.get_date(options.start, date.today() - timedelta(days=1))
stop = himutils.get_date(options.end, date.today() + timedelta(days=1))
gc = Gnocchi(options.config, debug=options.debug, log=logger)
instance = nc.get_instance(options.instance)
resources = gc.get_resource(resource_type='instance', resource_id=instance.id)
metrics = resources['metrics']
del resources['metrics']
printer.output_dict({'header': 'instance metadata'})
printer.output_dict(resources)
printer.output_dict({'header': 'instance metrics'})
output = defaultdict(int)
for k, v in metrics.iteritems():
measurement = gc.get_client().metric.get_measures(metric=v,
aggregation='max',
start=start,
stop=stop)
if measurement:
output[k] += measurement[0][2]
printer.output_dict(output)
def action_expired():
max_days = 90
projects = kc.get_projects(type='demo')
subject = '[NREC] Your demo instance is due for deletion'
logfile = 'logs/demo-logs/expired_instances/demo-notify-expired-instances-{}.log'.format(
date.today().isoformat())
mail = utils.get_client(Mail, options, logger)
fromaddr = mail.get_config('mail', 'from_addr')
cc = '*****@*****.**'
inputday = options.day
question = 'Send mail to instances that have been running for {} days?'.format(
inputday)
if not options.force and not utils.confirm_action(question):
return
template = options.template
if not utils.file_exists(template, logger):
utils.sys_error('Could not find template file {}'.format(template))
if not options.template:
utils.sys_error(
'Specify a template file. E.g. -t notify/demo-notify-expired-instances.txt'
)
if not options.day:
utils.sys_error(
'Specify the number of days for running demo instances. E.g. -d 30'
)
for region in regions:
nc = utils.get_client(Nova, options, logger, region)
for project in projects:
instances = nc.get_project_instances(project_id=project.id)
for instance in instances:
created = utils.get_date(instance.created, None,
'%Y-%m-%dT%H:%M:%SZ')
active_days = (date.today() - created).days
kc.debug_log('{} running for {} days'.format(
instance.id, active_days))
if (int(active_days) == int(inputday)):
mapping = dict(project=project.name,
enddate=int((max_days) - int(inputday)),
activity=int(active_days),
region=region.upper(),
instance=instance.name)
body_content = utils.load_template(inputfile=template,
mapping=mapping,
log=logger)
msg = mail.get_mime_text(subject, body_content, fromaddr,
cc)
kc.debug_log(
'Sending mail to {} that has been active for {} days'.
format(instance.id, active_days))
mail.send_mail(project.admin, msg, fromaddr)
utils.append_to_logfile(logfile, date.today(), region,
project.admin, instance.name,
active_days)
print('Mail sendt to {}'.format(project.admin))
def action_instance():
start = himutils.get_date(options.start, date.today() - timedelta(days=1))
stop = himutils.get_date(options.end, date.today() + timedelta(days=1))
gc = Gnocchi(options.config, debug=options.debug, log=logger)
instance = nc.get_by_id('server', options.instance)
resources = gc.get_resource(resource_type='instance',
resource_id=instance.id)
metrics = resources['metrics']
del resources['metrics']
printer.output_dict({'header': 'instance metadata'})
printer.output_dict(resources)
printer.output_dict({'header': 'instance metrics'})
output = defaultdict(int)
for k, v in metrics.iteritems():
measurement = gc.get_client().metric.get_measures(metric=v,
aggregation='max',
start=start,
stop=stop)
if measurement:
output[k] += measurement[0][2]
printer.output_dict(output)
def action_extend():
project = ksclient.get_project_by_name(options.project)
if not project:
msg = 'Could not find any project named {}'.format(options.project)
himutils.sys_error(msg)
enddate = himutils.get_date(options.enddate, None, '%d.%m.%Y')
ksclient.update_project(project_id=project.id,
enddate=str(enddate),
disabled='',
notified='',
enabled=True)
def action_whales():
start = himutils.get_date(options.start, date.today() - timedelta(days=1))
stop = himutils.get_date(options.end, date.today() + timedelta(days=1))
if start > stop:
himutils.sys_error('start %s must be fore stop %s' % (start, stop))
logger.debug('=> start date = %s', start)
logger.debug('=> stop date = %s', stop)
for region in regions:
nc = Nova(options.config,
debug=options.debug,
log=logger,
region=region)
cc = Cinder(options.config,
debug=options.debug,
log=logger,
region=region)
project_usage = nc.get_usage(start=start, end=stop)
logger.debug('=> threshold for whales filter %s', options.threshold)
print_header = True
for usage in project_usage:
project = kc.get_by_id(obj_type='project', obj_id=usage.tenant_id)
if not project:
logger.debug('=> project with id %s not found',
usage.tenant_id)
continue
if len(usage.server_usages) < options.threshold:
continue
cinderusage = cc.get_quota(usage.tenant_id, True)
admin = project.admin if hasattr(project, 'admin') else 'unknown!'
output = OrderedDict()
output['instances'] = len(usage.server_usages)
output['volume_gb'] = cinderusage['gigabytes']['in_use']
output['name'] = project.name
output['admin'] = admin
if print_header:
output[
'header'] = 'project usage %s (instances, volume (GB), name, id)' % region
print_header = False
printer.output_dict(objects=output, sort=False, one_line=True)
def action_flavors():
project = kc.get_project_by_name(options.project)
start = himutils.get_date(options.start, date.today() - timedelta(days=1))
stop = himutils.get_date(options.end, date.today() + timedelta(days=1))
if start > stop:
himutils.sys_error('start %s must be fore stop %s' % (start, stop))
logger.debug('=> start date = %s', start)
logger.debug('=> stop date = %s', stop)
flavors = dict()
for region in regions:
nc = Nova(options.config,
debug=options.debug,
log=logger,
region=region)
usage = nc.get_usage(project_id=project.id, start=start, end=stop)
if not hasattr(usage, 'server_usages'):
continue
for server in usage.server_usages:
flavors[server['flavor']] = flavors.get(server['flavor'], 0) + 1
flavors['header'] = 'flavor usage for %s in all regions' % project.name
printer.output_dict(flavors)
def action_resources():
project = kc.get_project_by_name(options.project)
start = himutils.get_date(options.start, date.today() - timedelta(days=1))
stop = himutils.get_date(options.end, date.today() + timedelta(days=1))
logger.debug('=> start date = %s', start)
logger.debug('=> stop date = %s', stop)
output = dict({'vcpu': 0, 'ram': 0})
for region in regions:
# instances
nc = Nova(options.config,
debug=options.debug,
log=logger,
region=region)
gc = Gnocchi(options.config,
debug=options.debug,
log=logger,
region=region)
deleted = nc.get_project_instances(project_id=project.id, deleted=True)
running = nc.get_project_instances(project_id=project.id)
for i in deleted + running:
resource = gc.get_resource(resource_type='instance',
resource_id=i.id)
if not resource:
continue
metrics = dict()
metrics['vcpu'] = gc.get_client().metric.get('vcpus', i.id)
metrics['ram'] = gc.get_client().metric.get('memory', i.id)
for key, value in metrics.iteritems():
measurement = gc.get_client().metric.get_measures(
metric=value['id'],
aggregation='max',
start=start,
stop=stop)
if measurement:
output[key] += measurement[0][2]
printer.output_dict(
{'header': 'resources used by %s in all regions' % project.name})
printer.output_dict(output)
def action_disable():
projects = kc.get_projects()
subject = '[NREC] Your project is due for deletion'
logfile = 'logs/expired-disabled-{}.log'.format(date.today().isoformat())
if options.template:
template = options.template
else:
template = 'notify/notify_expired_last.txt'
mail = utils.get_client(Mail, options, logger)
fromaddr = mail.get_config('mail', 'from_addr')
for project in projects:
project = expired_project(project)
if not project:
continue
# Allow 30 days gracetime before we disable
disabled_date = utils.get_date(project.notified, None, '%Y-%m-%d')
gracetime = timedelta(30)
if date.today() - disabled_date < gracetime:
continue
# stop instances
for region in regions:
nc = utils.get_client(Nova, options, logger, region)
instances = nc.get_project_instances(project_id=project.id)
for i in instances:
if i.status == 'ACTIVE':
i.stop()
mapping = dict(project=project.name, enddate=project.enddate)
body_content = utils.load_template(inputfile=template,
mapping=mapping,
log=logger)
msg = mail.get_mime_text(subject, body_content, fromaddr)
mail.send_mail(project.admin, msg, fromaddr)
print "mail sendt to {}".format(project.admin)
if not options.dry_run:
utils.append_to_file(logfile, project.admin)
# Add metadata to project for the time of project disable
kc.update_project(project_id=project.id,
enabled=False,
disabled=str(date.today()))
def action_instances():
projects = kc.get_projects(type='demo')
printer.output_dict(
{'header': 'Demo instances (id, lifetime in days, name, flavor)'})
count = 0
for project in projects:
for region in regions:
nc = utils.get_client(Nova, options, logger, region)
instances = nc.get_project_instances(project_id=project.id)
for i in instances:
created = utils.get_date(i.created, None, '%Y-%m-%dT%H:%M:%SZ')
active_days = (date.today() - created).days
if int(active_days) < int(options.day):
continue
output = {
'0': i.id,
'2': i.name,
'1': (date.today() - created).days,
'3': i.flavor['original_name']
}
count += 1
printer.output_dict(output, one_line=True)
printer.output_dict({'header': 'Count', 'count': count})
def action_delete():
days = 90
question = 'Delete demo instances older than {} days?'.format(days)
if not options.force and not utils.confirm_action(question):
return
projects = kc.get_projects(type='demo')
logfile = 'logs/demo-logs/deleted_instances/deleted-expired-demo-instances-{}.log'.format(
date.today().isoformat())
for region in regions:
for project in projects:
nc = utils.get_client(Nova, options, logger, region)
instances = nc.get_project_instances(project_id=project.id)
for instance in instances:
created = utils.get_date(instance.created, None,
'%Y-%m-%dT%H:%M:%SZ')
active_days = (date.today() - created).days
kc.debug_log('Found instance {} for user {}'.format(
instance.id, project.admin))
if int(active_days) >= days:
nc.delete_instance(instance)
if not options.dry_run:
utils.append_to_logfile(logfile, "deleted:",
project.name, instance.name,
"active for:", active_days)
def action_create():
if not ksclient.is_valid_user(
options.admin, options.domain) and options.type == 'personal':
himutils.sys_error('not valid user', 1)
quota = himutils.load_config('config/quotas/%s.yaml' % options.quota)
if options.quota and not quota:
himutils.sys_error('Could not find quota in config/quotas/%s.yaml' %
options.quota)
test = 1 if options.type == 'test' else 0
project_msg = project_msg_file
enddate = himutils.get_date(options.enddate, None, '%d.%m.%Y')
if options.type == 'hpc':
project_msg = project_hpc_msg_file
if not enddate:
himutils.sys_error('HPC projects must have an enddate', 1)
createdate = datetime.today()
# Parse the "contact" option, setting to None if not used
# Exit with error if contact is not a valid email address
contact = None
if options.contact is not None:
contact = options.contact.lower()
if not ksclient._Keystone__validate_email(contact):
errmsg = "%s is not a valid email address." % contact
himutils.sys_error(errmsg, 1)
if not options.force:
print 'Project name: %s\nDescription: %s\nAdmin: %s\nContact: %s\nOrganization: %s\nType: %s\nEnd date: %s\nQuota: %s\nRT: %s' \
% (options.project,
ksclient.convert_ascii(options.desc),
options.admin.lower(),
contact,
options.org,
options.type,
str(enddate),
options.quota,
options.rt)
if not himutils.confirm_action(
'Are you sure you want to create this project?'):
himutils.sys_error('Aborted', 1)
project = ksclient.create_project(project_name=options.project,
admin=options.admin.lower(),
contact=contact,
org=options.org,
test=test,
type=options.type,
description=options.desc,
enddate=str(enddate),
createdate=createdate.isoformat(),
quota=options.quota,
rt=options.rt)
if not ksclient.is_valid_user(options.admin, options.domain):
himutils.sys_error(
'WARNING: "%s" is not a valid user.' % options.admin, 0)
if not project:
himutils.sys_error('Failed creating %s' % options.project, 1)
else:
output = Keystone.get_dict(project)
output['header'] = "Show information for %s" % options.project
printer.output_dict(output)
# Do stuff for regions
for region in regions:
# Get objects
novaclient = himutils.get_client(Nova, options, logger, region)
cinderclient = himutils.get_client(Cinder, options, logger, region)
neutronclient = himutils.get_client(Neutron, options, logger, region)
glanceclient = himutils.get_client(Glance, options, logger, region)
# Find the project ID
project_id = Keystone.get_attr(project, 'id')
# Update quotas for Cinder, Nova, Neutron
if quota and 'cinder' in quota and project:
cinderclient.update_quota(project_id=project_id,
updates=quota['cinder'])
if quota and 'nova' in quota and project:
novaclient.update_quota(project_id=project_id,
updates=quota['nova'])
if quota and 'neutron' in quota and project:
neutronclient.update_quota(project_id=project_id,
updates=quota['neutron'])
# Grant UiO Managed images if shared UiO project
if options.org == 'uio' and options.type not in ['personal', 'demo']:
tags = ['uio']
filters = {'status': 'active', 'tag': tags, 'visibility': 'shared'}
images = glanceclient.get_images(filters=filters)
for image in images:
glanceclient.set_image_access(image_id=image.id,
project_id=project.id,
action='grant')
printer.output_msg(
'GRANT access to image {} for project {}'.format(
image.name, project.name))
if options.mail:
mail = Mail(options.config, debug=options.debug)
mail.set_dry_run(options.dry_run)
if options.rt is None:
himutils.sys_error('--rt parameter is missing.')
else:
mapping = dict(project_name=options.project,
admin=options.admin.lower(),
quota=options.quota,
end_date=str(enddate))
subject = 'NREC: Project %s has been created' % options.project
body_content = himutils.load_template(inputfile=project_msg,
mapping=mapping)
if not body_content:
himutils.sys_error('ERROR! Could not find and parse mail body in \
%s' % options.msg)
mime = mail.rt_mail(options.rt, subject, body_content)
mail.send_mail('*****@*****.**', mime)
