def action_whales(): start = himutils.get_date(options.start, date.today() - timedelta(days=1)) stop = himutils.get_date(options.end, date.today() + timedelta(days=1)) if start > stop: himutils.sys_error('start %s must be fore stop %s' % (start, stop)) logger.debug('=> start date = %s', start) logger.debug('=> stop date = %s', stop) for region in regions: nc = Nova(options.config, debug=options.debug, log=logger, region=region) cc = Cinder(options.config, debug=options.debug, log=logger, region=region) project_usage = nc.get_usage(start=start, end=stop) logger.debug('=> threshold for whales filter %s', options.threshold) print_header = True for usage in project_usage: project = kc.get_by_id(obj_type='project', obj_id=usage.tenant_id) if not project: logger.debug('=> project with id %s not found',usage.tenant_id) continue if len(usage.server_usages) < options.threshold: continue cinderusage = cc.get_usage(usage.tenant_id) admin = project.admin if hasattr(project, 'admin') else 'unknown!' output = OrderedDict() output['instances'] = len(usage.server_usages) output['volume_gb'] = cinderusage.gigabytes['in_use'] output['name'] = project.name output['admin'] = admin if print_header: output['header'] = 'project usage %s (instances, volume (GB), name, id)' % region print_header = False printer.output_dict(objects=output, sort=False, one_line=True)
def action_resources(): project = kc.get_project_by_name(options.project) start = himutils.get_date(options.start, date.today() - timedelta(days=1)) stop = himutils.get_date(options.end, date.today() + timedelta(days=1)) logger.debug('=> start date = %s', start) logger.debug('=> stop date = %s', stop) output = dict({'vcpu': 0, 'ram':0}) for region in regions: # instances nc = Nova(options.config, debug=options.debug, log=logger, region=region) gc = Gnocchi(options.config, debug=options.debug, log=logger, region=region) deleted = nc.get_project_instances(project_id=project.id, deleted=True) running = nc.get_project_instances(project_id=project.id) for i in deleted + running: resource = gc.get_resource(resource_type='instance', resource_id=i.id) if not resource: continue metrics = dict() metrics['vcpu'] = gc.get_client().metric.get('vcpus', i.id) metrics['ram'] = gc.get_client().metric.get('memory', i.id) for key, value in metrics.iteritems(): measurement = gc.get_client().metric.get_measures(metric=value['id'], aggregation='max', start=start, stop=stop) if measurement: output[key] += measurement[0][2] printer.output_dict({'header': 'resources used by %s in all regions' % project.name}) printer.output_dict(output)
def action_purge(): active = ksclient.get_users(domain=options.domain, enabled=True) users = ksclient.get_users(domain=options.domain, enabled=False) count = 0 disabled = list() for user in users: if not hasattr(user, 'disabled'): himutils.sys_error( "user %s is disabled but missing disabled date" % user.name) continue # Allow 30 days gracetime before we delete disabled_date = himutils.get_date(user.disabled, None, '%Y-%m-%d') gracetime = timedelta(30) if date.today() - disabled_date < gracetime: continue if options.org != 'all': org = ksclient.get_user_org(user.name) if org and org != options.org: continue if options.limit and count >= int(options.limit): break count += 1 disabled.append(user) q = 'This will delete %s disabled users (total active users %s)' \ % (len(disabled), len(active)) if not himutils.confirm_action(q): return for user in disabled: ksclient.user_cleanup(email=user.name) print "%s deleted" % user.name
def action_count(): projects = kc.get_projects(type='demo') count_all = count_60 = 0 for project in projects: for region in regions: nc = utils.get_client(Nova, options, logger, region) instances = nc.get_project_instances(project_id=project.id) if not instances: continue for i in instances: count_all += 1 created_at = utils.get_date(i.created, None, '%Y-%m-%dT%H:%M:%SZ') if (date.today() - created_at) >= timedelta(60): count_60 += 1 with open('/opt/himlarcli/logs/dryrun-logs/dryrun-logs.log' ) as f: if i.id in f.read(): printer.output_dict({ 'instance name': i.name, 'instance id': i.id, 'notify?': 'yes' }) f.close() else: print created_at printer.output_dict({'header': 'count', 'all': count_all, '>60': count_60})
def action_notify(): users = dict() instances = novaclient.get_instances(options.aggregate) # update metadata if not options.dry_run: metadata = {'mail': options.date} novaclient.update_aggregate(options.aggregate, metadata=metadata) # Generate instance list per user for i in instances: email = None user = ksclient.get_by_id('user', i.user_id) if not user: project = ksclient.get_by_id('project', i.tenant_id) if hasattr(project, 'admin'): email = project.admin else: continue if not email: if not user.name: continue if "@" not in user.name: continue email = user.name.lower() if email not in users: users[email] = dict() users[email][i.name] = {'status': i.status} if users: mail = Mail(options.config, debug=options.debug) # Email each users for user, instances in users.iteritems(): user_instances = "" for server, info in instances.iteritems(): user_instances += "%s (current status %s)\n" % (server, info['status']) action_date = himutils.get_date(options.date, date.today(), '%Y-%m-%d') mapping = dict(region=ksclient.region.upper(), date=action_date.strftime("%d %B %Y"), region_lower=ksclient.region.lower(), instances=user_instances) body_content = himutils.load_template(inputfile=options.template, mapping=mapping, log=ksclient.get_logger()) if not body_content: print 'ERROR! Could not find and parse mail body in \ %s' % options.msg sys.exit(1) msg = MIMEText(body_content, 'plain', 'utf-8') msg['Subject'] = ('[UH-IaaS]: Your legacy instances will be terminated on %s (%s)' % (options.date, ksclient.region)) if not options.dry_run: mail.send_mail(user, msg) print "Sending email to user %s" % user else: print "Dry-run: Mail would be sendt to user %s" % user pp = pprint.PrettyPrinter(indent=1) print "\nComplete list of users and instances:" print "=====================================" pp.pprint(users)
def action_flavors(): project = kc.get_project_by_name(options.project) start = himutils.get_date(options.start, date.today() - timedelta(days=1)) stop = himutils.get_date(options.end, date.today() + timedelta(days=1)) if start > stop: himutils.sys_error('start %s must be fore stop %s' % (start, stop)) logger.debug('=> start date = %s', start) logger.debug('=> stop date = %s', stop) flavors = dict() for region in regions: nc = Nova(options.config, debug=options.debug, log=logger, region=region) usage = nc.get_usage(project_id=project.id, start=start, end=stop) if not hasattr(usage, 'server_usages'): continue for server in usage.server_usages: flavors[server['flavor']] = flavors.get(server['flavor'], 0) + 1 flavors['header'] = 'flavor usage for %s in all regions' % project.name printer.output_dict(flavors)
def action_instance(): start = himutils.get_date(options.start, date.today() - timedelta(days=1)) stop = himutils.get_date(options.end, date.today() + timedelta(days=1)) gc = Gnocchi(options.config, debug=options.debug, log=logger) instance = nc.get_instance(options.instance) resources = gc.get_resource(resource_type='instance', resource_id=instance.id) metrics = resources['metrics'] del resources['metrics'] printer.output_dict({'header': 'instance metadata'}) printer.output_dict(resources) printer.output_dict({'header': 'instance metrics'}) output = defaultdict(int) for k, v in metrics.iteritems(): measurement = gc.get_client().metric.get_measures(metric=v, aggregation='max', start=start, stop=stop) if measurement: output[k] += measurement[0][2] printer.output_dict(output)
def action_expired(): max_days = 90 projects = kc.get_projects(type='demo') subject = '[NREC] Your demo instance is due for deletion' logfile = 'logs/demo-logs/expired_instances/demo-notify-expired-instances-{}.log'.format( date.today().isoformat()) mail = utils.get_client(Mail, options, logger) fromaddr = mail.get_config('mail', 'from_addr') cc = '*****@*****.**' inputday = options.day question = 'Send mail to instances that have been running for {} days?'.format( inputday) if not options.force and not utils.confirm_action(question): return template = options.template if not utils.file_exists(template, logger): utils.sys_error('Could not find template file {}'.format(template)) if not options.template: utils.sys_error( 'Specify a template file. E.g. -t notify/demo-notify-expired-instances.txt' ) if not options.day: utils.sys_error( 'Specify the number of days for running demo instances. E.g. -d 30' ) for region in regions: nc = utils.get_client(Nova, options, logger, region) for project in projects: instances = nc.get_project_instances(project_id=project.id) for instance in instances: created = utils.get_date(instance.created, None, '%Y-%m-%dT%H:%M:%SZ') active_days = (date.today() - created).days kc.debug_log('{} running for {} days'.format( instance.id, active_days)) if (int(active_days) == int(inputday)): mapping = dict(project=project.name, enddate=int((max_days) - int(inputday)), activity=int(active_days), region=region.upper(), instance=instance.name) body_content = utils.load_template(inputfile=template, mapping=mapping, log=logger) msg = mail.get_mime_text(subject, body_content, fromaddr, cc) kc.debug_log( 'Sending mail to {} that has been active for {} days'. format(instance.id, active_days)) mail.send_mail(project.admin, msg, fromaddr) utils.append_to_logfile(logfile, date.today(), region, project.admin, instance.name, active_days) print('Mail sendt to {}'.format(project.admin))
def action_instance(): start = himutils.get_date(options.start, date.today() - timedelta(days=1)) stop = himutils.get_date(options.end, date.today() + timedelta(days=1)) gc = Gnocchi(options.config, debug=options.debug, log=logger) instance = nc.get_by_id('server', options.instance) resources = gc.get_resource(resource_type='instance', resource_id=instance.id) metrics = resources['metrics'] del resources['metrics'] printer.output_dict({'header': 'instance metadata'}) printer.output_dict(resources) printer.output_dict({'header': 'instance metrics'}) output = defaultdict(int) for k, v in metrics.iteritems(): measurement = gc.get_client().metric.get_measures(metric=v, aggregation='max', start=start, stop=stop) if measurement: output[k] += measurement[0][2] printer.output_dict(output)
def action_extend(): project = ksclient.get_project_by_name(options.project) if not project: msg = 'Could not find any project named {}'.format(options.project) himutils.sys_error(msg) enddate = himutils.get_date(options.enddate, None, '%d.%m.%Y') ksclient.update_project(project_id=project.id, enddate=str(enddate), disabled='', notified='', enabled=True)
def action_whales(): start = himutils.get_date(options.start, date.today() - timedelta(days=1)) stop = himutils.get_date(options.end, date.today() + timedelta(days=1)) if start > stop: himutils.sys_error('start %s must be fore stop %s' % (start, stop)) logger.debug('=> start date = %s', start) logger.debug('=> stop date = %s', stop) for region in regions: nc = Nova(options.config, debug=options.debug, log=logger, region=region) cc = Cinder(options.config, debug=options.debug, log=logger, region=region) project_usage = nc.get_usage(start=start, end=stop) logger.debug('=> threshold for whales filter %s', options.threshold) print_header = True for usage in project_usage: project = kc.get_by_id(obj_type='project', obj_id=usage.tenant_id) if not project: logger.debug('=> project with id %s not found', usage.tenant_id) continue if len(usage.server_usages) < options.threshold: continue cinderusage = cc.get_quota(usage.tenant_id, True) admin = project.admin if hasattr(project, 'admin') else 'unknown!' output = OrderedDict() output['instances'] = len(usage.server_usages) output['volume_gb'] = cinderusage['gigabytes']['in_use'] output['name'] = project.name output['admin'] = admin if print_header: output[ 'header'] = 'project usage %s (instances, volume (GB), name, id)' % region print_header = False printer.output_dict(objects=output, sort=False, one_line=True)
def action_resources(): project = kc.get_project_by_name(options.project) start = himutils.get_date(options.start, date.today() - timedelta(days=1)) stop = himutils.get_date(options.end, date.today() + timedelta(days=1)) logger.debug('=> start date = %s', start) logger.debug('=> stop date = %s', stop) output = dict({'vcpu': 0, 'ram': 0}) for region in regions: # instances nc = Nova(options.config, debug=options.debug, log=logger, region=region) gc = Gnocchi(options.config, debug=options.debug, log=logger, region=region) deleted = nc.get_project_instances(project_id=project.id, deleted=True) running = nc.get_project_instances(project_id=project.id) for i in deleted + running: resource = gc.get_resource(resource_type='instance', resource_id=i.id) if not resource: continue metrics = dict() metrics['vcpu'] = gc.get_client().metric.get('vcpus', i.id) metrics['ram'] = gc.get_client().metric.get('memory', i.id) for key, value in metrics.iteritems(): measurement = gc.get_client().metric.get_measures( metric=value['id'], aggregation='max', start=start, stop=stop) if measurement: output[key] += measurement[0][2] printer.output_dict( {'header': 'resources used by %s in all regions' % project.name}) printer.output_dict(output)
def action_disable(): projects = kc.get_projects() subject = '[NREC] Your project is due for deletion' logfile = 'logs/expired-disabled-{}.log'.format(date.today().isoformat()) if options.template: template = options.template else: template = 'notify/notify_expired_last.txt' mail = utils.get_client(Mail, options, logger) fromaddr = mail.get_config('mail', 'from_addr') for project in projects: project = expired_project(project) if not project: continue # Allow 30 days gracetime before we disable disabled_date = utils.get_date(project.notified, None, '%Y-%m-%d') gracetime = timedelta(30) if date.today() - disabled_date < gracetime: continue # stop instances for region in regions: nc = utils.get_client(Nova, options, logger, region) instances = nc.get_project_instances(project_id=project.id) for i in instances: if i.status == 'ACTIVE': i.stop() mapping = dict(project=project.name, enddate=project.enddate) body_content = utils.load_template(inputfile=template, mapping=mapping, log=logger) msg = mail.get_mime_text(subject, body_content, fromaddr) mail.send_mail(project.admin, msg, fromaddr) print "mail sendt to {}".format(project.admin) if not options.dry_run: utils.append_to_file(logfile, project.admin) # Add metadata to project for the time of project disable kc.update_project(project_id=project.id, enabled=False, disabled=str(date.today()))
def action_instances(): projects = kc.get_projects(type='demo') printer.output_dict( {'header': 'Demo instances (id, lifetime in days, name, flavor)'}) count = 0 for project in projects: for region in regions: nc = utils.get_client(Nova, options, logger, region) instances = nc.get_project_instances(project_id=project.id) for i in instances: created = utils.get_date(i.created, None, '%Y-%m-%dT%H:%M:%SZ') active_days = (date.today() - created).days if int(active_days) < int(options.day): continue output = { '0': i.id, '2': i.name, '1': (date.today() - created).days, '3': i.flavor['original_name'] } count += 1 printer.output_dict(output, one_line=True) printer.output_dict({'header': 'Count', 'count': count})
def action_delete(): days = 90 question = 'Delete demo instances older than {} days?'.format(days) if not options.force and not utils.confirm_action(question): return projects = kc.get_projects(type='demo') logfile = 'logs/demo-logs/deleted_instances/deleted-expired-demo-instances-{}.log'.format( date.today().isoformat()) for region in regions: for project in projects: nc = utils.get_client(Nova, options, logger, region) instances = nc.get_project_instances(project_id=project.id) for instance in instances: created = utils.get_date(instance.created, None, '%Y-%m-%dT%H:%M:%SZ') active_days = (date.today() - created).days kc.debug_log('Found instance {} for user {}'.format( instance.id, project.admin)) if int(active_days) >= days: nc.delete_instance(instance) if not options.dry_run: utils.append_to_logfile(logfile, "deleted:", project.name, instance.name, "active for:", active_days)
def action_create(): if not ksclient.is_valid_user( options.admin, options.domain) and options.type == 'personal': himutils.sys_error('not valid user', 1) quota = himutils.load_config('config/quotas/%s.yaml' % options.quota) if options.quota and not quota: himutils.sys_error('Could not find quota in config/quotas/%s.yaml' % options.quota) test = 1 if options.type == 'test' else 0 project_msg = project_msg_file enddate = himutils.get_date(options.enddate, None, '%d.%m.%Y') if options.type == 'hpc': project_msg = project_hpc_msg_file if not enddate: himutils.sys_error('HPC projects must have an enddate', 1) createdate = datetime.today() # Parse the "contact" option, setting to None if not used # Exit with error if contact is not a valid email address contact = None if options.contact is not None: contact = options.contact.lower() if not ksclient._Keystone__validate_email(contact): errmsg = "%s is not a valid email address." % contact himutils.sys_error(errmsg, 1) if not options.force: print 'Project name: %s\nDescription: %s\nAdmin: %s\nContact: %s\nOrganization: %s\nType: %s\nEnd date: %s\nQuota: %s\nRT: %s' \ % (options.project, ksclient.convert_ascii(options.desc), options.admin.lower(), contact, options.org, options.type, str(enddate), options.quota, options.rt) if not himutils.confirm_action( 'Are you sure you want to create this project?'): himutils.sys_error('Aborted', 1) project = ksclient.create_project(project_name=options.project, admin=options.admin.lower(), contact=contact, org=options.org, test=test, type=options.type, description=options.desc, enddate=str(enddate), createdate=createdate.isoformat(), quota=options.quota, rt=options.rt) if not ksclient.is_valid_user(options.admin, options.domain): himutils.sys_error( 'WARNING: "%s" is not a valid user.' % options.admin, 0) if not project: himutils.sys_error('Failed creating %s' % options.project, 1) else: output = Keystone.get_dict(project) output['header'] = "Show information for %s" % options.project printer.output_dict(output) # Do stuff for regions for region in regions: # Get objects novaclient = himutils.get_client(Nova, options, logger, region) cinderclient = himutils.get_client(Cinder, options, logger, region) neutronclient = himutils.get_client(Neutron, options, logger, region) glanceclient = himutils.get_client(Glance, options, logger, region) # Find the project ID project_id = Keystone.get_attr(project, 'id') # Update quotas for Cinder, Nova, Neutron if quota and 'cinder' in quota and project: cinderclient.update_quota(project_id=project_id, updates=quota['cinder']) if quota and 'nova' in quota and project: novaclient.update_quota(project_id=project_id, updates=quota['nova']) if quota and 'neutron' in quota and project: neutronclient.update_quota(project_id=project_id, updates=quota['neutron']) # Grant UiO Managed images if shared UiO project if options.org == 'uio' and options.type not in ['personal', 'demo']: tags = ['uio'] filters = {'status': 'active', 'tag': tags, 'visibility': 'shared'} images = glanceclient.get_images(filters=filters) for image in images: glanceclient.set_image_access(image_id=image.id, project_id=project.id, action='grant') printer.output_msg( 'GRANT access to image {} for project {}'.format( image.name, project.name)) if options.mail: mail = Mail(options.config, debug=options.debug) mail.set_dry_run(options.dry_run) if options.rt is None: himutils.sys_error('--rt parameter is missing.') else: mapping = dict(project_name=options.project, admin=options.admin.lower(), quota=options.quota, end_date=str(enddate)) subject = 'NREC: Project %s has been created' % options.project body_content = himutils.load_template(inputfile=project_msg, mapping=mapping) if not body_content: himutils.sys_error('ERROR! Could not find and parse mail body in \ %s' % options.msg) mime = mail.rt_mail(options.rt, subject, body_content) mail.send_mail('*****@*****.**', mime)