def splunk(**options): splunk = Splunk() splunk.start() percent = splunk.progress while percent < 100: if options['verbose']: print(percent) time.sleep(30) percent = splunk.progress hosts = {} total_mb = 0 for item in splunk.results: host = item['host'] mb = float(item['MBytes']) total_mb += mb if host in hosts: hosts[host] += mb else: hosts[host] = mb if options['verbose']: print(len(hosts), 'hosts used', total_mb, ' MiBs total.') for host, usage in hosts.iteritems(): ip = IPAddress.objects.filter(hostname__startswith=host).order_by( '-last_seen') if not ip.count(): if options['verbose']: print('Warning: host', host, 'not found in device database.') continue dev = ip[0].device if not dev: if options['verbose']: print('Warning: host', host, 'not tied to a device in the ' 'database.') continue name = 'Splunk Volume 100 GiB' symbol = 'splunkvolume' model, created = ComponentModel.create( ComponentType.unknown, family=symbol, name=name, priority=0, ) res, created = SplunkUsage.concurrent_get_or_create( device=dev, day=date.today(), defaults={'model': model}, ) res.size = usage res.save() return True, 'done.', options
def splunk(**options): splunk = Splunk() splunk.start() percent = splunk.progress while percent < 100: if options['verbose']: print(percent) time.sleep(30) percent = splunk.progress hosts = {} total_mb = 0 for item in splunk.results: host = item['host'] mb = float(item['MBytes']) total_mb += mb if host in hosts: hosts[host] += mb else: hosts[host] = mb if options['verbose']: print(len(hosts), 'hosts used', total_mb, ' MiBs total.') for host, usage in hosts.iteritems(): ip = IPAddress.objects.filter( hostname__startswith=host).order_by('-last_seen') if not ip.count(): if options['verbose']: print('Warning: host', host, 'not found in device database.') continue dev = ip[0].device if not dev: if options['verbose']: print('Warning: host', host, 'not tied to a device in the ' 'database.') continue name = 'Splunk Volume 100 GiB' symbol = 'splunkvolume' model, created = ComponentModel.create( ComponentType.unknown, family=symbol, name=name, priority=0, ) res, created = SplunkUsage.concurrent_get_or_create( device=dev, day=date.today(), defaults={'model': model}, ) res.size = usage res.save() return True, 'done.', options
def splunk(**options): splunk = Splunk() splunk.start() percent = splunk.progress while percent < 100: if options["verbose"]: print(percent) time.sleep(30) percent = splunk.progress hosts = {} total_mb = 0 for item in splunk.results: host = item["host"] mb = float(item["MBytes"]) total_mb += mb if host in hosts: hosts[host] += mb else: hosts[host] = mb if options["verbose"]: print(len(hosts), "hosts used", total_mb, " MiBs total.") for host, usage in hosts.iteritems(): ip = IPAddress.objects.filter(hostname__startswith=host).order_by("-last_seen") if not ip.count(): if options["verbose"]: print("Warning: host", host, "not found in device database.") continue dev = ip[0].device if not dev: if options["verbose"]: print("Warning: host", host, "not tied to a device in the " "database.") continue name = "Splunk Volume 100 GiB" symbol = "splunkvolume" model, created = ComponentModel.concurrent_get_or_create( type=ComponentType.unknown.id, speed=0, cores=0, size=0, family=symbol, extra_hash="" ) if created: model.name = name model.save() res, created = SplunkUsage.concurrent_get_or_create(model=model, device=dev, day=date.today()) res.size = usage res.save() return True, "done.", options
def _report_ventures_data_provider(start, end, ventures_ids, extra_types): ventures = Venture.objects.filter(id__in=ventures_ids).order_by('path') total_cloud_cost = get_total_cost( HistoryCost.objects.filter( device__model__type=DeviceType.cloud_server.id, ), start, end, ) result = [] for venture in ventures: query = HistoryCost.objects.filter( db.Q(venture=venture) | db.Q(venture__parent=venture) | db.Q(venture__parent__parent=venture) | db.Q(venture__parent__parent__parent=venture) | db.Q(venture__parent__parent__parent__parent=venture)).exclude( device__deleted=True) data = _report_ventures_get_totals(start, end, query, extra_types) ( splunk_cost, splunk_count, splunk_count_now, splunk_size, ) = SplunkUsage.get_cost(venture, start, end) data.update({ 'id': venture.id, 'name': venture.name, 'symbol': venture.symbol, 'path': venture.path, 'department': unicode(venture.department or ''), 'margin': venture.get_margin(), 'top_level': venture.parent is None, 'venture_icon': get_venture_icon(venture), 'cloud_use': ((data['cloud_cost'] or 0) / total_cloud_cost) if total_cloud_cost else 0, 'splunk_cost': splunk_cost, }) result.append(data) if venture.parent is not None: continue if not venture.child_set.exists(): continue query = HistoryCost.objects.filter(venture=venture) data = _report_ventures_get_totals(start, end, query, extra_types) ( splunk_cost, splunk_count, splunk_count_now, splunk_size, ) = SplunkUsage.get_cost(venture, start, end, shallow=True) data.update({ 'id': venture.id, 'name': '-', 'symbol': venture.symbol, 'path': venture.path, 'department': unicode(venture.department or ''), 'margin': venture.get_margin(), 'top_level': False, 'venture_icon': get_venture_icon(venture), 'cloud_use': ((data['cloud_cost'] or 0) / total_cloud_cost) if total_cloud_cost else 0, 'splunk_cost': splunk_cost, }) result.append(data) return result
def _get_summaries(query, start, end, overlap=True, venture=None): if overlap: yield _total_dict('Servers', query.filter( device__model__type__in=(DeviceType.rack_server.id, DeviceType.blade_server.id, DeviceType.virtual_server.id)), start, end, _get_search_url(venture, type=(201, 202, 203))) for dc in DataCenter.objects.all(): yield _total_dict(' • Servers in %s' % dc.name, query.filter( device__model__type__in=(DeviceType.rack_server.id, DeviceType.blade_server.id, DeviceType.virtual_server.id) ).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(201, 202, 203)) ) if overlap: yield _total_dict( ' ∙ Rack servers in %s' % dc.name, query.filter( device__model__type=DeviceType.rack_server.id, ).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(201,)) ) for mg in DeviceModelGroup.objects.filter( type=DeviceType.rack_server.id).order_by('name'): yield _total_dict( ' %s in %s' % (mg, dc.name), query.filter( device__model__group=mg, ).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(201,), model_group=mg.id) ) yield _total_dict( ' ∙ Blade servers in %s' % dc.name, query.filter( device__model__type=DeviceType.blade_server.id, ).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(202,)) ) for mg in DeviceModelGroup.objects.filter( type=DeviceType.blade_server.id).order_by('name'): yield _total_dict( ' %s in %s' % (mg, dc.name), query.filter( device__model__group=mg, ).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(202,), model_group=mg.id) ) yield _total_dict( ' ∙ Virtual servers in %s' % dc.name, query.filter( device__model__type=DeviceType.virtual_server.id, ).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(203,)) ) if overlap: yield _total_dict('Loadbalancers', query.filter( device__model__type__in=(DeviceType.load_balancer.id,) ), start, end, _get_search_url(venture, type=(103,))) for dc in DataCenter.objects.all(): yield _total_dict(' • Loadbalancers in %s' % dc.name, query.filter( device__model__type__in=(DeviceType.load_balancer.id,) ).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(103,)) ) if overlap: yield _total_dict('Storage', query.filter( device__model__type__in=( DeviceType.storage.id, DeviceType.fibre_channel_switch.id, )), start, end, _get_search_url(venture, type=(301,)) ) for dc in DataCenter.objects.all(): yield _total_dict(' • Storage in %s' % dc.name, query.filter( device__model__type__in=( DeviceType.storage.id, DeviceType.fibre_channel_switch.id, ) ).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(301,)) ) if overlap: yield _total_dict('Network', query.filter( device__model__type__in=( DeviceType.switch.id, DeviceType.router.id, DeviceType.firewall.id, DeviceType.smtp_gateway.id, DeviceType.appliance.id, ) ), start, end, _get_search_url(venture, type=( DeviceType.switch.id, DeviceType.router.id, DeviceType.firewall.id, DeviceType.smtp_gateway.id, DeviceType.appliance.id, )) ) for dc in DataCenter.objects.all(): yield _total_dict(' • Network in %s' % dc.name, query.filter( device__model__type__in=( DeviceType.switch.id, DeviceType.router.id, DeviceType.firewall.id, DeviceType.smtp_gateway.id, DeviceType.appliance.id, ) ).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=( DeviceType.switch.id, DeviceType.router.id, DeviceType.firewall.id, DeviceType.smtp_gateway.id, DeviceType.appliance.id, )) ) yield _total_dict('Cloud', query.filter( device__model__type__in=(DeviceType.cloud_server.id,) ), start, end, _get_search_url(venture, type=(DeviceType.cloud_server.id,)) ) if overlap: yield _total_dict('Unknown', query.filter( device__model__type__in=(DeviceType.unknown.id,)), start, end, _get_search_url(venture, type=(DeviceType.unknown.id,)) ) for dc in DataCenter.objects.all(): yield _total_dict(' • Unknown in %s' % dc.name, query.filter( device__model__type__in=(DeviceType.unknown.id,) ).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(DeviceType.unknown.id,)) ) ( splunk_cost, splunk_count, splunk_count_now, splunk_size, ) = SplunkUsage.get_cost(venture, start, end) if splunk_cost: url = None try: splunk_model = ComponentModel.objects.get(family='splunkvolume') except ComponentModel.DoesNotExist: pass else: if splunk_model.group_id: url = ('/ui/search/components/' '?component_group=%d' % splunk_model.group_id) yield { 'name': 'Splunk usage ({:,.0f} MB)'.format( splunk_size).replace(',', ' '), 'cost': splunk_cost, 'count': splunk_count, 'count_now': splunk_count_now, 'url': url, } for extra_id, in query.values_list('extra_id').distinct(): if extra_id is None: continue extra = VentureExtraCost.objects.get(id=extra_id) q = query.filter(extra=extra) cost = get_total_cost(q, start, end) count, count_now, devices = get_total_count(q, start, end) if count: yield { 'name': extra.name + ' (from %s)' % extra.venture.name, 'count': 'expires %s' % extra.expire.strftime( '%Y-%m-%d') if extra.expire else '', 'cost': cost, 'count_now': count_now, } if overlap: yield _total_dict( 'Total', query, start, end, _get_search_url(venture, type=()), ) yield _total_dict( 'Total physical', query.exclude( device__model__type__in=( DeviceType.cloud_server, DeviceType.virtual_server, DeviceType.unknown, DeviceType.data_center, DeviceType.rack, DeviceType.management, ), ).exclude( device=None, ), start, end, _get_search_url(venture, type=()), )
def setUp(self): super(DeviceWithPricingResourceTest, self).setUp() self.resource = 'devicewithpricing' self.user = User.objects.create_user( 'ralph', '*****@*****.**', 'ralph' ) self.venture = Venture(name='Infra').save() self.deprecation_kind = DeprecationKind( name='12 months', months=12 ).save() srv1 = { 'sn': 'srv-1', 'model_name': 'server', 'model_type': DeviceType.virtual_server, 'venture': self.venture, 'name': 'Srv 1', 'purchase_date': datetime.datetime(2020, 1, 1, 0, 0), 'deprecation_kind': self.deprecation_kind, } srv1_cpu = { 'model_name': 'Intel PCU1', 'label': 'CPU 1', 'priority': 0, 'family': 'Intsels', 'price': 120, 'count': 2, 'speed': 1200, } srv1_memory = { 'priority': 0, 'family': 'Noname RAM', 'label': 'Memory 1GB', 'price': 100, 'speed': 1033, 'size': 512, 'count': 6, } srv1_storage = { 'model_name': 'Store 1TB', 'label': 'store 1TB', 'priority': 0, 'family': 'Noname Store', 'price': 180, 'count': 10, } self.device = create_device( device=srv1, cpu=srv1_cpu, memory=srv1_memory, storage=srv1_storage, ) self.device.save() name = 'Splunk Volume 100 GiB' symbol = 'splunkvolume' model, created = ComponentModel.create( ComponentType.unknown, family=symbol, name=name, priority=0, ) model_group, created = ComponentModelGroup.objects.get_or_create( name='Group Splunk', price=128, type=ComponentType.unknown, ) model.group = model_group model.save() res, created = SplunkUsage.concurrent_get_or_create( device=self.device, day=datetime.date.today(), defaults={'model': model}, ) res.size = 10 res.save()
def setUp(self): super(DeviceWithPricingResourceTest, self).setUp() self.resource = 'devicewithpricing' self.user = User.objects.create_user('ralph', '*****@*****.**', 'ralph') self.venture = Venture(name='Infra').save() self.deprecation_kind = DeprecationKind(name='12 months', months=12).save() srv1 = { 'sn': 'srv-1', 'model_name': 'server', 'model_type': DeviceType.virtual_server, 'venture': self.venture, 'name': 'Srv 1', 'purchase_date': datetime.datetime(2020, 1, 1, 0, 0), 'deprecation_kind': self.deprecation_kind, } srv1_cpu = { 'model_name': 'Intel PCU1', 'label': 'CPU 1', 'priority': 0, 'family': 'Intsels', 'price': 120, 'count': 2, 'speed': 1200, } srv1_memory = { 'priority': 0, 'family': 'Noname RAM', 'label': 'Memory 1GB', 'price': 100, 'speed': 1033, 'size': 512, 'count': 6, } srv1_storage = { 'model_name': 'Store 1TB', 'label': 'store 1TB', 'priority': 0, 'family': 'Noname Store', 'price': 180, 'count': 10, } self.device = create_device( device=srv1, cpu=srv1_cpu, memory=srv1_memory, storage=srv1_storage, ) self.device.save() name = 'Splunk Volume 100 GiB' symbol = 'splunkvolume' model, created = ComponentModel.create( ComponentType.unknown, family=symbol, name=name, priority=0, ) model_group, created = ComponentModelGroup.objects.get_or_create( name='Group Splunk', price=128, type=ComponentType.unknown, ) model.group = model_group model.save() res, created = SplunkUsage.concurrent_get_or_create( device=self.device, day=datetime.date.today(), defaults={'model': model}, ) res.size = 10 res.save()
def _get_summaries(query, start, end, overlap=True, venture=None): if overlap: yield _total_dict( 'Servers', query.filter( device__model__type__in=(DeviceType.rack_server.id, DeviceType.blade_server.id, DeviceType.virtual_server.id)), start, end, _get_search_url(venture, type=(201, 202, 203))) for dc in DataCenter.objects.all(): yield _total_dict( ' • Servers in %s' % dc.name, query.filter( device__model__type__in=(DeviceType.rack_server.id, DeviceType.blade_server.id, DeviceType.virtual_server.id)).filter( device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(201, 202, 203))) if overlap: yield _total_dict( ' ∙ Rack servers in %s' % dc.name, query.filter( device__model__type=DeviceType.rack_server.id, ).filter( device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(201, ))) for mg in DeviceModelGroup.objects.filter( type=DeviceType.rack_server.id).order_by('name'): yield _total_dict( ' %s in %s' % (mg, dc.name), query.filter(device__model__group=mg, ).filter( device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(201, ), model_group=mg.id)) yield _total_dict( ' ∙ Blade servers in %s' % dc.name, query.filter( device__model__type=DeviceType.blade_server.id, ).filter( device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(202, ))) for mg in DeviceModelGroup.objects.filter( type=DeviceType.blade_server.id).order_by('name'): yield _total_dict( ' %s in %s' % (mg, dc.name), query.filter(device__model__group=mg, ).filter( device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(202, ), model_group=mg.id)) yield _total_dict( ' ∙ Virtual servers in %s' % dc.name, query.filter( device__model__type=DeviceType.virtual_server.id, ).filter( device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(203, ))) if overlap: yield _total_dict( 'Loadbalancers', query.filter( device__model__type__in=(DeviceType.load_balancer.id, )), start, end, _get_search_url(venture, type=(103, ))) for dc in DataCenter.objects.all(): yield _total_dict( ' • Loadbalancers in %s' % dc.name, query.filter(device__model__type__in=( DeviceType.load_balancer.id, )).filter( device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(103, ))) if overlap: yield _total_dict( 'Storage', query.filter(device__model__type__in=( DeviceType.storage.id, DeviceType.fibre_channel_switch.id, )), start, end, _get_search_url(venture, type=(301, ))) for dc in DataCenter.objects.all(): yield _total_dict( ' • Storage in %s' % dc.name, query.filter(device__model__type__in=( DeviceType.storage.id, DeviceType.fibre_channel_switch.id, )).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(301, ))) if overlap: yield _total_dict( 'Network', query.filter(device__model__type__in=( DeviceType.switch.id, DeviceType.router.id, DeviceType.firewall.id, DeviceType.smtp_gateway.id, DeviceType.appliance.id, )), start, end, _get_search_url(venture, type=( DeviceType.switch.id, DeviceType.router.id, DeviceType.firewall.id, DeviceType.smtp_gateway.id, DeviceType.appliance.id, ))) for dc in DataCenter.objects.all(): yield _total_dict( ' • Network in %s' % dc.name, query.filter(device__model__type__in=( DeviceType.switch.id, DeviceType.router.id, DeviceType.firewall.id, DeviceType.smtp_gateway.id, DeviceType.appliance.id, )).filter(device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=( DeviceType.switch.id, DeviceType.router.id, DeviceType.firewall.id, DeviceType.smtp_gateway.id, DeviceType.appliance.id, ))) yield _total_dict( 'Cloud', query.filter(device__model__type__in=(DeviceType.cloud_server.id, )), start, end, _get_search_url(venture, type=(DeviceType.cloud_server.id, ))) if overlap: yield _total_dict( 'Unknown', query.filter(device__model__type__in=(DeviceType.unknown.id, )), start, end, _get_search_url(venture, type=(DeviceType.unknown.id, ))) for dc in DataCenter.objects.all(): yield _total_dict( ' • Unknown in %s' % dc.name, query.filter( device__model__type__in=(DeviceType.unknown.id, )).filter( device__dc__iexact=dc.name), start, end, _get_search_url(venture, dc=dc, type=(DeviceType.unknown.id, ))) ( splunk_cost, splunk_count, splunk_count_now, splunk_size, ) = SplunkUsage.get_cost(venture, start, end) if splunk_cost: url = None try: splunk_model = ComponentModel.objects.get(family='splunkvolume') except ComponentModel.DoesNotExist: pass else: if splunk_model.group_id: url = ('/ui/search/components/' '?component_group=%d' % splunk_model.group_id) yield { 'name': 'Splunk usage ({:,.0f} MB)'.format(splunk_size).replace(',', ' '), 'cost': splunk_cost, 'count': splunk_count, 'count_now': splunk_count_now, 'url': url, } for extra_id, in query.values_list('extra_id').distinct(): if extra_id is None: continue extra = VentureExtraCost.objects.get(id=extra_id) q = query.filter(extra=extra) cost = get_total_cost(q, start, end) count, count_now, devices = get_total_count(q, start, end) if count: yield { 'name': extra.name + ' (from %s)' % extra.venture.name, 'count': 'expires %s' % extra.expire.strftime('%Y-%m-%d') if extra.expire else '', 'cost': cost, 'count_now': count_now, } if overlap: yield _total_dict( 'Total', query, start, end, _get_search_url(venture, type=()), ) yield _total_dict( 'Total physical', query.exclude(device__model__type__in=( DeviceType.cloud_server, DeviceType.virtual_server, DeviceType.unknown, DeviceType.data_center, DeviceType.rack, DeviceType.management, ), ).exclude(device=None, ), start, end, _get_search_url(venture, type=()), )
def _report_ventures_data_provider(start, end, ventures_ids, extra_types): ventures = Venture.objects.filter(id__in=ventures_ids).order_by('path') total_cloud_cost = get_total_cost( HistoryCost.objects.filter( device__model__type=DeviceType.cloud_server.id, ), start, end, ) result = [] for venture in ventures: query = HistoryCost.objects.filter( db.Q(venture=venture) | db.Q(venture__parent=venture) | db.Q(venture__parent__parent=venture) | db.Q(venture__parent__parent__parent=venture) | db.Q(venture__parent__parent__parent__parent=venture) ).exclude(device__deleted=True) data = _report_ventures_get_totals(start, end, query, extra_types) ( splunk_cost, splunk_count, splunk_count_now, splunk_size, ) = SplunkUsage.get_cost(venture, start, end) data.update({ 'id': venture.id, 'name': venture.name, 'symbol': venture.symbol, 'path': venture.path, 'department': unicode(venture.department or ''), 'margin': venture.get_margin(), 'top_level': venture.parent is None, 'venture_icon': get_venture_icon(venture), 'cloud_use': ( (data['cloud_cost'] or 0) / total_cloud_cost ) if total_cloud_cost else 0, 'splunk_cost': splunk_cost, }) result.append(data) if venture.parent is not None: continue if not venture.child_set.exists(): continue query = HistoryCost.objects.filter(venture=venture) data = _report_ventures_get_totals(start, end, query, extra_types) ( splunk_cost, splunk_count, splunk_count_now, splunk_size, ) = SplunkUsage.get_cost(venture, start, end, shallow=True) data.update({ 'id': venture.id, 'name': '-', 'symbol': venture.symbol, 'path': venture.path, 'department': unicode(venture.department or ''), 'margin': venture.get_margin(), 'top_level': False, 'venture_icon': get_venture_icon(venture), 'cloud_use': ( (data['cloud_cost'] or 0) / total_cloud_cost ) if total_cloud_cost else 0, 'splunk_cost': splunk_cost, }) result.append(data) return result