def splunk(**options):
splunk = Splunk()
splunk.start()
percent = splunk.progress
while percent < 100:
if options['verbose']:
print(percent)
time.sleep(30)
percent = splunk.progress
hosts = {}
total_mb = 0
for item in splunk.results:
host = item['host']
mb = float(item['MBytes'])
total_mb += mb
if host in hosts:
hosts[host] += mb
else:
hosts[host] = mb
if options['verbose']:
print(len(hosts), 'hosts used', total_mb, ' MiBs total.')
for host, usage in hosts.iteritems():
ip = IPAddress.objects.filter(hostname__startswith=host).order_by(
'-last_seen')
if not ip.count():
if options['verbose']:
print('Warning: host', host, 'not found in device database.')
continue
dev = ip[0].device
if not dev:
if options['verbose']:
print('Warning: host', host, 'not tied to a device in the '
'database.')
continue
name = 'Splunk Volume 100 GiB'
symbol = 'splunkvolume'
model, created = ComponentModel.create(
ComponentType.unknown,
family=symbol,
name=name,
priority=0,
)
res, created = SplunkUsage.concurrent_get_or_create(
device=dev,
day=date.today(),
defaults={'model': model},
)
res.size = usage
res.save()
return True, 'done.', options
def splunk(**options):
splunk = Splunk()
splunk.start()
percent = splunk.progress
while percent < 100:
if options['verbose']:
print(percent)
time.sleep(30)
percent = splunk.progress
hosts = {}
total_mb = 0
for item in splunk.results:
host = item['host']
mb = float(item['MBytes'])
total_mb += mb
if host in hosts:
hosts[host] += mb
else:
hosts[host] = mb
if options['verbose']:
print(len(hosts), 'hosts used', total_mb, ' MiBs total.')
for host, usage in hosts.iteritems():
ip = IPAddress.objects.filter(
hostname__startswith=host).order_by('-last_seen')
if not ip.count():
if options['verbose']:
print('Warning: host', host, 'not found in device database.')
continue
dev = ip[0].device
if not dev:
if options['verbose']:
print('Warning: host', host, 'not tied to a device in the '
'database.')
continue
name = 'Splunk Volume 100 GiB'
symbol = 'splunkvolume'
model, created = ComponentModel.create(
ComponentType.unknown,
family=symbol,
name=name,
priority=0,
)
res, created = SplunkUsage.concurrent_get_or_create(
device=dev,
day=date.today(),
defaults={'model': model},
)
res.size = usage
res.save()
return True, 'done.', options
def splunk(**options):
splunk = Splunk()
splunk.start()
percent = splunk.progress
while percent < 100:
if options["verbose"]:
print(percent)
time.sleep(30)
percent = splunk.progress
hosts = {}
total_mb = 0
for item in splunk.results:
host = item["host"]
mb = float(item["MBytes"])
total_mb += mb
if host in hosts:
hosts[host] += mb
else:
hosts[host] = mb
if options["verbose"]:
print(len(hosts), "hosts used", total_mb, " MiBs total.")
for host, usage in hosts.iteritems():
ip = IPAddress.objects.filter(hostname__startswith=host).order_by("-last_seen")
if not ip.count():
if options["verbose"]:
print("Warning: host", host, "not found in device database.")
continue
dev = ip[0].device
if not dev:
if options["verbose"]:
print("Warning: host", host, "not tied to a device in the " "database.")
continue
name = "Splunk Volume 100 GiB"
symbol = "splunkvolume"
model, created = ComponentModel.concurrent_get_or_create(
type=ComponentType.unknown.id, speed=0, cores=0, size=0, family=symbol, extra_hash=""
)
if created:
model.name = name
model.save()
res, created = SplunkUsage.concurrent_get_or_create(model=model, device=dev, day=date.today())
res.size = usage
res.save()
return True, "done.", options
def _report_ventures_data_provider(start, end, ventures_ids, extra_types):
ventures = Venture.objects.filter(id__in=ventures_ids).order_by('path')
total_cloud_cost = get_total_cost(
HistoryCost.objects.filter(
device__model__type=DeviceType.cloud_server.id, ),
start,
end,
)
result = []
for venture in ventures:
query = HistoryCost.objects.filter(
db.Q(venture=venture) | db.Q(venture__parent=venture)
| db.Q(venture__parent__parent=venture)
| db.Q(venture__parent__parent__parent=venture)
| db.Q(venture__parent__parent__parent__parent=venture)).exclude(
device__deleted=True)
data = _report_ventures_get_totals(start, end, query, extra_types)
(
splunk_cost,
splunk_count,
splunk_count_now,
splunk_size,
) = SplunkUsage.get_cost(venture, start, end)
data.update({
'id':
venture.id,
'name':
venture.name,
'symbol':
venture.symbol,
'path':
venture.path,
'department':
unicode(venture.department or ''),
'margin':
venture.get_margin(),
'top_level':
venture.parent is None,
'venture_icon':
get_venture_icon(venture),
'cloud_use': ((data['cloud_cost'] or 0) /
total_cloud_cost) if total_cloud_cost else 0,
'splunk_cost':
splunk_cost,
})
result.append(data)
if venture.parent is not None:
continue
if not venture.child_set.exists():
continue
query = HistoryCost.objects.filter(venture=venture)
data = _report_ventures_get_totals(start, end, query, extra_types)
(
splunk_cost,
splunk_count,
splunk_count_now,
splunk_size,
) = SplunkUsage.get_cost(venture, start, end, shallow=True)
data.update({
'id':
venture.id,
'name':
'-',
'symbol':
venture.symbol,
'path':
venture.path,
'department':
unicode(venture.department or ''),
'margin':
venture.get_margin(),
'top_level':
False,
'venture_icon':
get_venture_icon(venture),
'cloud_use': ((data['cloud_cost'] or 0) /
total_cloud_cost) if total_cloud_cost else 0,
'splunk_cost':
splunk_cost,
})
result.append(data)
return result
def _get_summaries(query, start, end, overlap=True, venture=None):
if overlap:
yield _total_dict('Servers', query.filter(
device__model__type__in=(DeviceType.rack_server.id,
DeviceType.blade_server.id,
DeviceType.virtual_server.id)), start, end,
_get_search_url(venture, type=(201, 202, 203)))
for dc in DataCenter.objects.all():
yield _total_dict(' • Servers in %s' % dc.name, query.filter(
device__model__type__in=(DeviceType.rack_server.id,
DeviceType.blade_server.id,
DeviceType.virtual_server.id)
).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(201, 202, 203))
)
if overlap:
yield _total_dict(
' ∙ Rack servers in %s' % dc.name, query.filter(
device__model__type=DeviceType.rack_server.id,
).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(201,))
)
for mg in DeviceModelGroup.objects.filter(
type=DeviceType.rack_server.id).order_by('name'):
yield _total_dict(
' %s in %s' % (mg, dc.name), query.filter(
device__model__group=mg,
).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(201,),
model_group=mg.id)
)
yield _total_dict(
' ∙ Blade servers in %s' % dc.name, query.filter(
device__model__type=DeviceType.blade_server.id,
).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(202,))
)
for mg in DeviceModelGroup.objects.filter(
type=DeviceType.blade_server.id).order_by('name'):
yield _total_dict(
' %s in %s' % (mg, dc.name), query.filter(
device__model__group=mg,
).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(202,),
model_group=mg.id)
)
yield _total_dict(
' ∙ Virtual servers in %s' % dc.name, query.filter(
device__model__type=DeviceType.virtual_server.id,
).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(203,))
)
if overlap:
yield _total_dict('Loadbalancers', query.filter(
device__model__type__in=(DeviceType.load_balancer.id,)
), start, end, _get_search_url(venture, type=(103,)))
for dc in DataCenter.objects.all():
yield _total_dict(' • Loadbalancers in %s' % dc.name, query.filter(
device__model__type__in=(DeviceType.load_balancer.id,)
).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(103,))
)
if overlap:
yield _total_dict('Storage', query.filter(
device__model__type__in=(
DeviceType.storage.id,
DeviceType.fibre_channel_switch.id,
)), start, end,
_get_search_url(venture, type=(301,))
)
for dc in DataCenter.objects.all():
yield _total_dict(' • Storage in %s' % dc.name, query.filter(
device__model__type__in=(
DeviceType.storage.id,
DeviceType.fibre_channel_switch.id,
)
).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(301,))
)
if overlap:
yield _total_dict('Network', query.filter(
device__model__type__in=(
DeviceType.switch.id,
DeviceType.router.id,
DeviceType.firewall.id,
DeviceType.smtp_gateway.id,
DeviceType.appliance.id,
)
), start, end,
_get_search_url(venture, type=(
DeviceType.switch.id,
DeviceType.router.id,
DeviceType.firewall.id,
DeviceType.smtp_gateway.id,
DeviceType.appliance.id,
))
)
for dc in DataCenter.objects.all():
yield _total_dict(' • Network in %s' % dc.name, query.filter(
device__model__type__in=(
DeviceType.switch.id,
DeviceType.router.id,
DeviceType.firewall.id,
DeviceType.smtp_gateway.id,
DeviceType.appliance.id,
)
).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(
DeviceType.switch.id,
DeviceType.router.id,
DeviceType.firewall.id,
DeviceType.smtp_gateway.id,
DeviceType.appliance.id,
))
)
yield _total_dict('Cloud', query.filter(
device__model__type__in=(DeviceType.cloud_server.id,)
), start, end,
_get_search_url(venture, type=(DeviceType.cloud_server.id,))
)
if overlap:
yield _total_dict('Unknown', query.filter(
device__model__type__in=(DeviceType.unknown.id,)), start, end,
_get_search_url(venture, type=(DeviceType.unknown.id,))
)
for dc in DataCenter.objects.all():
yield _total_dict(' • Unknown in %s' % dc.name, query.filter(
device__model__type__in=(DeviceType.unknown.id,)
).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(DeviceType.unknown.id,))
)
(
splunk_cost,
splunk_count,
splunk_count_now,
splunk_size,
) = SplunkUsage.get_cost(venture, start, end)
if splunk_cost:
url = None
try:
splunk_model = ComponentModel.objects.get(family='splunkvolume')
except ComponentModel.DoesNotExist:
pass
else:
if splunk_model.group_id:
url = ('/ui/search/components/'
'?component_group=%d' % splunk_model.group_id)
yield {
'name': 'Splunk usage ({:,.0f} MB)'.format(
splunk_size).replace(',', ' '),
'cost': splunk_cost,
'count': splunk_count,
'count_now': splunk_count_now,
'url': url,
}
for extra_id, in query.values_list('extra_id').distinct():
if extra_id is None:
continue
extra = VentureExtraCost.objects.get(id=extra_id)
q = query.filter(extra=extra)
cost = get_total_cost(q, start, end)
count, count_now, devices = get_total_count(q, start, end)
if count:
yield {
'name': extra.name + ' (from %s)' % extra.venture.name,
'count': 'expires %s' % extra.expire.strftime(
'%Y-%m-%d') if extra.expire else '',
'cost': cost,
'count_now': count_now,
}
if overlap:
yield _total_dict(
'Total',
query,
start,
end,
_get_search_url(venture, type=()),
)
yield _total_dict(
'Total physical',
query.exclude(
device__model__type__in=(
DeviceType.cloud_server,
DeviceType.virtual_server,
DeviceType.unknown,
DeviceType.data_center,
DeviceType.rack,
DeviceType.management,
),
).exclude(
device=None,
),
start,
end,
_get_search_url(venture, type=()),
)
def setUp(self):
super(DeviceWithPricingResourceTest, self).setUp()
self.resource = 'devicewithpricing'
self.user = User.objects.create_user(
'ralph',
'*****@*****.**',
'ralph'
)
self.venture = Venture(name='Infra').save()
self.deprecation_kind = DeprecationKind(
name='12 months',
months=12
).save()
srv1 = {
'sn': 'srv-1',
'model_name': 'server',
'model_type': DeviceType.virtual_server,
'venture': self.venture,
'name': 'Srv 1',
'purchase_date': datetime.datetime(2020, 1, 1, 0, 0),
'deprecation_kind': self.deprecation_kind,
}
srv1_cpu = {
'model_name': 'Intel PCU1',
'label': 'CPU 1',
'priority': 0,
'family': 'Intsels',
'price': 120,
'count': 2,
'speed': 1200,
}
srv1_memory = {
'priority': 0,
'family': 'Noname RAM',
'label': 'Memory 1GB',
'price': 100,
'speed': 1033,
'size': 512,
'count': 6,
}
srv1_storage = {
'model_name': 'Store 1TB',
'label': 'store 1TB',
'priority': 0,
'family': 'Noname Store',
'price': 180,
'count': 10,
}
self.device = create_device(
device=srv1,
cpu=srv1_cpu,
memory=srv1_memory,
storage=srv1_storage,
)
self.device.save()
name = 'Splunk Volume 100 GiB'
symbol = 'splunkvolume'
model, created = ComponentModel.create(
ComponentType.unknown,
family=symbol,
name=name,
priority=0,
)
model_group, created = ComponentModelGroup.objects.get_or_create(
name='Group Splunk',
price=128,
type=ComponentType.unknown,
)
model.group = model_group
model.save()
res, created = SplunkUsage.concurrent_get_or_create(
device=self.device,
day=datetime.date.today(),
defaults={'model': model},
)
res.size = 10
res.save()
def setUp(self):
super(DeviceWithPricingResourceTest, self).setUp()
self.resource = 'devicewithpricing'
self.user = User.objects.create_user('ralph', '*****@*****.**',
'ralph')
self.venture = Venture(name='Infra').save()
self.deprecation_kind = DeprecationKind(name='12 months',
months=12).save()
srv1 = {
'sn': 'srv-1',
'model_name': 'server',
'model_type': DeviceType.virtual_server,
'venture': self.venture,
'name': 'Srv 1',
'purchase_date': datetime.datetime(2020, 1, 1, 0, 0),
'deprecation_kind': self.deprecation_kind,
}
srv1_cpu = {
'model_name': 'Intel PCU1',
'label': 'CPU 1',
'priority': 0,
'family': 'Intsels',
'price': 120,
'count': 2,
'speed': 1200,
}
srv1_memory = {
'priority': 0,
'family': 'Noname RAM',
'label': 'Memory 1GB',
'price': 100,
'speed': 1033,
'size': 512,
'count': 6,
}
srv1_storage = {
'model_name': 'Store 1TB',
'label': 'store 1TB',
'priority': 0,
'family': 'Noname Store',
'price': 180,
'count': 10,
}
self.device = create_device(
device=srv1,
cpu=srv1_cpu,
memory=srv1_memory,
storage=srv1_storage,
)
self.device.save()
name = 'Splunk Volume 100 GiB'
symbol = 'splunkvolume'
model, created = ComponentModel.create(
ComponentType.unknown,
family=symbol,
name=name,
priority=0,
)
model_group, created = ComponentModelGroup.objects.get_or_create(
name='Group Splunk',
price=128,
type=ComponentType.unknown,
)
model.group = model_group
model.save()
res, created = SplunkUsage.concurrent_get_or_create(
device=self.device,
day=datetime.date.today(),
defaults={'model': model},
)
res.size = 10
res.save()
def _get_summaries(query, start, end, overlap=True, venture=None):
if overlap:
yield _total_dict(
'Servers',
query.filter(
device__model__type__in=(DeviceType.rack_server.id,
DeviceType.blade_server.id,
DeviceType.virtual_server.id)), start,
end, _get_search_url(venture, type=(201, 202, 203)))
for dc in DataCenter.objects.all():
yield _total_dict(
' • Servers in %s' % dc.name,
query.filter(
device__model__type__in=(DeviceType.rack_server.id,
DeviceType.blade_server.id,
DeviceType.virtual_server.id)).filter(
device__dc__iexact=dc.name),
start, end, _get_search_url(venture, dc=dc, type=(201, 202, 203)))
if overlap:
yield _total_dict(
' ∙ Rack servers in %s' % dc.name,
query.filter(
device__model__type=DeviceType.rack_server.id, ).filter(
device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(201, )))
for mg in DeviceModelGroup.objects.filter(
type=DeviceType.rack_server.id).order_by('name'):
yield _total_dict(
' %s in %s' % (mg, dc.name),
query.filter(device__model__group=mg, ).filter(
device__dc__iexact=dc.name), start, end,
_get_search_url(venture,
dc=dc,
type=(201, ),
model_group=mg.id))
yield _total_dict(
' ∙ Blade servers in %s' % dc.name,
query.filter(
device__model__type=DeviceType.blade_server.id, ).filter(
device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(202, )))
for mg in DeviceModelGroup.objects.filter(
type=DeviceType.blade_server.id).order_by('name'):
yield _total_dict(
' %s in %s' % (mg, dc.name),
query.filter(device__model__group=mg, ).filter(
device__dc__iexact=dc.name), start, end,
_get_search_url(venture,
dc=dc,
type=(202, ),
model_group=mg.id))
yield _total_dict(
' ∙ Virtual servers in %s' % dc.name,
query.filter(
device__model__type=DeviceType.virtual_server.id, ).filter(
device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(203, )))
if overlap:
yield _total_dict(
'Loadbalancers',
query.filter(
device__model__type__in=(DeviceType.load_balancer.id, )),
start, end, _get_search_url(venture, type=(103, )))
for dc in DataCenter.objects.all():
yield _total_dict(
' • Loadbalancers in %s' % dc.name,
query.filter(device__model__type__in=(
DeviceType.load_balancer.id, )).filter(
device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(103, )))
if overlap:
yield _total_dict(
'Storage',
query.filter(device__model__type__in=(
DeviceType.storage.id,
DeviceType.fibre_channel_switch.id,
)), start, end, _get_search_url(venture, type=(301, )))
for dc in DataCenter.objects.all():
yield _total_dict(
' • Storage in %s' % dc.name,
query.filter(device__model__type__in=(
DeviceType.storage.id,
DeviceType.fibre_channel_switch.id,
)).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(301, )))
if overlap:
yield _total_dict(
'Network',
query.filter(device__model__type__in=(
DeviceType.switch.id,
DeviceType.router.id,
DeviceType.firewall.id,
DeviceType.smtp_gateway.id,
DeviceType.appliance.id,
)), start, end,
_get_search_url(venture,
type=(
DeviceType.switch.id,
DeviceType.router.id,
DeviceType.firewall.id,
DeviceType.smtp_gateway.id,
DeviceType.appliance.id,
)))
for dc in DataCenter.objects.all():
yield _total_dict(
' • Network in %s' % dc.name,
query.filter(device__model__type__in=(
DeviceType.switch.id,
DeviceType.router.id,
DeviceType.firewall.id,
DeviceType.smtp_gateway.id,
DeviceType.appliance.id,
)).filter(device__dc__iexact=dc.name), start, end,
_get_search_url(venture,
dc=dc,
type=(
DeviceType.switch.id,
DeviceType.router.id,
DeviceType.firewall.id,
DeviceType.smtp_gateway.id,
DeviceType.appliance.id,
)))
yield _total_dict(
'Cloud',
query.filter(device__model__type__in=(DeviceType.cloud_server.id, )),
start, end,
_get_search_url(venture, type=(DeviceType.cloud_server.id, )))
if overlap:
yield _total_dict(
'Unknown',
query.filter(device__model__type__in=(DeviceType.unknown.id, )),
start, end, _get_search_url(venture,
type=(DeviceType.unknown.id, )))
for dc in DataCenter.objects.all():
yield _total_dict(
' • Unknown in %s' % dc.name,
query.filter(
device__model__type__in=(DeviceType.unknown.id, )).filter(
device__dc__iexact=dc.name), start, end,
_get_search_url(venture, dc=dc, type=(DeviceType.unknown.id, )))
(
splunk_cost,
splunk_count,
splunk_count_now,
splunk_size,
) = SplunkUsage.get_cost(venture, start, end)
if splunk_cost:
url = None
try:
splunk_model = ComponentModel.objects.get(family='splunkvolume')
except ComponentModel.DoesNotExist:
pass
else:
if splunk_model.group_id:
url = ('/ui/search/components/'
'?component_group=%d' % splunk_model.group_id)
yield {
'name':
'Splunk usage ({:,.0f} MB)'.format(splunk_size).replace(',', ' '),
'cost':
splunk_cost,
'count':
splunk_count,
'count_now':
splunk_count_now,
'url':
url,
}
for extra_id, in query.values_list('extra_id').distinct():
if extra_id is None:
continue
extra = VentureExtraCost.objects.get(id=extra_id)
q = query.filter(extra=extra)
cost = get_total_cost(q, start, end)
count, count_now, devices = get_total_count(q, start, end)
if count:
yield {
'name':
extra.name + ' (from %s)' % extra.venture.name,
'count':
'expires %s' %
extra.expire.strftime('%Y-%m-%d') if extra.expire else '',
'cost':
cost,
'count_now':
count_now,
}
if overlap:
yield _total_dict(
'Total',
query,
start,
end,
_get_search_url(venture, type=()),
)
yield _total_dict(
'Total physical',
query.exclude(device__model__type__in=(
DeviceType.cloud_server,
DeviceType.virtual_server,
DeviceType.unknown,
DeviceType.data_center,
DeviceType.rack,
DeviceType.management,
), ).exclude(device=None, ),
start,
end,
_get_search_url(venture, type=()),
)
def _report_ventures_data_provider(start, end, ventures_ids, extra_types):
ventures = Venture.objects.filter(id__in=ventures_ids).order_by('path')
total_cloud_cost = get_total_cost(
HistoryCost.objects.filter(
device__model__type=DeviceType.cloud_server.id,
),
start,
end,
)
result = []
for venture in ventures:
query = HistoryCost.objects.filter(
db.Q(venture=venture) |
db.Q(venture__parent=venture) |
db.Q(venture__parent__parent=venture) |
db.Q(venture__parent__parent__parent=venture) |
db.Q(venture__parent__parent__parent__parent=venture)
).exclude(device__deleted=True)
data = _report_ventures_get_totals(start, end, query, extra_types)
(
splunk_cost,
splunk_count,
splunk_count_now,
splunk_size,
) = SplunkUsage.get_cost(venture, start, end)
data.update({
'id': venture.id,
'name': venture.name,
'symbol': venture.symbol,
'path': venture.path,
'department': unicode(venture.department or ''),
'margin': venture.get_margin(),
'top_level': venture.parent is None,
'venture_icon': get_venture_icon(venture),
'cloud_use': (
(data['cloud_cost'] or 0) / total_cloud_cost
) if total_cloud_cost else 0,
'splunk_cost': splunk_cost,
})
result.append(data)
if venture.parent is not None:
continue
if not venture.child_set.exists():
continue
query = HistoryCost.objects.filter(venture=venture)
data = _report_ventures_get_totals(start, end, query, extra_types)
(
splunk_cost,
splunk_count,
splunk_count_now,
splunk_size,
) = SplunkUsage.get_cost(venture, start, end, shallow=True)
data.update({
'id': venture.id,
'name': '-',
'symbol': venture.symbol,
'path': venture.path,
'department': unicode(venture.department or ''),
'margin': venture.get_margin(),
'top_level': False,
'venture_icon': get_venture_icon(venture),
'cloud_use': (
(data['cloud_cost'] or 0) / total_cloud_cost
) if total_cloud_cost else 0,
'splunk_cost': splunk_cost,
})
result.append(data)
return result