def choose_ip_addr(request):
if 'network' not in request.GET:
servers = list(
Query(
{'servertype': 'route_network'},
['hostname', 'intern_ip'],
['hostname'],
))
return TemplateResponse(request, 'servershell/choose_ip_addr.html',
{'servers': servers})
network = request.GET['network']
servers = list(
Query(
{
'servertype':
Any(*(s.servertype_id for s in Servertype.objects.filter(
ip_addr_type='network'))),
'intern_ip':
ContainedOnlyBy(network),
},
['hostname', 'intern_ip'],
['hostname'],
))
if servers:
return TemplateResponse(request, 'servershell/choose_ip_addr.html',
{'servers': servers})
network_query = Query({'intern_ip': network}, ['intern_ip'])
return TemplateResponse(
request, 'servershell/choose_ip_addr.html',
{'ip_addrs': islice(network_query.get_free_ip_addrs(), 1000)})
def _create_serveradmin_domain():
serveradmin_domain = Query().new_object('domain')
serveradmin_domain['hostname'] = 'innogames.net'
serveradmin_domain['type'] = 'NATIVE'
serveradmin_domain.commit(user=User.objects.first())
return serveradmin_domain
def test_change_server_hostname(self):
server = self._get_server('network')
server['intern_ip'] = '10.0.0.0/30'
server.commit(user=User.objects.first())
to_rename = Query({'hostname': server['hostname']}, ['hostname'])
to_rename.update(hostname=self.faker.hostname())
self.assertIsNone(to_rename.commit(user=User.objects.first()))
def test_change_server_network_overlaps(self):
first = self._get_server('network')
first['intern_ip'] = '10.0.0.0/30'
first['ip_config'] = '10.0.1.0/30'
first.commit(user=User.objects.first())
host = Query({'hostname': first['hostname']}, ['ip_config'])
host.update(ip_config=IPv4Network('10.0.1.0/28'))
self.assertIsNone(host.commit(user=User.objects.first()))
def test_change_server_hostname(self):
server = self._get_server('loadbalancer')
server['intern_ip'] = '10.0.0.1/32'
server['ip_config'] = '10.0.0.2/32'
server.commit(user=User.objects.first())
to_rename = Query({'hostname': server['hostname']}, ['hostname'])
to_rename.update(hostname=self.faker.hostname())
self.assertIsNone(to_rename.commit(user=User.objects.first()))
def edit(request):
if 'object_id' in request.GET:
server = Query({'object_id': request.GET['object_id']}, None).get()
else:
servertype = request.POST.get('attr_servertype')
if not Servertype.objects.filter(pk=servertype).exists():
raise Http404('Servertype {} does not exist'.format(servertype))
server = Query().new_object(servertype)
return _edit(request, server, True)
def inspect(request):
if 'object_id' in request.GET:
query = Query({'object_id': request.GET['object_id']}, None)
elif 'hostname' in request.GET:
query = Query({'hostname': request.GET['hostname']}, None)
else:
return HttpResponseBadRequest(
'object_id or hostname parameter is mandatory')
if not query:
return HttpResponseNotFound('No such object exists')
return _edit(request, query.get(), template='inspect')
def graph_popup(request):
try:
hostname = request.GET['hostname']
graph_id = request.GET['graph']
except KeyError:
return HttpResponseBadRequest('Hostname and graph not supplied')
# It would be more efficient to filter the collections on the database,
# but we don't bother because they are unlikely to be more than a few
# marked as overview.
for collection in Collection.objects.filter(overview=True):
servers = list(
Query({
GRAPHITE_ATTRIBUTE_ID: collection.name,
'hostname': hostname,
}))
if servers:
table = collection.graph_table(servers[0])
params = [v2 for k1, v1 in table for k2, v2 in v1][int(graph_id)]
url = reverse(graph) + '?' + params
return TemplateResponse(request, 'resources/graph_popup.html',
{'image': url})
return HttpResponseBadRequest('No graph found')
def handle(self, *args, **kwargs):
"""The entry point of the command"""
start = time.time()
sprite_params = settings.GRAPHITE_SPRITE_PARAMS
sprite_dir = settings.MEDIA_ROOT + '/graph_sprite'
if not isdir(sprite_dir):
mkdir(sprite_dir)
# We will make sure to generate a single sprite for a single hostname.
for collection in Collection.objects.filter(overview=True):
collection_start = time.time()
collection_dir = sprite_dir + '/' + collection.name
if not isdir(collection_dir):
mkdir(collection_dir)
for server in Query({
GRAPHITE_ATTRIBUTE_ID: collection.name,
'state': filters.Not('retired'),
}):
graph_table = collection.graph_table(server, sprite_params)
if graph_table:
self.generate_sprite(collection_dir, graph_table, server)
self.cache_numerics(collection, server)
collection_duration = time.time() - collection_start
print('[{}] Collection {} finished after {} seconds'.format(
datetime.now(), collection.name, collection_duration))
duration = time.time() - start
print('[{}] Finished after {} seconds'.format(datetime.now(),
duration))
def create_domains(sender, **kwargs):
"""Create PowerDNS domain for newly created objects
:param sender:
:param kwargs:
:return:
"""
if not kwargs['created']:
return
domain_settings = settings.PDNS.get('domain')
for new_object in kwargs['created']:
servertype = new_object['servertype']
for domain_setting in domain_settings:
if servertype != domain_setting['servertype']:
continue
# The Serveradmin attribute object_id is not present in the
# post_commit created data so we query the data again.
attrs = domain_setting['attributes']
queried_object = Query({
'hostname': new_object[attrs['name']]
}, list(attrs.values())).get()
# All attributes are mandatory
domain = Domain()
domain.id = queried_object['object_id']
domain.name = queried_object[attrs['name']]
domain.type = queried_object[attrs['type']]
domain.save()
def new_object(request):
try:
servertype = request.GET.get('servertype')
new_object = Query().new_object(servertype)
except Servertype.DoesNotExist:
raise Http404
return _edit(request, new_object)
def test_filter_regexp(self):
hostnames = set()
for s in Query({'hostname': Regexp('^test[02]$')}):
hostnames.add(s['hostname'])
self.assertIn('test0', hostnames)
self.assertNotIn('test1', hostnames)
self.assertIn('test2', hostnames)
def get_results(request):
term = request.GET.get('term', '')
shown_attributes = request.GET.get('shown_attributes').split(',')
# We need servertypes to return the attribute properties.
if 'servertype' not in shown_attributes:
shown_attributes.append('servertype')
try:
offset = int(request.GET.get('offset', '0'))
limit = int(request.GET.get('limit', '0'))
except ValueError:
offset = 0
limit = NUM_SERVERS_DEFAULT
if 'order_by' in request.GET:
order_by = [request.GET['order_by']]
else:
order_by = None
try:
query = Query(parse_query(term), shown_attributes, order_by)
num_servers = len(query)
except (DatatypeError, ObjectDoesNotExist, ValidationError) as error:
return HttpResponse(
json.dumps({
'status': 'error',
'message': str(error)
}))
servers = list(islice(query, offset, offset + limit))
request.session['term'] = term
request.session['per_page'] = limit
# Add information about available, editable attributes on servertypes
servertype_ids = {s['servertype'] for s in servers}
editable_attributes = dict()
for servertype_id in servertype_ids:
editable_attributes[servertype_id] = list(Attribute.specials)
for sa in ServertypeAttribute.objects.filter(
servertype_id__in=servertype_ids,
attribute_id__in=shown_attributes,
related_via_attribute_id__isnull=True,
attribute__readonly=False,
):
editable_attributes[sa.servertype_id].append(sa.attribute_id)
return HttpResponse(json.dumps(
{
'status': 'success',
'understood': repr(query),
'servers': servers,
'num_servers': num_servers,
'editable_attributes': editable_attributes,
},
default=json_encode_extra),
content_type='application/x-json')
def export(request):
term = request.GET.get('term', '')
try:
query = Query(parse_query(term), ['hostname'])
except (DatatypeError, ObjectDoesNotExist, ValidationError) as error:
return HttpResponse(str(error), status=400)
hostnames = ' '.join(server['hostname'] for server in query)
return HttpResponse(hostnames, content_type='text/plain')
def clone_object(request):
try:
old_object = Query(
{'object_id': request.GET.get('object_id')},
list(Attribute.specials) + list(
Attribute.objects.filter(clone=True)
.values_list('attribute_id', flat=True)
),
).get()
except ValidationError as e:
messages.error(request, e.message)
return redirect('servershell_index')
new_object = Query().new_object(old_object['servertype'])
for attribute_id, value in old_object.items():
new_object[attribute_id] = value
return _edit(request, new_object)
def clone_object(request):
try:
old_object = Query(
{
'hostname': request.GET.get('hostname')
},
list(Attribute.specials) + list(
Attribute.objects.filter(clone=True).values_list(
'attribute_id', flat=True)),
).get()
except ValidationError:
raise Http404
new_object = Query().new_object(old_object['servertype'])
for attribute_id, value in old_object.items():
new_object[attribute_id] = value
return _edit(request, new_object)
def test_query_iterate(self):
hostnames = set()
for server in Query({}):
hostnames.add(server['hostname'])
self.assertIn('test0', hostnames)
self.assertIn('test1', hostnames)
self.assertIn('test2', hostnames)
self.assertIn('test3', hostnames)
def test_filter_any(self):
hostnames = set()
for s in Query({'hostname': Any('test1', 'test3')}):
hostnames.add(s['hostname'])
self.assertNotIn('test0', hostnames)
self.assertIn('test1', hostnames)
self.assertNotIn('test2', hostnames)
self.assertIn('test3', hostnames)
def new_object(request):
servertype = request.GET.get('servertype')
try:
new_object = Query().new_object(servertype)
except Servertype.DoesNotExist:
messages.error(request,
'The servertype {} does not exist!'.format(servertype))
return redirect('servershell_index')
return _edit(request, new_object)
def clone_object(request):
try:
cloned_attributes = list(Attribute.specials)
# intern_ip is usually unique (except for loadbalancers) therefore it
# makes sense to not clone it.
cloned_attributes.remove('intern_ip')
cloned_attributes.extend(
list(
Attribute.objects.filter(clone=True).values_list(
'attribute_id', flat=True)))
old_object = Query({
'object_id': request.GET.get('object_id')
}, cloned_attributes).get()
except ValidationError as e:
messages.error(request, e.message)
return redirect('servershell_index')
new_object = Query().new_object(old_object['servertype'])
for attribute_id, value in old_object.items():
new_object[attribute_id] = value
return _edit(request, new_object)
def autocomplete(request):
autocomplete_list = []
if 'hostname' in request.GET:
hostname = request.GET['hostname']
try:
query = Query({'hostname': StartsWith(hostname)}, ['hostname'])
autocomplete_list += islice((h['hostname'] for h in query), 100)
except (DatatypeError, ValidationError):
pass # If there is no valid query, just don't auto-complete
return HttpResponse(
json.dumps({'autocomplete': autocomplete_list}),
content_type='application/x-json',
)
def diff(request: HttpRequest) -> HttpResponse:
attrs = request.GET.getlist('attr')
objects = request.GET.getlist('object')
if not objects or not all([o.isnumeric() for o in objects]):
return bad_request(request, HttpResponseBadRequest)
# Can raise ApiError for unknown attributes - let it flow ...
qs = Query({'object_id': Any(*objects)}, attrs if attrs else None)
diff_data = []
for attribute in sorted(set(chain(*[o.keys() for o in qs]))):
# object_id is always different and special
if attribute == 'object_id':
continue
# Show hostname only if request by user
if attribute == 'hostname' and attrs != [] and attribute not in attrs:
continue
values = []
for obj in qs:
values.append(obj[attribute])
diff_data.append([attribute, values])
# Fetch hostnames if not requested by user to display as header in result.
if 'hostname' in attrs:
hosts = qs
else:
hosts = Query({'object_id': Any(*objects)}, ['hostname'])
context = {
'hosts': hosts,
'diff_data': diff_data,
}
return render(request, 'servershell/diff.html', context)
def test_commit_query(self):
q = Query({'hostname': 'test1'}, ['os', 'intern_ip'])
s = q.get()
s['os'] = 'wheezy'
s['intern_ip'] = IPv4Address('10.16.2.1')
q.commit(user=User.objects.first())
s = Query({'hostname': 'test1'}, ['os', 'intern_ip']).get()
self.assertEqual(s['os'], 'wheezy')
self.assertEqual(s['intern_ip'], IPv4Address('10.16.2.1'))
def test_set_attribute(self):
"""Try to set and retrieve a datetime attribute"""
dt = datetime.utcnow().replace(tzinfo=timezone.utc)
q = Query({'hostname': 'test0'}, ['last_edited'])
s = q.get()
s['last_edited'] = dt
q.commit(user=User.objects.first())
s = Query({'hostname': 'test0'}, ['last_edited']).get()
self.assertEqual(s['last_edited'], dt)
def handle(self, *args, **kwargs):
"""The entry point of the command"""
sprite_params = settings.GRAPHITE_SPRITE_PARAMS
sprite_dir = settings.MEDIA_ROOT + '/graph_sprite'
if not isdir(sprite_dir):
mkdir(sprite_dir)
# We will make sure to generate a single sprite for a single hostname.
for collection in Collection.objects.filter(overview=True):
collection_dir = sprite_dir + '/' + collection.name
if not isdir(collection_dir):
mkdir(collection_dir)
for server in Query({GRAPHITE_ATTRIBUTE_ID: collection.name}):
graph_table = collection.graph_table(server, sprite_params)
if graph_table:
self.generate_sprite(collection_dir, graph_table, server)
self.cache_numerics(collection, server)
def test_utc_conversion(self):
"""Ensure datetimes are converted to UTC
Users can pass datetimes in any timezone they want. Serveradmin will
transform them to UTC and only ever return them in UTC form.
"""
class TZ(tzinfo):
def utcoffset(self, dt):
return timedelta(minutes=+3)
q = Query({'hostname': 'test0'}, ['last_edited'])
s = q.get()
s['last_edited'] = datetime(1970, 1, 1, 0, 3, 0).replace(tzinfo=TZ())
q.commit(user=User.objects.first())
s = Query({'hostname': 'test0'}, ['last_edited']).get()
self.assertEqual(str(s['last_edited']), '1970-01-01 00:00:00+00:00')
def update_domains(sender, **kwargs):
"""Update PowerDNS domain when changed
:param sender:
:param kwargs:
:return:
"""
if not kwargs['changed']:
return
# Is any of the updated objects mapped to PowerDNS domain ?
object_ids = [changed['object_id'] for changed in kwargs['changed']]
domains_to_update = Domain.objects.filter(id__in=object_ids)
if not domains_to_update.exists():
return
domain_settings = DomainSettings()
for domain in domains_to_update:
attributes = domain_settings.get_attributes() + ['servertype']
# @TODO Find a way to avoid this extra Query if possible
queried_object = Query({'object_id': domain.id}, attributes).get()
servertype = queried_object['servertype']
this_settings = domain_settings.get_settings(servertype)
changed_object = next(
filter(lambda o: o['object_id'] == domain.id, kwargs['changed']))
has_changed = False
for pdns_key, sa_key in this_settings['attributes'].items():
if sa_key not in changed_object.keys():
continue
has_changed = True
setattr(domain, pdns_key, changed_object[sa_key]['new'])
if has_changed:
domain.save()
def index(request):
"""The hardware resources page"""
term = request.GET.get('term', request.session.get('term', ''))
collections = list(Collection.objects.filter(overview=True))
# If a graph collection was specified, use it. Otherwise use the first
# one.
for collection in collections:
if request.GET.get('current_collection'):
if str(collection.id) != request.GET['current_collection']:
continue
current_collection = collection
break
else:
return HttpResponseBadRequest('No matching current collection')
template_info = {
'search_term': term,
'collections': collections,
'current_collection': current_collection.id,
}
# TODO: Generalize this part using the relations
hostnames = []
matched_hostnames = []
if term:
try:
query_args = parse_query(term)
host_query = Query(query_args, ['hostname', 'hypervisor'])
for host in host_query:
matched_hostnames.append(host['hostname'])
if host.get('hypervisor'):
hostnames.append(host['hypervisor'])
else:
# If it's not guest, it might be a server, so we add it
hostnames.append(host['hostname'])
understood = repr(host_query)
request.session['term'] = term
if len(hostnames) == 0:
template_info.update({
'understood': understood,
})
return TemplateResponse(request, 'resources/index.html',
template_info)
except (DatatypeError, ValidationError) as error:
template_info.update({'error': str(error)})
return TemplateResponse(request, 'resources/index.html',
template_info)
else:
understood = repr(Query({}))
variations = list(current_collection.variation_set.all())
columns = []
attribute_ids = ['hostname', 'servertype']
graph_index = 0
sprite_width = settings.GRAPHITE_SPRITE_WIDTH
for template in current_collection.template_set.all():
for variation in variations:
columns.append({
'name': str(template) + ' ' + str(variation),
'type': 'graph',
'graph_index': graph_index,
'sprite_offset': graph_index * sprite_width,
})
graph_index += 1
for numeric in current_collection.numeric_set.all():
columns.append({
'name': str(numeric),
'type': 'numeric',
})
attribute_ids.append(numeric.attribute_id)
for relation in current_collection.relation_set.all():
columns.append({
'name': str(relation),
'type': 'relation',
})
attribute_ids.append(relation.attribute_id)
hosts = OrderedDict()
filters = {GRAPHITE_ATTRIBUTE_ID: collection.name}
if len(hostnames) > 0:
filters['hostname'] = Any(*hostnames)
for server in Query(filters, attribute_ids):
hosts[server['hostname']] = dict(server)
sprite_url = settings.MEDIA_URL + 'graph_sprite/' + collection.name
template_info.update({
'columns': columns,
'hosts': hosts.values(),
'matched_hostnames': matched_hostnames,
'understood': understood,
'error': None,
'sprite_url': sprite_url,
})
return TemplateResponse(request, 'resources/index.html', template_info)
def test_startswith_servertype(self):
q = Query({'servertype': StartsWith('tes')})
self.assertEqual(len(q), 4)
def test_startswith(self):
s = Query({'os': StartsWith('whee')}).get()
self.assertEqual(s['hostname'], 'test0')