def testGetFromPools(self):
p1 = clusto.get_or_create('p1', Pool)
p2 = clusto.get_or_create('p2', Pool)
s1 = BasicServer('s1')
s2 = BasicServer('s2')
s3 = BasicServer('s3')
s4 = BasicServer('s4')
s5 = BasicServer('s5')
s6 = BasicServer('s6')
p1.insert(s1)
p1.insert(s2)
p1.insert(s3)
p2.insert(s3)
p2.insert(s4)
p2.insert(s5)
self.assertEqual(
sorted([s1, s2, s3]),
sorted(
clusto.get_from_pools(pools=[p1], clusto_types=[BasicServer])))
self.assertEqual(
sorted([s3, s4, s5]),
sorted(
clusto.get_from_pools(pools=[p2], clusto_types=[BasicServer])))
self.assertEqual(
sorted([s3]),
sorted(
clusto.get_from_pools(pools=[p1, p2],
clusto_types=[BasicServer])))
self.assertRaises(LookupError, clusto.get_from_pools, 'p1',
'non-existent-pool')
def testGetFromPools(self):
p1 = clusto.get_or_create('p1', Pool)
p2 = clusto.get_or_create('p2', Pool)
s1 = BasicServer('s1')
s2 = BasicServer('s2')
s3 = BasicServer('s3')
s4 = BasicServer('s4')
s5 = BasicServer('s5')
s6 = BasicServer('s6')
p1.insert(s1)
p1.insert(s2)
p1.insert(s3)
p2.insert(s3)
p2.insert(s4)
p2.insert(s5)
self.assertEqual(sorted([s1, s2, s3]),
sorted(clusto.get_from_pools(pools=[p1],
clusto_types=[BasicServer])))
self.assertEqual(sorted([s3, s4, s5]),
sorted(clusto.get_from_pools(pools=[p2],
clusto_types=[BasicServer])))
self.assertEqual(sorted([s3]),
sorted(clusto.get_from_pools(pools=[p1, p2],
clusto_types=[BasicServer])))
self.assertRaises(LookupError,
clusto.get_from_pools, 'p1', 'non-existent-pool')
def siblings(self,
parent_filter=None,
parent_kwargs=None,
additional_pools=None,
**kwargs):
"""Return a list of Things that have the same parents as me.
parameters:
parent_filter - a function used to filter out unwanted parents
parent_kwargs - arguments to be passed to self.parents()
additional_pools - a list of additional pools to use as sibling parents
**kwargs - arguments to clusto.get_from_pools()
"""
if parent_kwargs is None:
parent_kwargs = dict(clusto_types=[clusto.drivers.Pool])
parents = self.parents(**parent_kwargs)
if parent_filter:
parents = filter(parent_filter, parents)
if additional_pools:
parents.extend(additional_pools)
return [
s for s in clusto.get_from_pools(
parents, search_children=False, **kwargs) if s != self
]
def run(self, args):
keys = args.keys.split(",")
writer = csv.writer(sys.stdout)
writer.writerow(['name', 'zone'] + keys)
for entity in clusto.get_from_pools(args.pools):
attrs = [entity.name, entity.parents(clusto_types=[EC2Zone])[0].name]
for key in keys:
k, sk = key.split('_', 1)
attrs += [unicode(x).strip() for x in entity.attr_values(key=k, subkey=sk)]
writer.writerow(attrs)
def get_from_pools(self, request):
if not 'pools' in request.params:
return Response(status=400, body='400 Bad Request\nYou must specify at least one pool\n')
pools = request.params['pools'].split(',')
if 'types' in request.params:
clusto_types = request.params['types'].split(',')
else:
clusto_types = None
result = [unclusto(x) for x in clusto.get_from_pools(pools, clusto_types)]
return dumps(request, result)
def get_from_pools(self, request):
if not "pools" in request.params:
return Response(status=400, body="400 Bad Request\nYou must specify at least one pool\n")
pools = request.params["pools"].split(",")
if "types" in request.params:
clusto_types = request.params["types"].split(",")
else:
clusto_types = None
result = [unclusto(x) for x in clusto.get_from_pools(pools, clusto_types)]
return dumps(request, result)
def testGetFromPools(self):
p1 = clusto.get_or_create('p1', Pool)
p2 = clusto.get_or_create('p2', Pool)
s1 = BasicServer('s1')
s2 = BasicServer('s2')
s3 = BasicServer('s3')
s4 = BasicServer('s4')
s5 = BasicServer('s5')
p1.insert(s1)
p1.insert(s2)
p1.insert(s3)
p2.insert(s3)
p2.insert(s4)
p2.insert(s5)
self.assertEqual(sorted([s1, s2, s3]),
sorted(clusto.get_from_pools(pools=[p1],
clusto_types=[BasicServer])))
self.assertEqual(sorted([s3, s4, s5]),
sorted(clusto.get_from_pools(pools=[p2],
clusto_types=[BasicServer])))
self.assertEqual(sorted([s3]),
sorted(clusto.get_from_pools(pools=[p1, p2],
clusto_types=[BasicServer])))
self.assertRaises(LookupError,
clusto.get_from_pools, 'p1', 'non-existent-pool')
# mixing types in get_from_pools should raise a TypeError now
self.assertRaises(TypeError,
clusto.get_from_pools, ['p1', 's1'], 'dont-mix-types')
# lax joining in get_from_entities should return an empty list in this case because s1 has no children
self.assertEqual([], sorted(clusto.get_from_entities(['p1', 's1'])))
def get_from_pools(self, request):
if not 'pools' in request.params:
return Response(
status=400,
body='400 Bad Request\nYou must specify at least one pool\n')
pools = request.params['pools'].split(',')
if 'types' in request.params:
clusto_types = request.params['types'].split(',')
else:
clusto_types = None
result = [
unclusto(x) for x in clusto.get_from_pools(pools, clusto_types)
]
return Response(status=200, body=dumps(request, result))
def run(self, args):
self.debug('Querying for pools %s' % args.pool)
self.debug('Recursive search = %s' % args.recursive)
serverset = clusto.get_from_pools(args.pool,
clusto_types=[drivers.servers.BasicServer],
search_children=args.recursive)
for server in serverset:
if args.names:
print server.name
else:
try:
ip = server.get_ips()
except Exception, e:
self.debug(e)
ip = None
if ip:
print ip[0]
else:
print server.name
def get_from_pools(self, request):
if not 'pools' in request.params:
return Response(status=400, body='400 Bad Request\nYou must specify at least one pool\n')
pools = request.params['pools'].split(',')
if 'types' in request.params:
clusto_types = request.params['types'].split(',')
else:
clusto_types = None
prefetch_attrs = None
if 'prefetch_attrs' in request.params:
# Prefetch all the matching attributes while pulling data
# Prefetch format would like something like
# prefetch_attrs=[{'key': 'disk', 'subkey': 'make'}, {'key': 'system', 'subkey': 'version'}]
prefetch_attrs = loads(request, request.params['prefetch_attrs'])
result = [unclusto(x, prefetch_attrs) for x in clusto.get_from_pools(pools, clusto_types)]
return dumps(request, result)
def get_from_pools(self, request):
if not 'pools' in request.params:
return Response(status=400, body='400 Bad Request\nYou must specify at least one pool\n')
pools = request.params['pools'].split(',')
if 'types' in request.params:
clusto_types = request.params['types'].split(',')
else:
clusto_types = None
prefetch_attrs = None
if 'prefetch_attrs' in request.params:
# Prefetch all the matching attributes while pulling data
# Prefetch format would like something like
# prefetch_attrs=[{'key': 'disk', 'subkey': 'make'}, {'key': 'system', 'subkey': 'version'}]
prefetch_attrs = loads(request, request.params['prefetch_attrs'])
result = [unclusto(x, prefetch_attrs) for x in clusto.get_from_pools(pools, clusto_types)]
return dumps(request, result)
def run(self, args):
if len(args.pool) > 2:
self.warn('Specified more than 2 pools, using only first 2')
self.debug('Querying for pools %s' % args.pool)
self.debug('Recursive search = %s' % args.recursive)
serverset = clusto.get_from_pools(args.pool[:2],
clusto_types=[drivers.servers.BasicServer],
search_children=args.recursive)
for server in serverset:
if args.names:
print server.name
else:
try:
ip = server.get_ips()
except Exception, e:
self.debug(e)
ip = None
if ip:
print ip[0]
else:
print server.name
def siblings(self, parent_filter=None, parent_kwargs=None,
additional_pools=None, **kwargs):
"""Return a list of Things that have the same parents as me.
parameters:
parent_filter - a function used to filter out unwanted parents
parent_kwargs - arguments to be passed to self.parents()
additional_pools - a list of additional pools to use as sibling parents
**kwargs - arguments to clusto.get_from_pools()
"""
if parent_kwargs is None:
parent_kwargs = dict(clusto_types=[clusto.drivers.Pool])
parents = self.parents(**parent_kwargs)
if parent_filter:
parents = filter(parent_filter, parents)
if additional_pools:
parents.extend(additional_pools)
return [s for s in clusto.get_from_pools(parents, search_children=False, **kwargs) if s != self]
def get_from_pools():
"""
One of the main ``clusto`` operations. Parameters:
* Required: at least one ``pool`` parameter
* Optional: one or more ``driver`` parameter to filter out results
* Optional: one or more ``type`` parameter to filter out results
* Optional: a boolean ``children`` parameter to search for children
recursively (True by default)
Examples:
.. code:: bash
$ ${get} ${server_url}/from-pools
"Provide at least one pool to get data from"
HTTP: 412
Content-type: application/json
$ ${get} -H 'Clusto-Page: notanint' -d 'pool=emptypool' ${server_url}/from-pools
"invalid literal for int() with base 10: 'notanint'"
HTTP: 400
Content-type: application/json
$ ${get} -d 'pool=emptypool' ${server_url}/from-pools
[]
HTTP: 200
Content-type: application/json
$ ${get} -d 'pool=singlepool' -d 'pool=multipool' ${server_url}/from-pools
[
"/basicserver/testserver1"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Mode: expanded' -d 'pool=multipool' ${server_url}/from-pools
[
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey1",
"value": "value1"
}
],
"contents": [],
"driver": "basicserver",
"ips": [],
"name": "testserver1",
"parents": [
"/pool/singlepool",
"/pool/multipool"
],
"type": "server"
},
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey2",
"value": "value2"
}
],
"contents": [],
"driver": "basicserver",
"ips": [],
"name": "testserver2",
"parents": [
"/pool/multipool"
],
"type": "server"
}
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Page: 1' -H 'Clusto-Per-Page: 1' -d 'pool=multipool' ${server_url}/from-pools
[
"/basicserver/testserver1"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Page: 1' -H 'Clusto-Per-Page: 100' -d 'pool=multipool' ${server_url}/from-pools
[
"/basicserver/testserver1",
"/basicserver/testserver2"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Page: 100' -H 'Clusto-Per-Page: 100' -d 'pool=multipool' ${server_url}/from-pools
[]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Minify: True' -d 'pool=multipool' ${server_url}/from-pools
["/basicserver/testserver1", "/basicserver/testserver2"]
HTTP: 200
Content-type: application/json
"""
pools = bottle.request.params.getall('pool')
if not pools:
return util.dumps('Provide at least one pool to get data from', 412)
types = bottle.request.params.getall('type')
drivers = bottle.request.params.getall('driver')
children = bottle.request.params.get('children', default=True, type=bool)
mode = bottle.request.headers.get('Clusto-Mode', default='compact')
headers = {}
try:
# Assignments are moved into the try block because of the int casting.
current = int(bottle.request.headers.get('Clusto-Page', default='0'))
per = int(bottle.request.headers.get('Clusto-Per-Page', default='50'))
ents = clusto.get_from_pools(
pools, clusto_types=types, clusto_drivers=drivers, search_children=children
)
results = []
if current:
ents, total = util.page(list(ents), current=current, per=per)
headers['Clusto-Pages'] = total
headers['Clusto-Per-Page'] = per
headers['Clusto-Page'] = current
for ent in ents:
results.append(util.show(ent, mode))
return util.dumps(results, headers=headers)
except ValueError as ve:
return util.dumps('%s' % (ve,), 400)
except TypeError as te:
return util.dumps('%s' % (te,), 409)
except LookupError as le:
return util.dumps('%s' % (le,), 404)
except Exception as e:
return util.dumps('%s' % (e,), 500)
def get_from_pools():
"""
One of the main ``clusto`` operations. Parameters:
* Required: at least one ``pool`` parameter
* Optional: one or more ``driver`` parameter to filter out results
* Optional: one or more ``type`` parameter to filter out results
* Optional: a boolean ``children`` parameter to search for children
recursively (True by default)
Examples:
.. code:: bash
$ ${get} ${server_url}/from-pools
"Provide at least one pool to get data from"
HTTP: 412
Content-type: application/json
$ ${get} -d 'pool=emptypool' ${server_url}/from-pools
[]
HTTP: 200
Content-type: application/json
$ ${get} -d 'pool=singlepool' -d 'pool=multipool' ${server_url}/from-pools
[
"/basicserver/testserver1"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Mode: expanded' -d 'pool=multipool' ${server_url}/from-pools
[
{
"attrs": [],
"contents": [],
"driver": "basicserver",
"name": "testserver1",
"parents": [
"/pool/singlepool",
"/pool/multipool"
]
},
{
"attrs": [],
"contents": [],
"driver": "basicserver",
"name": "testserver2",
"parents": [
"/pool/multipool"
]
}
]
HTTP: 200
Content-type: application/json
"""
pools = bottle.request.params.getall('pool')
if not pools:
return util.dumps('Provide at least one pool to get data from', 412)
types = bottle.request.params.getall('type')
drivers = bottle.request.params.getall('driver')
children = bottle.request.params.get('children', default=True, type=bool)
mode = bottle.request.headers.get('Clusto-Mode', default='compact')
current = int(bottle.request.headers.get('Clusto-Page', default='0'))
per = int(bottle.request.headers.get('Clusto-Per-Page', default='50'))
try:
ents = clusto.get_from_pools(
pools, clusto_types=types, clusto_drivers=drivers, search_children=children
)
results = []
headers = {}
if current:
ents, total = util.page(list(ents), current=current, per=per)
headers = {
'Clusto-Pages': total,
'Clusto-Per-Page': per,
'Clusto-Page': current
}
for ent in ents:
results.append(util.show(ent, mode))
return util.dumps(results, headers=headers)
except TypeError as te:
return util.dumps('%s' % (te,), 409)
except LookupError as le:
return util.dumps('%s' % (le,), 404)
except Exception as e:
return util.dumps('%s' % (e,), 500)
def testGetEntities(self):
d1 = Driver('d1')
dv1 = Device('dv1')
Location('l1')
BasicDatacenter('dc1')
namelist = ['e1', 'e2', 'dv1']
self.assertEqual(sorted([n.name
for n in clusto.get_entities(names=namelist)]),
sorted(namelist))
dl = [Driver]
self.assertEqual(sorted([n.name
for n in clusto.get_entities(clusto_drivers=dl)]),
sorted(['d1','e1','e2','e3']))
tl = [Location, BasicDatacenter]
self.assertEqual(sorted([n.name
for n in clusto.get_entities(clusto_types=tl)]),
sorted(['l1','dc1']))
p1 = Pool('p1')
p2 = Pool('p2')
p3 = Pool('p3')
p4 = Pool('p4')
s1 = BasicServer('s1')
s2 = BasicServer('s2')
s3 = BasicServer('s3')
p1.insert(s1)
p1.insert(s2)
p2.insert(s2)
p2.insert(s3)
p2.insert(d1)
p3.insert(s3)
p3.insert(d1)
p3.insert(s1)
p4.insert(p3)
self.assertEqual(sorted([s2,s3]),
sorted(clusto.get_from_pools(pools=[p2],
clusto_types=[BasicServer])))
self.assertEqual(sorted([s2]),
sorted(clusto.get_from_pools(pools=[p2, 'p1'],
clusto_types=[BasicServer])))
self.assertEqual(sorted([s3]),
sorted(clusto.get_from_pools(pools=['p2', 'p3'],
clusto_types=[BasicServer])))
self.assertEqual(sorted([s1]),
sorted(clusto.get_from_pools(pools=['p4', 'p1'],
clusto_types=[BasicServer])))
def run(self, args):
for entity in clusto.get_from_pools(args.pools):
name = [entity.name] if args.print_name else []
attrs = name + entity.attr_values(key=args.key, subkey=args.subkey)
print ' '.join(attrs)
def testGetEntities(self):
d1 = Driver('d1')
dv1 = Device('dv1')
Location('l1')
BasicDatacenter('dc1')
namelist = ['e1', 'e2', 'dv1']
self.assertEqual(
sorted([n.name for n in clusto.get_entities(names=namelist)]),
sorted(namelist))
dl = [Driver]
self.assertEqual(
sorted([n.name for n in clusto.get_entities(clusto_drivers=dl)]),
sorted(['d1', 'e1', 'e2', 'e3']))
tl = [Location, BasicDatacenter]
self.assertEqual(
sorted([n.name for n in clusto.get_entities(clusto_types=tl)]),
sorted(['l1', 'dc1']))
p1 = Pool('p1')
p2 = Pool('p2')
p3 = Pool('p3')
p4 = Pool('p4')
s1 = BasicServer('s1')
s2 = BasicServer('s2')
s3 = BasicServer('s3')
p1.insert(s1)
p1.insert(s2)
p2.insert(s2)
p2.insert(s3)
p2.insert(d1)
p3.insert(s3)
p3.insert(d1)
p3.insert(s1)
p4.insert(p3)
self.assertEqual(
sorted([s2, s3]),
sorted(
clusto.get_from_pools(pools=[p2], clusto_types=[BasicServer])))
self.assertEqual(
sorted([s2]),
sorted(
clusto.get_from_pools(pools=[p2, 'p1'],
clusto_types=[BasicServer])))
self.assertEqual(
sorted([s3]),
sorted(
clusto.get_from_pools(pools=['p2', 'p3'],
clusto_types=[BasicServer])))
self.assertEqual(
sorted([s1]),
sorted(
clusto.get_from_pools(pools=['p4', 'p1'],
clusto_types=[BasicServer])))
def get_from_pools():
"""
One of the main ``clusto`` operations. Parameters:
* Required: at least one ``pool`` parameter
* Optional: one or more ``driver`` parameter to filter out results
* Optional: one or more ``type`` parameter to filter out results
* Optional: a boolean ``children`` parameter to search for children
recursively (True by default)
Examples:
.. code:: bash
$ ${get} ${server_url}/from-pools
"Provide at least one pool to get data from"
HTTP: 412
Content-type: application/json
$ ${get} -H 'Clusto-Page: notanint' -d 'pool=emptypool' ${server_url}/from-pools
"invalid literal for int() with base 10: 'notanint'"
HTTP: 400
Content-type: application/json
$ ${get} -d 'pool=emptypool' ${server_url}/from-pools
[]
HTTP: 200
Content-type: application/json
$ ${get} -d 'pool=singlepool' -d 'pool=multipool' ${server_url}/from-pools
[
"/basicserver/testserver1"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Mode: expanded' -d 'pool=multipool' ${server_url}/from-pools
[
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey1",
"value": "value1"
}
],
"contents": [],
"driver": "basicserver",
"name": "testserver1",
"parents": [
"/pool/singlepool",
"/pool/multipool"
]
},
{
"attrs": [
{
"datatype": "string",
"key": "key1",
"number": null,
"subkey": "subkey2",
"value": "value2"
}
],
"contents": [],
"driver": "basicserver",
"name": "testserver2",
"parents": [
"/pool/multipool"
]
}
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Page: 1' -H 'Clusto-Per-Page: 1' -d 'pool=multipool' ${server_url}/from-pools
[
"/basicserver/testserver1"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Page: 1' -H 'Clusto-Per-Page: 100' -d 'pool=multipool' ${server_url}/from-pools
[
"/basicserver/testserver1",
"/basicserver/testserver2"
]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Page: 100' -H 'Clusto-Per-Page: 100' -d 'pool=multipool' ${server_url}/from-pools
[]
HTTP: 200
Content-type: application/json
$ ${get} -H 'Clusto-Minify: True' -d 'pool=multipool' ${server_url}/from-pools
["/basicserver/testserver1", "/basicserver/testserver2"]
HTTP: 200
Content-type: application/json
"""
pools = bottle.request.params.getall('pool')
if not pools:
return util.dumps('Provide at least one pool to get data from', 412)
types = bottle.request.params.getall('type')
drivers = bottle.request.params.getall('driver')
children = bottle.request.params.get('children', default=True, type=bool)
mode = bottle.request.headers.get('Clusto-Mode', default='compact')
headers = {
'Clusto-Minify': bottle.request.headers.get('Clusto-Minify', default='False')
}
try:
# Assignments are moved into the try block because of the int casting.
current = int(bottle.request.headers.get('Clusto-Page', default='0'))
per = int(bottle.request.headers.get('Clusto-Per-Page', default='50'))
ents = clusto.get_from_pools(
pools, clusto_types=types, clusto_drivers=drivers, search_children=children
)
results = []
if current:
ents, total = util.page(list(ents), current=current, per=per)
headers['Clusto-Pages'] = total
headers['Clusto-Per-Page'] = per
headers['Clusto-Page'] = current
for ent in ents:
results.append(util.show(ent, mode))
return util.dumps(results, headers=headers)
except ValueError as ve:
return util.dumps('%s' % (ve,), 400)
except TypeError as te:
return util.dumps('%s' % (te,), 409)
except LookupError as le:
return util.dumps('%s' % (le,), 404)
except Exception as e:
return util.dumps('%s' % (e,), 500)