def test_servergroup_crud(self):
err, msg = Servers.create_servers(self.newsg)
assert (msg != "")
assert (err != 1)
s, err = Servers.list_servers()
assert (err == 200)
print s
def servers(request):
servers = [{
'backends': make_backends(backends_count=1, group_id=g)
} for g in 1, 2]
_servers = Servers(servers=servers, isolated=True, path='isolated_servers')
yield _servers
_servers.stop()
def servers(request):
groups = [int(g) for g in request.config.option.groups.split(',')]
_servers = Servers(without_cocaine=True,
servers=make_servers(groups, nodes_count=2, backends_count=1),
isolated=True,
path='special_servers')
yield _servers
_servers.stop()
def servers(request):
groups = [int(g) for g in request.config.option.groups.split(',')]
_servers = Servers(servers=make_servers(groups,
nodes_count=2,
backends_count=1),
isolated=True,
path='special_servers')
yield _servers
_servers.stop()
def server(request):
groups = [int(g) for g in request.config.option.groups.split(',')]
servers = Servers(groups=groups,
without_cocaine=request.config.option.without_cocaine,
nodes_count=int(request.config.option.nodes_count),
backends_count=int(request.config.option.backends_count))
request.config.option.remotes = servers.remotes
request.config.option.monitors = servers.monitors
yield servers
servers.stop()
def server(request):
groups = [int(g) for g in request.config.option.groups.split(',')]
servers = Servers(groups=groups,
without_cocaine=request.config.option.without_cocaine,
nodes_count=int(request.config.option.nodes_count),
backends_count=int(request.config.option.backends_count))
request.config.option.remotes = servers.remotes
request.config.option.monitors = servers.monitors
yield servers
servers.stop()
def servers(request):
'''
Creates elliptics server nodes to work against.
Returns node ensemble configuration.
'''
groups = [int(g) for g in request.config.option.groups.split(',')]
_servers = Servers(servers=make_servers(groups, int(request.config.option.nodes_count),
int(request.config.option.backends_count)))
request.config.option.remotes = _servers.remotes
request.config.option.monitors = _servers.monitors
yield _servers
_servers.stop()
def servers(request):
'''
Creates elliptics server nodes to work against.
Returns node ensemble configuration.
'''
groups = [int(g) for g in request.config.option.groups.split(',')]
_servers = Servers(without_cocaine=request.config.option.without_cocaine,
servers=make_servers(groups, int(request.config.option.nodes_count),
int(request.config.option.backends_count)))
request.config.option.remotes = _servers.remotes
request.config.option.monitors = _servers.monitors
yield _servers
_servers.stop()
def run_server(group_id,
records_in_blob,
path,
datasort_dir=None,
single_pass_threshold=None):
"""Run server under contextmanager, so it will be killed at exit.
Server will be run with one backend in @group, all other parameters will be placed into config
Args:
group_id(int): id of the group which should be served by the backend
records_in_blob(int): limit for a blob in the backend
path(str or None): where backend should store its blobs (if None use default provided by test env)
datasort_dir(str or None): where backend should store temporary chunks during defrag
(if None use temporary directory at @path)
Yields:
Servers: configured server with one backend and specified paramters.
"""
config = [{
'backends': [{
'group': group_id,
'records_in_blob': records_in_blob,
'periodic_timeout': 1,
}]
}]
# set datasort_dir only if it is provided
if datasort_dir is not None:
config[0]['backends'][0]['datasort_dir'] = datasort_dir
if single_pass_threshold is not None:
config[0]['backends'][0][
'single_pass_file_size_threshold'] = single_pass_threshold
server = Servers(servers=config, path=path)
try:
yield server
finally:
server.stop()
def servers(request):
groups = [int(g) for g in request.config.option.groups.split(',')]
servers = Servers(groups=groups,
without_cocaine=True,
nodes_count=2,
backends_count=1,
isolated=True,
path='special_servers')
def fin():
servers.stop()
request.addfinalizer(fin)
return servers
def server(request):
groups = [int(g) for g in request.config.option.groups.split(',')]
servers = Servers(groups=groups,
without_cocaine=request.config.option.without_cocaine,
nodes_count=int(request.config.option.nodes_count),
backends_count=int(request.config.option.backends_count))
request.config.option.remotes = servers.remotes
request.config.option.monitors = servers.monitors
def fin():
servers.stop()
request.addfinalizer(fin)
return servers
def __init__(self, servers):
self._servers = Servers(servers)
class Planner:
def __init__(self, servers):
self._servers = Servers(servers)
@memo
@listify
def _plans_default(self, node, servers=None):
for inputplans in product(*map(self.get_plans, node.inputs)):
profile = Profile.build(node, inputplans)
for server in servers or self._servers:
authorized = self._servers.authorize(profile, server.id)
if authorized:
yield self.makeplan(node, inputplans, server, authorized)
@memo
@listify
def _plans_tablescan(self, node):
tbl = node.get('table')[1]
profile = Profile.build(node, None)
for server in [s for s in self._servers if tbl in s.tables]:
authorized = self._servers.authorize(profile, server.id)
if authorized:
yield self.makeplan(node, [], server, authorized)
@memo
def _plans_enumerable(self, node):
return self._plans_default(node, [self._servers['CL']])
@memo
def _planfn(self, relOp):
planfn = {
'JdbcToEnumerableConverter': self._plans_enumerable,
'JdbcTableScan': self._plans_tablescan
}
try:
return planfn[relOp]
except:
return self._plans_default
@memo
def get_plans(self, node):
plans = self._planfn(node.get('relOp'))(node)
logging.debug('#%d plans for node %s' % (len(plans), node.get('id')))
return plans
def get_best_plan(self, node):
plans = self.get_plans(node)
return min(plans, key=methodcaller('totalcost')) if plans else None
def makecost(self, node, inputplans, server):
nodecost = node.get('cost')
costs = Counter({
server.id:
server.cpu * nodecost['cpu'] + server.io * nodecost['io']
})
for plan in inputplans:
inserver = self._servers[plan.serverid()]
if inserver is not server:
costs['%s->*' % inserver.id] += plan.rows * inserver.netout
costs['*->%s' % server.id] += plan.rows * server.netin
return costs
def makeplan(self, node, inputplans, server, profile):
root = node.get('id')
rows = node.get('rowCount')
executers = dict(
chain.from_iterable(plan.executers.iteritems()
for plan in inputplans))
executers[root] = server.id
profiles = dict(
chain.from_iterable(plan.profiles.iteritems()
for plan in inputplans))
profiles[root] = profile
costs = sum((plan.costs for plan in inputplans),
self.makecost(node, inputplans, server))
return Plan(root, profile, executers, costs, profiles, rows, node)
def __init__(self, servers):
self._servers = Servers(servers)
class Planner:
def __init__(self, servers):
self._servers = Servers(servers)
@memo
@listify
def _plans_default(self, node, servers=None):
for inputplans in product(*map(self.get_plans, node.inputs)):
profile = Profile.build(node, inputplans)
for server in servers or self._servers:
authorized = self._servers.authorize(profile, server.id)
if authorized:
yield self.makeplan(node, inputplans, server, authorized)
@memo
@listify
def _plans_tablescan(self, node):
tbl = node.get('table')[1]
profile = Profile.build(node, None)
for server in [s for s in self._servers if tbl in s.tables]:
authorized = self._servers.authorize(profile, server.id)
if authorized:
yield self.makeplan(node, [], server, authorized)
@memo
def _plans_enumerable(self, node):
return self._plans_default(node, [self._servers['CL']])
@memo
def _planfn(self, relOp):
planfn = { 'JdbcToEnumerableConverter': self._plans_enumerable,
'JdbcTableScan': self._plans_tablescan }
try:
return planfn[relOp]
except:
return self._plans_default
@memo
def get_plans(self, node):
plans = self._planfn(node.get('relOp'))(node)
logging.debug( '#%d plans for node %s' % (len(plans), node.get('id')))
return plans
def get_best_plan(self, node):
plans = self.get_plans(node)
return min(plans, key=methodcaller('totalcost')) if plans else None
def makecost(self, node, inputplans, server):
nodecost = node.get('cost')
costs = Counter({server.id:
server.cpu * nodecost['cpu'] + server.io * nodecost['io']})
for plan in inputplans:
inserver = self._servers[plan.serverid()]
if inserver is not server:
costs['%s->*' % inserver.id] += plan.rows * inserver.netout
costs['*->%s' % server.id] += plan.rows * server.netin
return costs
def makeplan(self, node, inputplans, server, profile):
root = node.get('id')
rows = node.get('rowCount')
executers = dict(chain.from_iterable(
plan.executers.iteritems() for plan in inputplans))
executers[root] = server.id
profiles = dict(chain.from_iterable(
plan.profiles.iteritems() for plan in inputplans))
profiles[root] = profile
costs = sum((plan.costs for plan in inputplans),
self.makecost(node, inputplans, server))
return Plan(root, profile, executers, costs, profiles, rows, node)