def test_servergroup_crud(self): err, msg = Servers.create_servers(self.newsg) assert (msg != "") assert (err != 1) s, err = Servers.list_servers() assert (err == 200) print s
def servers(request): servers = [{ 'backends': make_backends(backends_count=1, group_id=g) } for g in 1, 2] _servers = Servers(servers=servers, isolated=True, path='isolated_servers') yield _servers _servers.stop()
def servers(request): groups = [int(g) for g in request.config.option.groups.split(',')] _servers = Servers(without_cocaine=True, servers=make_servers(groups, nodes_count=2, backends_count=1), isolated=True, path='special_servers') yield _servers _servers.stop()
def servers(request): groups = [int(g) for g in request.config.option.groups.split(',')] _servers = Servers(servers=make_servers(groups, nodes_count=2, backends_count=1), isolated=True, path='special_servers') yield _servers _servers.stop()
def server(request): groups = [int(g) for g in request.config.option.groups.split(',')] servers = Servers(groups=groups, without_cocaine=request.config.option.without_cocaine, nodes_count=int(request.config.option.nodes_count), backends_count=int(request.config.option.backends_count)) request.config.option.remotes = servers.remotes request.config.option.monitors = servers.monitors yield servers servers.stop()
def servers(request): ''' Creates elliptics server nodes to work against. Returns node ensemble configuration. ''' groups = [int(g) for g in request.config.option.groups.split(',')] _servers = Servers(servers=make_servers(groups, int(request.config.option.nodes_count), int(request.config.option.backends_count))) request.config.option.remotes = _servers.remotes request.config.option.monitors = _servers.monitors yield _servers _servers.stop()
def servers(request): ''' Creates elliptics server nodes to work against. Returns node ensemble configuration. ''' groups = [int(g) for g in request.config.option.groups.split(',')] _servers = Servers(without_cocaine=request.config.option.without_cocaine, servers=make_servers(groups, int(request.config.option.nodes_count), int(request.config.option.backends_count))) request.config.option.remotes = _servers.remotes request.config.option.monitors = _servers.monitors yield _servers _servers.stop()
def run_server(group_id, records_in_blob, path, datasort_dir=None, single_pass_threshold=None): """Run server under contextmanager, so it will be killed at exit. Server will be run with one backend in @group, all other parameters will be placed into config Args: group_id(int): id of the group which should be served by the backend records_in_blob(int): limit for a blob in the backend path(str or None): where backend should store its blobs (if None use default provided by test env) datasort_dir(str or None): where backend should store temporary chunks during defrag (if None use temporary directory at @path) Yields: Servers: configured server with one backend and specified paramters. """ config = [{ 'backends': [{ 'group': group_id, 'records_in_blob': records_in_blob, 'periodic_timeout': 1, }] }] # set datasort_dir only if it is provided if datasort_dir is not None: config[0]['backends'][0]['datasort_dir'] = datasort_dir if single_pass_threshold is not None: config[0]['backends'][0][ 'single_pass_file_size_threshold'] = single_pass_threshold server = Servers(servers=config, path=path) try: yield server finally: server.stop()
def servers(request): groups = [int(g) for g in request.config.option.groups.split(',')] servers = Servers(groups=groups, without_cocaine=True, nodes_count=2, backends_count=1, isolated=True, path='special_servers') def fin(): servers.stop() request.addfinalizer(fin) return servers
def server(request): groups = [int(g) for g in request.config.option.groups.split(',')] servers = Servers(groups=groups, without_cocaine=request.config.option.without_cocaine, nodes_count=int(request.config.option.nodes_count), backends_count=int(request.config.option.backends_count)) request.config.option.remotes = servers.remotes request.config.option.monitors = servers.monitors def fin(): servers.stop() request.addfinalizer(fin) return servers
def __init__(self, servers): self._servers = Servers(servers)
class Planner: def __init__(self, servers): self._servers = Servers(servers) @memo @listify def _plans_default(self, node, servers=None): for inputplans in product(*map(self.get_plans, node.inputs)): profile = Profile.build(node, inputplans) for server in servers or self._servers: authorized = self._servers.authorize(profile, server.id) if authorized: yield self.makeplan(node, inputplans, server, authorized) @memo @listify def _plans_tablescan(self, node): tbl = node.get('table')[1] profile = Profile.build(node, None) for server in [s for s in self._servers if tbl in s.tables]: authorized = self._servers.authorize(profile, server.id) if authorized: yield self.makeplan(node, [], server, authorized) @memo def _plans_enumerable(self, node): return self._plans_default(node, [self._servers['CL']]) @memo def _planfn(self, relOp): planfn = { 'JdbcToEnumerableConverter': self._plans_enumerable, 'JdbcTableScan': self._plans_tablescan } try: return planfn[relOp] except: return self._plans_default @memo def get_plans(self, node): plans = self._planfn(node.get('relOp'))(node) logging.debug('#%d plans for node %s' % (len(plans), node.get('id'))) return plans def get_best_plan(self, node): plans = self.get_plans(node) return min(plans, key=methodcaller('totalcost')) if plans else None def makecost(self, node, inputplans, server): nodecost = node.get('cost') costs = Counter({ server.id: server.cpu * nodecost['cpu'] + server.io * nodecost['io'] }) for plan in inputplans: inserver = self._servers[plan.serverid()] if inserver is not server: costs['%s->*' % inserver.id] += plan.rows * inserver.netout costs['*->%s' % server.id] += plan.rows * server.netin return costs def makeplan(self, node, inputplans, server, profile): root = node.get('id') rows = node.get('rowCount') executers = dict( chain.from_iterable(plan.executers.iteritems() for plan in inputplans)) executers[root] = server.id profiles = dict( chain.from_iterable(plan.profiles.iteritems() for plan in inputplans)) profiles[root] = profile costs = sum((plan.costs for plan in inputplans), self.makecost(node, inputplans, server)) return Plan(root, profile, executers, costs, profiles, rows, node)
class Planner: def __init__(self, servers): self._servers = Servers(servers) @memo @listify def _plans_default(self, node, servers=None): for inputplans in product(*map(self.get_plans, node.inputs)): profile = Profile.build(node, inputplans) for server in servers or self._servers: authorized = self._servers.authorize(profile, server.id) if authorized: yield self.makeplan(node, inputplans, server, authorized) @memo @listify def _plans_tablescan(self, node): tbl = node.get('table')[1] profile = Profile.build(node, None) for server in [s for s in self._servers if tbl in s.tables]: authorized = self._servers.authorize(profile, server.id) if authorized: yield self.makeplan(node, [], server, authorized) @memo def _plans_enumerable(self, node): return self._plans_default(node, [self._servers['CL']]) @memo def _planfn(self, relOp): planfn = { 'JdbcToEnumerableConverter': self._plans_enumerable, 'JdbcTableScan': self._plans_tablescan } try: return planfn[relOp] except: return self._plans_default @memo def get_plans(self, node): plans = self._planfn(node.get('relOp'))(node) logging.debug( '#%d plans for node %s' % (len(plans), node.get('id'))) return plans def get_best_plan(self, node): plans = self.get_plans(node) return min(plans, key=methodcaller('totalcost')) if plans else None def makecost(self, node, inputplans, server): nodecost = node.get('cost') costs = Counter({server.id: server.cpu * nodecost['cpu'] + server.io * nodecost['io']}) for plan in inputplans: inserver = self._servers[plan.serverid()] if inserver is not server: costs['%s->*' % inserver.id] += plan.rows * inserver.netout costs['*->%s' % server.id] += plan.rows * server.netin return costs def makeplan(self, node, inputplans, server, profile): root = node.get('id') rows = node.get('rowCount') executers = dict(chain.from_iterable( plan.executers.iteritems() for plan in inputplans)) executers[root] = server.id profiles = dict(chain.from_iterable( plan.profiles.iteritems() for plan in inputplans)) profiles[root] = profile costs = sum((plan.costs for plan in inputplans), self.makecost(node, inputplans, server)) return Plan(root, profile, executers, costs, profiles, rows, node)