Esempio n. 1
0
 def test_servergroup_crud(self):
     err, msg = Servers.create_servers(self.newsg)
     assert (msg != "")
     assert (err != 1)
     s, err = Servers.list_servers()
     assert (err == 200)
     print s
Esempio n. 2
0
def servers(request):
    servers = [{
        'backends': make_backends(backends_count=1, group_id=g)
    } for g in 1, 2]
    _servers = Servers(servers=servers, isolated=True, path='isolated_servers')

    yield _servers

    _servers.stop()
def servers(request):
    groups = [int(g) for g in request.config.option.groups.split(',')]

    _servers = Servers(without_cocaine=True,
                       servers=make_servers(groups, nodes_count=2, backends_count=1),
                       isolated=True,
                       path='special_servers')

    yield _servers

    _servers.stop()
Esempio n. 4
0
def servers(request):
    groups = [int(g) for g in request.config.option.groups.split(',')]

    _servers = Servers(servers=make_servers(groups,
                                            nodes_count=2,
                                            backends_count=1),
                       isolated=True,
                       path='special_servers')

    yield _servers

    _servers.stop()
Esempio n. 5
0
def server(request):
    groups = [int(g) for g in request.config.option.groups.split(',')]

    servers = Servers(groups=groups,
                      without_cocaine=request.config.option.without_cocaine,
                      nodes_count=int(request.config.option.nodes_count),
                      backends_count=int(request.config.option.backends_count))

    request.config.option.remotes = servers.remotes
    request.config.option.monitors = servers.monitors

    yield servers

    servers.stop()
Esempio n. 6
0
def server(request):
    groups = [int(g) for g in request.config.option.groups.split(',')]

    servers = Servers(groups=groups,
                      without_cocaine=request.config.option.without_cocaine,
                      nodes_count=int(request.config.option.nodes_count),
                      backends_count=int(request.config.option.backends_count))

    request.config.option.remotes = servers.remotes
    request.config.option.monitors = servers.monitors

    yield servers

    servers.stop()
Esempio n. 7
0
def servers(request):
    '''
    Creates elliptics server nodes to work against.
    Returns node ensemble configuration.
    '''
    groups = [int(g) for g in request.config.option.groups.split(',')]

    _servers = Servers(servers=make_servers(groups, int(request.config.option.nodes_count),
                                            int(request.config.option.backends_count)))

    request.config.option.remotes = _servers.remotes
    request.config.option.monitors = _servers.monitors

    yield _servers

    _servers.stop()
Esempio n. 8
0
def servers(request):
    '''
    Creates elliptics server nodes to work against.
    Returns node ensemble configuration.
    '''
    groups = [int(g) for g in request.config.option.groups.split(',')]

    _servers = Servers(without_cocaine=request.config.option.without_cocaine,
                       servers=make_servers(groups, int(request.config.option.nodes_count),
                                            int(request.config.option.backends_count)))

    request.config.option.remotes = _servers.remotes
    request.config.option.monitors = _servers.monitors

    yield _servers

    _servers.stop()
Esempio n. 9
0
def run_server(group_id,
               records_in_blob,
               path,
               datasort_dir=None,
               single_pass_threshold=None):
    """Run server under contextmanager, so it will be killed at exit.

    Server will be run with one backend in @group, all other parameters will be placed into config

    Args:
        group_id(int): id of the group which should be served by the backend
        records_in_blob(int): limit for a blob in the backend
        path(str or None): where backend should store its blobs (if None use default provided by test env)
        datasort_dir(str or None): where backend should store temporary chunks during defrag
            (if None use temporary directory at @path)

    Yields:
        Servers: configured server with one backend and specified paramters.
    """
    config = [{
        'backends': [{
            'group': group_id,
            'records_in_blob': records_in_blob,
            'periodic_timeout': 1,
        }]
    }]

    # set datasort_dir only if it is provided
    if datasort_dir is not None:
        config[0]['backends'][0]['datasort_dir'] = datasort_dir

    if single_pass_threshold is not None:
        config[0]['backends'][0][
            'single_pass_file_size_threshold'] = single_pass_threshold
    server = Servers(servers=config, path=path)

    try:
        yield server
    finally:
        server.stop()
Esempio n. 10
0
def servers(request):
    groups = [int(g) for g in request.config.option.groups.split(',')]

    servers = Servers(groups=groups,
                      without_cocaine=True,
                      nodes_count=2,
                      backends_count=1,
                      isolated=True,
                      path='special_servers')

    def fin():
        servers.stop()

    request.addfinalizer(fin)

    return servers
Esempio n. 11
0
def server(request):
    groups = [int(g) for g in request.config.option.groups.split(',')]

    servers = Servers(groups=groups,
                      without_cocaine=request.config.option.without_cocaine,
                      nodes_count=int(request.config.option.nodes_count),
                      backends_count=int(request.config.option.backends_count))

    request.config.option.remotes = servers.remotes
    request.config.option.monitors = servers.monitors

    def fin():
        servers.stop()
    request.addfinalizer(fin)

    return servers
Esempio n. 12
0
 def __init__(self, servers):
     self._servers = Servers(servers)
Esempio n. 13
0
class Planner:
    def __init__(self, servers):
        self._servers = Servers(servers)

    @memo
    @listify
    def _plans_default(self, node, servers=None):
        for inputplans in product(*map(self.get_plans, node.inputs)):
            profile = Profile.build(node, inputplans)
            for server in servers or self._servers:
                authorized = self._servers.authorize(profile, server.id)
                if authorized:
                    yield self.makeplan(node, inputplans, server, authorized)

    @memo
    @listify
    def _plans_tablescan(self, node):
        tbl = node.get('table')[1]
        profile = Profile.build(node, None)
        for server in [s for s in self._servers if tbl in s.tables]:
            authorized = self._servers.authorize(profile, server.id)
            if authorized:
                yield self.makeplan(node, [], server, authorized)

    @memo
    def _plans_enumerable(self, node):
        return self._plans_default(node, [self._servers['CL']])

    @memo
    def _planfn(self, relOp):
        planfn = {
            'JdbcToEnumerableConverter': self._plans_enumerable,
            'JdbcTableScan': self._plans_tablescan
        }
        try:
            return planfn[relOp]
        except:
            return self._plans_default

    @memo
    def get_plans(self, node):
        plans = self._planfn(node.get('relOp'))(node)
        logging.debug('#%d plans for node %s' % (len(plans), node.get('id')))
        return plans

    def get_best_plan(self, node):
        plans = self.get_plans(node)
        return min(plans, key=methodcaller('totalcost')) if plans else None

    def makecost(self, node, inputplans, server):
        nodecost = node.get('cost')
        costs = Counter({
            server.id:
            server.cpu * nodecost['cpu'] + server.io * nodecost['io']
        })
        for plan in inputplans:
            inserver = self._servers[plan.serverid()]
            if inserver is not server:
                costs['%s->*' % inserver.id] += plan.rows * inserver.netout
                costs['*->%s' % server.id] += plan.rows * server.netin
        return costs

    def makeplan(self, node, inputplans, server, profile):
        root = node.get('id')
        rows = node.get('rowCount')
        executers = dict(
            chain.from_iterable(plan.executers.iteritems()
                                for plan in inputplans))
        executers[root] = server.id
        profiles = dict(
            chain.from_iterable(plan.profiles.iteritems()
                                for plan in inputplans))
        profiles[root] = profile
        costs = sum((plan.costs for plan in inputplans),
                    self.makecost(node, inputplans, server))
        return Plan(root, profile, executers, costs, profiles, rows, node)
Esempio n. 14
0
 def __init__(self, servers):
     self._servers = Servers(servers)
Esempio n. 15
0
class Planner:

    def __init__(self, servers):
        self._servers = Servers(servers)

    @memo
    @listify
    def _plans_default(self, node, servers=None):
        for inputplans in product(*map(self.get_plans, node.inputs)):
            profile = Profile.build(node, inputplans)
            for server in servers or self._servers:
                authorized = self._servers.authorize(profile, server.id)
                if authorized:
                    yield self.makeplan(node, inputplans, server, authorized)

    @memo
    @listify
    def _plans_tablescan(self, node):
        tbl = node.get('table')[1]
        profile = Profile.build(node, None)
        for server in [s for s in self._servers if tbl in s.tables]:
            authorized = self._servers.authorize(profile, server.id)
            if authorized:
                yield self.makeplan(node, [], server, authorized)

    @memo
    def _plans_enumerable(self, node):
        return self._plans_default(node, [self._servers['CL']])

    @memo
    def _planfn(self, relOp):
        planfn = { 'JdbcToEnumerableConverter': self._plans_enumerable,
                  'JdbcTableScan': self._plans_tablescan }
        try:
            return planfn[relOp]
        except:
            return self._plans_default

    @memo
    def get_plans(self, node):
        plans = self._planfn(node.get('relOp'))(node)
        logging.debug( '#%d plans for node %s' % (len(plans), node.get('id')))
        return plans

    def get_best_plan(self, node):
        plans = self.get_plans(node)
        return min(plans, key=methodcaller('totalcost')) if plans else None

    def makecost(self, node, inputplans, server):
        nodecost = node.get('cost')
        costs = Counter({server.id:
            server.cpu * nodecost['cpu'] + server.io * nodecost['io']})
        for plan in inputplans:
            inserver = self._servers[plan.serverid()]
            if inserver is not server:
                costs['%s->*' % inserver.id] += plan.rows * inserver.netout
                costs['*->%s' % server.id] += plan.rows * server.netin
        return costs

    def makeplan(self, node, inputplans, server, profile):
        root = node.get('id')
        rows = node.get('rowCount')
        executers = dict(chain.from_iterable(
            plan.executers.iteritems() for plan in inputplans))
        executers[root] = server.id
        profiles = dict(chain.from_iterable(
            plan.profiles.iteritems() for plan in inputplans))
        profiles[root] = profile
        costs = sum((plan.costs for plan in inputplans),
            self.makecost(node, inputplans, server))
        return Plan(root, profile, executers, costs, profiles, rows, node)