Example #1
0
def connect(endpoints, groups, **kw):
    remotes = []
    for r in endpoints:
        remotes.append(elliptics.Address.from_host_port_family(r))

    def rename(kw, old, new):
        if old in kw:
            kw[new] = kw.pop(old)

    # drop impedeing attrs, just in case
    kw.pop('elog', None)
    kw.pop('cfg', None)
    kw.pop('remotes', None)
    # rename good names to required bad ones
    rename(kw, 'logfile', 'log_file')
    rename(kw, 'loglevel', 'log_level')

    n = elliptics.create_node(**kw)
    n.add_remotes(remotes)

    s = elliptics.Session(n)
    s.add_groups(groups)

#    return PassthroughWrapper(n, s)
    return s
Example #2
0
def connect(endpoints, groups, **kw):
    remotes = []
    for r in endpoints:
        remotes.append(elliptics.Address.from_host_port_family(r))

    def rename(kw, old, new):
        if old in kw:
            kw[new] = kw.pop(old)

    # drop impeding attrs, just in case
    kw.pop('elog', None)
    kw.pop('cfg', None)
    kw.pop('remotes', None)
    # rename good names to required bad ones
    rename(kw, 'logfile', 'log_file')
    rename(kw, 'loglevel', 'log_level')

    n = elliptics.create_node(**kw)
    n.add_remotes(remotes)

    s = elliptics.Session(n)
    s.add_groups(groups)

#    return PassthroughWrapper(n, s)
    return s
Example #3
0
def connect(endpoints, groups, **kw):
    remotes = []
    for r in endpoints:
        parts = r.split(":")
        remotes.append((parts[0], int(parts[1])))

    def rename(kw, old, new):
        if old in kw:
            kw[new] = kw.pop(old)

    # drop impedeing attrs, just in case
    kw.pop('elog', None)
    kw.pop('cfg', None)
    kw.pop('remotes', None)
    # rename good names to required bad ones
    rename(kw, 'logfile', 'log_file')
    rename(kw, 'loglevel', 'log_level')

    n = elliptics.create_node(**kw)

    for r in remotes:
        try:
            n.add_remote(r[0], r[1])
        except Exception:
            pass

    s = elliptics.Session(n)
    s.add_groups(groups)

#    return PassthroughWrapper(n, s)
    return s
Example #4
0
def connect(endpoints, groups, **kw):
	remotes = []
	for r in endpoints:
		parts = r.split(":")
		remotes.append((parts[0], int(parts[1])))

	def rename(new, old):
		if old in kw:
			kw[new] = kw.pop(old)

	kw.pop('elog', None)
	kw.pop('cfg', None)
	kw.pop('remotes', None)
	rename('log_file', 'logfile')
	rename('log_level', 'loglevel')

	n = elliptics.create_node(**kw)

#	def create_node(**kw):
#		log = elliptics.Logger(kw.get('logfile', '/dev/stderr'), kw.get('loglevel', 1))
#		config = elliptics.Config()
#		config.config.wait_timeout = kw.get('wait-timeout', 60)
#		return elliptics.Node(log, config)
#	n = create_node(**kw)

	for r in remotes:
		try:
			n.add_remote(r[0], r[1])
		except Exception as e:
			pass

	s = elliptics.Session(n)
	s.add_groups(groups)
	#XXX: Is it time to drop PassthroughWrapper binder?
	return PassthroughWrapper(n, s)
Example #5
0
    def upload_app(self):
        if self.without_cocaine:
            return
        import subprocess
        self.__create_manifest__()
        self.__create_profile__()

        app = subprocess.Popen(['cocaine-tool', 'app', 'upload',
                                '--manifest', self.name + '/dnet_cpp_srw_test_app.manifest',
                                '--package', '../dnet_cpp_srw_test_app.tar',
                                '--name', 'dnet_cpp_srw_test_app',
                                '--host={0}'.format(self.addr),
                                '--port={0}'.format(self.locator_port)])
        app.wait()
        assert(app.returncode == 0)

        profile = subprocess.Popen(['cocaine-tool', 'profile', 'upload',
                                    '--profile', self.name + '/dnet_cpp_srw_test_app.profile',
                                    '--name', 'dnet_cpp_srw_test_app',
                                    '--host={0}'.format(self.addr),
                                    '--port={0}'.format(self.locator_port)])
        profile.wait()
        assert(profile.returncode == 0)

        import elliptics

        n = elliptics.create_node(remotes=[self.get_addr()], log_level=self.log_level)
        s = elliptics.Session(n)
        s.groups = [self.group]
        s.cflags = elliptics.command_flags.nolock
        s.set_filter(elliptics.filters.all_with_ack)
        s.exec_(id=None, event='dnet_cpp_srw_test_app@start-task').wait()
Example #6
0
def prepare_session(testdir, remote, group_id):
    """Initialize and return session for specified remote

    Args:
        testdir: fixture for test's temporary dirs/files
        remote(str): address of the node
        group_id(int): id of the group to work with

    Returns:
        elliptics.newapi.Session: session configured for working with the backend
    """
    node = elliptics.create_node(log_file=str(testdir.join("client.log")),
                                 log_level=elliptics.log_level.debug,
                                 remotes=[remote])

    session = elliptics.newapi.Session(node)
    session.groups = [group_id]
    return session
Example #7
0
def connect(endpoints, groups, **kw):
    def rename(kw, old, new):
        if old in kw:
            kw[new] = kw.pop(old)

    # drop impedeing attrs, just in case
    kw.pop('elog', None)
    kw.pop('cfg', None)
    kw.pop('remotes', None)
    # rename good names to required bad ones
    rename(kw, 'logfile', 'log_file')
    rename(kw, 'loglevel', 'log_level')

    n = elliptics.create_node(**kw)
    n.add_remote(endpoints)

    s = elliptics.Session(n)
    s.add_groups(groups)

#    return PassthroughWrapper(n, s)
    return s
Example #8
0
def process_uncommitted(ctx, results):
    '''
    Removes uncommitted keys. If a key has any committed replicas, then this key is
    appended to the file containing committed keys.
    If an uncommitted key's replica hasn't exceeded prepare timeout, then skip recovering of the key,
    because the key is under writing and can be committed in the nearest future.
    '''
    if ctx.dry_run or ctx.safe:
        return

    node = elliptics.create_node(log_file=ctx.log_file,
                                 log_level=int(ctx.log_level),
                                 log_watched=True,
                                 wait_timeout=ctx.wait_timeout,
                                 flags=elliptics.config_flags.no_route_list,
                                 net_thread_num=1,
                                 io_thread_num=1,
                                 remotes=ctx.remotes)
    session = elliptics.newapi.Session(node)
    session.trace_id = ctx.trace_id
    session.exceptions_policy = elliptics.exceptions_policy.no_exceptions
    session.set_filter(elliptics.filters.all_final)
    session.ioflags |= elliptics.io_flags.cas_timestamp
    session.timestamp = ctx.prepare_timeout

    stats = ctx.stats['recover']
    stats_cmd = ctx.stats['commands']

    for r in results:
        with open(r.filename, 'ab') as f:
            for _, batch in itertools.groupby(
                    enumerate(load_key_data(r.uncommitted_filename)),
                    key=lambda x: x[0] / ctx.batch_size):
                batch = [item[1] for item in batch]
                tasks = []
                statuses = {}  # (key, group_id) -> status

                for key, key_infos in batch:
                    for info in key_infos:
                        if info.flags & elliptics.record_flags.uncommitted:
                            if info.group_id in ctx.ro_groups:
                                stats.counter(
                                    'skip_remove_uncommitted_key_from_ro_group',
                                    1)
                                statuses[(key, info.group_id
                                          )] = 0  # mark status as successful
                                continue
                            tasks.append((key, info.group_id, info.size))

                for attempt in range(ctx.attempts):
                    if not tasks:
                        break

                    if attempt > 0:
                        stats.counter('remove_retries', len(tasks))

                    batch_sizes = defaultdict(int)  # group_id -> batch_size
                    for _, group_id, key_size in tasks:
                        batch_sizes[group_id] += key_size

                    timeouts = {
                        group_id: max(60, batch_size / ctx.data_flow_rate)
                        for group_id, batch_size in batch_sizes.iteritems()
                    }

                    responses = []
                    for key, group_id, _ in tasks:
                        session.groups = [group_id]
                        session.timeout = timeouts[group_id]
                        responses.append(session.remove(key))

                    failed_tasks = []
                    for i, r in enumerate(responses):
                        key, group_id, _ = tasks[i]
                        status = r.get()[0].status
                        log.info(
                            'Removed uncommitted key: %s, group: %s, status: %s, attempts: %s/%s',
                            key, group_id, status, attempt, ctx.attempts)
                        statuses[(key, group_id)] = status

                        if status == 0:
                            stats.counter('removed_uncommitted_keys', 1)
                        else:
                            stats_cmd.counter('remove.{0}'.format(status), 1)

                        if status not in (0, -errno.ENOENT, -errno.EBADFD):
                            failed_tasks.append(tasks[i])
                    tasks = failed_tasks

                for key, key_infos in batch:
                    # Filter uncommitted replicas, then append a key to the 'merged' file for recovery.
                    # If an uncommitted replica hasn't exceeded prepare timeout,
                    # then removal status is EBADFD and this key must be skipped.
                    infos = []
                    for info in key_infos:
                        if info.flags & elliptics.record_flags.uncommitted:
                            status = statuses[(key, info.group_id)]
                            if status == -errno.EBADFD:
                                stats.counter('skipped_uncommitted_keys', 1)
                                infos = None
                                break
                        else:
                            infos.append(info)

                    if infos:
                        dump_key_data((key, infos), f)