Beispiel #1
0
def run(protocol, csvfile, step_duration, step_distance):
    shared.seed_random(42)

    node_count = 50
    state = topology.create_nodes(node_count)
    mobility.randomize_positions(state, xy_range=1000)
    mobility.connect_range(state, max_links=150)

    # create network and start routing software
    network.apply(state, link_command=get_tc_command, remotes=remotes)
    software.start(protocol)

    test_beg_ms = shared.millis()
    for n in range(0, 30):
        print(f'{protocol}: iteration {n}')

        #with open(f'graph-{step_duration}-{step_distance}-{n:03d}.json', 'w+') as file:
        #	json.dump(state, file, indent='  ')

        # connect nodes range
        wait_beg_ms = shared.millis()

        # update network representation
        mobility.move_random(state, distance=step_distance)
        mobility.connect_range(state, max_links=150)

        # update network
        tmp_ms = shared.millis()
        network.apply(state=state,
                      link_command=get_tc_command,
                      remotes=remotes)
        #software.apply(protocol=protocol, state=state) # we do not change the node count
        network_ms = shared.millis() - tmp_ms

        # Wait until wait seconds are over, else error
        shared.wait(wait_beg_ms, step_duration)

        paths = ping.get_random_paths(state, 2 * 400)
        paths = ping.filter_paths(state, paths, min_hops=2, path_count=200)
        ping_result = ping.ping(paths=paths,
                                duration_ms=2000,
                                verbosity='verbose',
                                remotes=remotes)

        # add data to csv file
        extra = (['node_count',
                  'time_ms'], [node_count,
                               shared.millis() - test_beg_ms])
        shared.csv_update(csvfile, '\t', extra, ping_result.getData())

    software.clear(remotes)
    network.clear(remotes)
Beispiel #2
0
def start_routing_protocol(protocol, rmap, ids, ignore_error=False):
    beg_count = count_instances(protocol, rmap)
    beg_ms = millis()

    for id in ids:
        remote = rmap[id]
        interface_up(remote, id, 'uplink', ignore_error)

    if protocol == 'babel':
        start_babel_instances(ids, rmap)
    elif protocol == 'batman-adv':
        start_batmanadv_instances(ids, rmap)
    elif protocol == 'bmx6':
        start_bmx6_instances(ids, rmap)
    elif protocol == 'cjdns':
        start_cjdns_instances(ids, rmap)
    elif protocol == 'olsr1':
        start_olsr1_instances(ids, rmap)
    elif protocol == 'olsr2':
        start_olsr2_instances(ids, rmap)
    elif protocol == 'ospf':
        start_ospf_instances(ids, rmap)
    elif protocol == 'yggdrasil':
        start_yggdrasil_instances(ids, rmap)
    elif protocol == 'none':
        return
    else:
        eprint(f'Error: unknown routing protocol: {protocol}')
        exit(1)

    wait_for_completion()

    # wait for last started process to fork
    # otherwise we might have one extra counted instance
    time.sleep(0.5)

    end_ms = millis()
    end_count = count_instances(protocol, rmap)

    count = end_count - beg_count
    if count != len(ids):
        eprint(
            f'Error: Failed to start {protocol} instances: {count}/{len(ids)} started'
        )
        stop_all_terminals()
        exit(1)

    if verbosity != 'quiet':
        print('Started {} {} instances in {}'.format(
            count, protocol, format_duration(end_ms - beg_ms)))
Beispiel #3
0
def run(topology, path, state):
    (node_count, link_count) = shared.json_count(state)

    with open(f'{prefix}scalability1-{protocol}-{topology}.csv',
              'a') as csvfile:
        print(f'run {protocol} on {path}')

        # Start routing software
        software_start_ms = shared.millis()
        software.start(protocol, remotes)
        software_stop_ms = shared.millis()

        # Let the nodes start up and discover themselves.
        shared.sleep(60)

        traffic_start_ms = shared.millis()
        traffic_begin = traffic.traffic(remotes)

        # Send <node_count> pings.
        # For a good routing algorithm, the traffic per node should be constant.
        paths = ping.get_random_paths(state, 2 * node_count)
        paths = ping.filter_paths(state,
                                  paths,
                                  min_hops=2,
                                  path_count=node_count)
        ping_result = ping.ping(remotes=remotes,
                                paths=paths,
                                duration_ms=300000,
                                verbosity='verbose')

        traffic_stop_ms = shared.millis()
        traffic_end = traffic.traffic(remotes)

        sysload_result = shared.sysload(remotes)

        # Stop routing software
        software.clear(remotes)

        # Add data to csv file
        extra = ([
            'node_count', 'software_startup_ms', 'traffic_measurement_ms'
        ], [
            node_count, (software_stop_ms - software_start_ms),
            (traffic_stop_ms - traffic_start_ms)
        ])
        shared.csv_update(csvfile, '\t', extra,
                          (traffic_end - traffic_begin).getData(),
                          ping_result.getData(), sysload_result)

        return (100.0 * ping_result.received / ping_result.send)
Beispiel #4
0
def stop_routing_protocol(protocol, rmap, ids, ignore_error=False):
    beg_count = count_instances(protocol, rmap)
    beg_ms = millis()

    if protocol == 'babel':
        stop_babel_instances(ids, rmap)
    elif protocol == 'batman-adv':
        stop_batmanadv_instances(ids, rmap)
    elif protocol == 'bmx6':
        stop_bmx6_instances(ids, rmap)
    elif protocol == 'cjdns':
        stop_cjdns_instances(ids, rmap)
    elif protocol == 'olsr1':
        stop_olsr1_instances(ids, rmap)
    elif protocol == 'olsr2':
        stop_olsr2_instances(ids, rmap)
    elif protocol == 'ospf':
        stop_ospf_instances(ids, rmap)
    elif protocol == 'yggdrasil':
        stop_yggdrasil_instances(ids, rmap)
    elif protocol == 'none':
        pass
    else:
        eprint('Error: unknown routing protocol: {}'.format(protocol))
        exit(1)

    for id in ids:
        remote = rmap[id]
        interface_down(remote, id, 'uplink', ignore_error=ignore_error)

    wait_for_completion()

    # wait for last stopped process to disappear
    # otherwise we might have one extra counted instance
    time.sleep(0.5)

    end_ms = millis()
    end_count = count_instances(protocol, rmap)

    count = beg_count - end_count
    if count != len(ids):
        eprint(
            f'Error: Failed to stop {protocol} instances: {count}/{len(ids)} left'
        )
        exit(1)

    if not ignore_error and verbosity != 'quiet':
        print('Stopped {} {} instances in {}'.format(
            len(ids), protocol, format_duration(end_ms - beg_ms)))
Beispiel #5
0
def run(protocol, files, csvfile):
    shared.seed_random(1234)

    for path in sorted(glob.glob(files)):
        state = shared.load_json(path)
        (node_count, link_count) = shared.json_count(state)

        print(f'run {protocol} on {path}')

        network.apply(state=state,
                      link_command=get_tc_command,
                      remotes=remotes)

        shared.sleep(10)

        for offset in range(0, 60, 2):
            tmp_ms = shared.millis()
            traffic_beg = traffic.traffic(remotes)
            traffic_ms = shared.millis() - tmp_ms

            tmp_ms = shared.millis()
            software.start(protocol)
            software_ms = shared.millis() - tmp_ms

            # Wait until wait seconds are over, else error
            shared.sleep(offset)

            paths = ping.get_random_paths(state, 2 * 200)
            paths = ping.filter_paths(state, paths, min_hops=2, path_count=200)
            ping_result = ping.ping(paths=paths,
                                    duration_ms=2000,
                                    verbosity='verbose',
                                    remotes=remotes)

            traffic_end = traffic.traffic(remotes)

            sysload_result = shared.sysload(remotes)

            software.clear(remotes)

            # add data to csv file
            extra = (['node_count', 'traffic_ms', 'software_ms', 'offset_ms'],
                     [node_count, traffic_ms, software_ms, offset * 1000])
            shared.csv_update(csvfile, '\t', extra,
                              (traffic_end - traffic_beg).getData(),
                              ping_result.getData(), sysload_result)

        network.clear(remotes)
Beispiel #6
0
def run(protocol, files, csvfile):
    for path in sorted(glob.glob(files)):
        state = shared.load_json(path)
        (node_count, link_count) = shared.json_count(state)

        # Limit node count to 300
        if node_count > 300:
            continue

        print(f'run {protocol} on {path}')

        network.apply(state=state,
                      link_command=get_tc_command,
                      remotes=remotes)

        shared.sleep(10)

        software_start_ms = shared.millis()
        software.start(protocol, remotes)
        software_startup_ms = shared.millis() - software_start_ms

        shared.sleep(300)

        start_ms = shared.millis()
        traffic_beg = traffic.traffic(remotes)

        paths = ping.get_random_paths(state, 2 * 200)
        paths = ping.filter_paths(state, paths, min_hops=2, path_count=200)
        ping_result = ping.ping(remotes=remotes,
                                paths=paths,
                                duration_ms=300000,
                                verbosity='verbose')

        traffic_ms = shared.millis() - start_ms
        traffic_end = traffic.traffic(remotes)

        sysload_result = shared.sysload(remotes)

        software.clear(remotes)
        network.clear(remotes)

        # add data to csv file
        extra = (['node_count', 'traffic_ms', 'software_startup_ms'],
                 [node_count, traffic_ms, software_startup_ms])
        shared.csv_update(csvfile, '\t', extra,
                          (traffic_end - traffic_beg).getData(),
                          ping_result.getData(), sysload_result)
Beispiel #7
0
def run(protocol, csvfile):
    for path in sorted(glob.glob(f'../../data/grid4/*.json')):
        state = shared.load_json(path)
        (node_count, link_count) = shared.json_count(state)

        print(f'run {protocol} on {path}')

        network.apply(state=state,
                      link_command=get_tc_command,
                      remotes=remotes)
        shared.sleep(10)

        software_start_ms = shared.millis()
        software.start(protocol, remotes)
        software_startup_ms = shared.millis() - software_start_ms

        shared.sleep(30)

        paths = ping.get_random_paths(state, 2 * link_count)
        paths = ping.filter_paths(state,
                                  paths,
                                  min_hops=2,
                                  path_count=link_count)
        ping_result = ping.ping(remotes=remotes,
                                paths=paths,
                                duration_ms=30000,
                                verbosity='verbose')

        sysload_result = shared.sysload(remotes)

        software.clear(remotes)

        # add data to csv file
        extra = (['node_count',
                  'software_startup_ms'], [node_count, software_startup_ms])
        shared.csv_update(csvfile, '\t', extra, ping_result.getData(),
                          sysload_result)

        network.clear(remotes)

        # abort benchmark when less then 40% of the pings arrive
        if ping_result.transmitted == 0 or (ping_result.received /
                                            ping_result.transmitted) < 0.4:
            break
Beispiel #8
0
def wait(beg_ms, until_sec):
    now_ms = millis()

    # wait until time is over
    if (now_ms - beg_ms) < (until_sec * 1000):
        time.sleep(((until_sec * 1000) - (now_ms - beg_ms)) / 1000.0)
    else:
        eprint('Wait timeout already passed by {:.2f}sec'.format(
            ((now_ms - beg_ms) - (until_sec * 1000)) / 1000))
        exit(1)
Beispiel #9
0
def run(protocol, csvfile):
	shared.seed_random(1377)

	for path in sorted(glob.glob(f'../../data/freifunk/*.json')):
		state = shared.load_json(path)

		(node_count, link_count) = shared.json_count(state)
		dataset_name = '{}-{:04d}'.format(os.path.basename(path)[9:-5], node_count)

		# limit to what the host can handle
		if node_count > 310:
			continue

		print(f'run {protocol} on {path}')

		state = network.apply(state=state, link_command=get_tc_command, remotes=remotes)
		shared.sleep(10)

		software.start(protocol, remotes)

		shared.sleep(300)

		start_ms = shared.millis()
		traffic_beg = traffic.traffic(remotes)

		paths = ping.get_random_paths(state, 2 * node_count)
		paths = shared.filter_paths(state, paths, min_hops=2, path_count=node_count)
		ping_result = shared.ping(remotes=remotes, paths=paths, duration_ms=300000, verbosity='verbose')

		sysload_result = shared.sysload(remotes)

		traffic_ms = shared.millis() - start_ms
		traffic_end = traffic.traffic(remotes)
		software.clear(remotes)

		# add data to csv file
		extra = (['dataset_name', 'node_count', 'traffic_ms'], [dataset_name, node_count, traffic_ms])
		shared.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result)

		network.clear(remotes)
Beispiel #10
0
def run(protocol, tasks, csvfile):
	for path, gateways in tasks:
		state = shared.load_json(path)
		(node_count, link_count) = shared.json_count(state)

		# Limit node count to 300
		if node_count > 300:
			continue

		print(f'run {protocol} on {path}')

		network.apply(state=state, remotes=remotes)

		shared.sleep(10)

		software_start_ms = shared.millis()
		software.start(protocol, remotes)
		software_startup_ms = shared.millis() - software_start_ms

		shared.sleep(30)

		start_ms = shared.millis()
		traffic_beg = traffic.traffic(remotes)

		paths = ping.get_paths_to_gateways(state, gateways)
		ping_result = ping.ping(remotes=remotes, paths=paths, duration_ms=300000, verbosity='verbose')

		traffic_ms = shared.millis() - start_ms
		traffic_end = traffic.traffic(remotes)

		sysload_result = shared.sysload(remotes)

		software.clear(remotes)
		network.clear(remotes)

		# add data to csv file
		extra = (['node_count', 'traffic_ms', 'software_startup_ms'], [node_count, traffic_ms, software_startup_ms])
		shared.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result)
Beispiel #11
0
def run(protocol, csvfile):
	shared.seed_random(23)

	node_count = 50
	state = topology.create_nodes(node_count)
	mobility.randomize_positions(state, xy_range=1000)
	mobility.connect_range(state, max_links=150)

	# create network and start routing software
	network.apply(state=state, link_command=get_tc_command, remotes=remotes)
	software.start(protocol)
	shared.sleep(30)

	for step_distance in [50, 100, 150, 200, 250, 300, 350, 400]:
		print(f'{protocol}: step_distance {step_distance}')

		traffic_beg = traffic.traffic(remotes)
		for n in range(0, 6):
			#with open(f'graph-{step_distance}-{n}.json', 'w+') as file:
			#	json.dump(state, file, indent='  ')

			# connect nodes range
			wait_beg_ms = shared.millis()

			# update network representation
			mobility.move_random(state, distance=step_distance)
			mobility.connect_range(state, max_links=150)

			# update network
			network.apply(state=state, link_command=get_tc_command, remotes=remotes)

			# Wait until wait seconds are over, else error
			shared.wait(wait_beg_ms, 15)

			paths = ping.get_random_paths(state, 2 * 400)
			paths = ping.filter_paths(state, paths, min_hops=2, path_count=200)
			ping_result = ping.ping(remotes=remotes, paths=paths, duration_ms=2000, verbosity='verbose')

			packets_arrived_pc = 100 * (ping_result.received / ping_result.send)
			traffic_end = traffic.traffic(remotes)

			# add data to csv file
			extra = (['node_count', 'time_ms', 'step_distance_m', 'n', 'packets_arrived_pc'], [node_count, shared.millis() - wait_beg_ms, step_distance, n, packets_arrived_pc])
			shared.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData())

			traffic_beg = traffic_end

	software.clear(remotes)
	network.clear(remotes)
Beispiel #12
0
def apply(state={},
          node_command=None,
          link_command=None,
          remotes=default_remotes):
    check_access(remotes)

    new_state = state
    (cur_state, cur_state_rmap) = get_current_state(remotes)

    # handle different new_state types
    if isinstance(new_state, str):
        if new_state == 'none':
            new_state = {}
        else:
            if not os.path.isfile(new_state):
                eprint(f'File not found: {new_state}')
                stop_all_terminals()
                exit(1)

            with open(new_state) as file:
                new_state = json.load(file)

    # map each node to a remote or local computer
    # distribute evenly with minimized interconnects
    rmap = _get_remote_mapping(cur_state, new_state, remotes, cur_state_rmap)
    data = _get_task(cur_state, new_state)

    beg_ms = millis()

    # add "switch" namespace
    if state_empty(cur_state):
        for remote in remotes:
            # add switch if it does not exist yet
            exec(remote, 'ip netns add "switch" || true')
            # disable IPv6 in switch namespace (no need, less overhead)
            exec(
                remote,
                'ip netns exec "switch" sysctl -q -w net.ipv6.conf.all.disable_ipv6=1'
            )

    for node in data.nodes_update:
        update_node(node, node_command, rmap)

    for link in data.links_update:
        update_link(link, link_command, rmap)

    for node in data.nodes_create:
        create_node(node, node_command, rmap)

    for link in data.links_create:
        create_link(link, link_command, rmap)

    for link in data.links_remove:
        remove_link(link, rmap)

    for node in data.nodes_remove:
        remove_node(node, rmap)

    # remove "switch" namespace
    if state_empty(new_state):
        for remote in remotes:
            exec(remote, 'ip netns del "switch" || true')

    # wait for tasks to complete
    wait_for_completion()
    end_ms = millis()

    if verbosity != 'quiet':
        print('Network setup in {}:'.format(format_duration(end_ms - beg_ms)))
        print(
            f'  nodes: {len(data.nodes_create)} created, {len(data.nodes_remove)} removed, {len(data.nodes_update)} updated'
        )
        print(
            f'  links: {len(data.links_create)} created, {len(data.links_remove)} removed, {len(data.links_update)} updated'
        )

    return new_state
Beispiel #13
0
def ping(
    paths,
    duration_ms=1000,
    remotes=default_remotes,
    interface=None,
    verbosity="normal",
    address_type=None,
    ping_deadline=1,
    ping_timeout=None,
):
    ping_count = 1
    rmap = get_remote_mapping(remotes)
    path_count = len(paths)

    # prepare ping tasks
    tasks = []
    for (source, target) in paths:
        source_remote = rmap[source]
        target_remote = rmap[target]

        if interface is None:
            interface = _get_interface(source_remote, source)

        target_addr = _get_ip_address(target_remote, target, interface,
                                      address_type)

        if target_addr is None:
            eprint(f"Cannot get address of {interface} in ns-{target}")
        else:
            debug = f"ping {source:>4} => {target:>4} ({target_addr:<18} / {interface})"
            command = (
                f"ip netns exec ns-{source} ping -c {ping_count} " +
                (f"-w {ping_deadline} " if ping_deadline is not None else "") +
                (f"-W {ping_timeout} " if ping_timeout is not None else "") +
                f"-D -I {interface} {target_addr}")
            tasks.append((source_remote, command, debug))

    processes = []
    started = 0

    def process_results():
        for (process, started_ms, debug, result) in processes:
            if not result.processed and process.poll() is not None:
                process.wait()
                (output, err) = process.communicate()
                _parse_ping(result, output.decode())
                result.processed = True

    # keep track of status ouput lines to delete them for updates
    lines_printed = 0

    def print_processes():
        nonlocal lines_printed

        # delete previous printed lines
        for _ in range(lines_printed):
            sys.stdout.write("\x1b[1A\x1b[2K")

        lines_printed = 0
        process_counter = 0
        for (process, started_ms, debug, result) in processes:
            process_counter += 1
            status = "???"
            if result.processed:
                if result.packet_loss == 0.0:
                    status = "success"
                elif result.packet_loss == 100.0:
                    status = "failed"
                else:
                    status = f"mixed ({result.packet_loss:0.2f}% loss)"
            else:
                status = "running"

            print(
                f"[{process_counter:03}:{started_ms:06}] {debug} => {status}")
            lines_printed += 1

    # start tasks in the given time frame
    start_ms = millis()
    last_processed = millis()
    tasks_count = len(tasks)
    while started < tasks_count:
        started_expected = math.ceil(tasks_count *
                                     ((millis() - start_ms) / duration_ms))
        if started_expected > started:
            for _ in range(0, started_expected - started):
                if len(tasks) == 0:
                    break

                (remote, command, debug) = tasks.pop()
                process = create_process(remote, command)
                started_ms = millis() - start_ms
                processes.append(
                    (process, started_ms, debug, _PingResult(ping_count)))

                # process results and print updates once per second
                if (last_processed + 1000) < millis():
                    last_processed = millis()
                    process_results()
                    if verbosity != "quiet":
                        print_processes()

                started += 1
        else:
            # sleep a small amount
            time.sleep(duration_ms / tasks_count / 1000.0 / 10.0)

    stop1_ms = millis()

    # wait until rest fraction of duration_ms is over
    if (stop1_ms - start_ms) < duration_ms:
        time.sleep((duration_ms - (stop1_ms - start_ms)) / 1000.0)

    stop2_ms = millis()

    process_results()
    if verbosity != "quiet":
        print_processes()

    # collect results
    rtt_avg_ms_count = 0
    ret = _PingStats()
    for (process, started_ms, debug, result) in processes:
        ret.send += result.send
        if result.processed:
            ret.received += int(result.send * (1.0 -
                                               (result.packet_loss / 100.0)))
            # failing ping outputs do not have rtt values
            if not math.isnan(result.rtt_avg):
                ret.rtt_avg_ms += result.rtt_avg
                rtt_avg_ms_count += 1

    if rtt_avg_ms_count > 0:
        ret.rtt_avg_ms /= float(rtt_avg_ms_count)

    result_duration_ms = stop1_ms - start_ms
    result_filler_ms = stop2_ms - stop1_ms

    if verbosity != "quiet":
        print(
            "pings send: {}, received: {} ({}), measurement span: {}ms".format(
                ret.send,
                ret.received,
                "-" if (ret.send == 0) else
                f"{100.0 * (ret.received / ret.send):0.2f}%",
                result_duration_ms + result_filler_ms,
            ))

    return ret
Beispiel #14
0
def ping_paths(paths,
               duration_ms=1000,
               remotes=default_remotes,
               interface=None,
               verbosity='normal'):
    ping_deadline = 1
    ping_count = 1
    processes = []
    start_ms = millis()
    started = 0
    rmap = get_remote_mapping(remotes)
    path_count = len(paths)
    while started < path_count:
        # number of expected tests to have been run
        started_expected = math.ceil(path_count *
                                     ((millis() - start_ms) / duration_ms))
        if started_expected > started:
            for _ in range(0, started_expected - started):
                if len(paths) == 0:
                    break
                (source, target) = paths.pop()

                source_remote = rmap[source]
                target_remote = rmap[target]

                if interface is None:
                    interface = _get_interface(source_remote, source)

                target_addr = _get_ip_address(target_remote, target, interface)

                if target_addr is None:
                    eprint(f'Cannot get address of {interface} in ns-{target}')
                    # count as started
                    started += 1
                else:
                    debug = '[{:06}] Ping {} => {} ({} / {})'.format(
                        millis() - start_ms, source, target, target_addr,
                        interface)
                    process = create_process(
                        source_remote,
                        f'ip netns exec ns-{source} ping -c {ping_count} -w {ping_deadline} -D {target_addr}'
                    )
                    processes.append((process, debug))
                    started += 1
        else:
            # sleep a small amount
            time.sleep(duration_ms / path_count / 1000.0 / 10.0)

    stop1_ms = millis()

    # wait until duration_ms is over
    if (stop1_ms - start_ms) < duration_ms:
        time.sleep((duration_ms - (stop1_ms - start_ms)) / 1000.0)

    stop2_ms = millis()

    ret = _PingResult()

    # wait/collect for results from pings (prolongs testing up to 1 second!)
    for (process, debug) in processes:
        process.wait()
        (output, err) = process.communicate()
        result = _parse_ping(output.decode())
        result.send = ping_count  # TODO: nicer

        ret.send += result.send
        ret.transmitted += result.transmitted
        ret.received += result.received
        ret.rtt_avg += result.rtt_avg

        if verbosity != 'quiet':
            if result.send != result.received:
                print(f'{debug} => failed')
            else:
                # success
                print(f'{debug}')

    ret.rtt_avg = 0 if ret.received == 0 else int(ret.rtt_avg / ret.received)
    result_duration_ms = stop1_ms - start_ms
    result_filler_ms = stop2_ms - stop1_ms

    if verbosity != 'quiet':
        print('send: {}, received: {}, arrived: {}%, measurement span: {}ms'.
              format(
                  ret.send, ret.received, '-' if (ret.send == 0) else
                  '{:0.2f}'.format(100.0 * (ret.received / ret.send)),
                  result_duration_ms + result_filler_ms))

    return ret
Beispiel #15
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--verbosity',
                        choices=['verbose', 'normal', 'quiet'],
                        default='normal',
                        help='Set verbosity.')
    parser.add_argument(
        '--remotes',
        help='Distribute nodes and links on remotes described in the JSON file.'
    )
    parser.set_defaults(to_state=None)

    subparsers = parser.add_subparsers(dest='action',
                                       required=True,
                                       help='Action')

    parser_start = subparsers.add_parser(
        'start', help='Run start script in every namespace.')
    parser_start.add_argument('protocol',
                              help='Routing protocol script prefix.')
    parser_start.add_argument('to_state',
                              nargs='?',
                              default=None,
                              help='To state')

    parser_stop = subparsers.add_parser(
        'stop', help='Run stop script in every namespace.')
    parser_stop.add_argument('protocol',
                             help='Routing protocol script prefix.')
    parser_stop.add_argument('to_state',
                             nargs='?',
                             default=None,
                             help='To state')

    parser_change = subparsers.add_parser(
        'apply', help='Run stop/start scripts in every namespace.')
    parser_change.add_argument('protocol',
                               help='Routing protocol script prefix.')
    parser_change.add_argument('to_state',
                               nargs='?',
                               default=None,
                               help='To state')

    parser_run = subparsers.add_parser(
        'run', help='Execute any command in every namespace.')
    parser_run.add_argument(
        'command',
        nargs=argparse.REMAINDER,
        help=
        'Shell command that is run. Remote address and namespace id is added to call arguments.'
    )
    parser_run.add_argument('to_state',
                            nargs='?',
                            default=None,
                            help='To state')

    parser_copy = subparsers.add_parser('copy', help='Copy to all remotes.')
    parser_copy.add_argument('source', nargs='+')
    parser_copy.add_argument('destination')

    parser_clear = subparsers.add_parser(
        'clear', help='Run all stop scripts in every namespaces.')

    args = parser.parse_args()

    if args.remotes:
        if not os.path.isfile(args.remotes):
            eprint(f'File not found: {args.remotes}')
            stop_all_terminals()
            exit(1)

        with open(args.remotes) as file:
            args.remotes = [Remote.from_json(obj) for obj in json.load(file)]
    else:
        args.remotes = default_remotes

    check_access(args.remotes)

    global verbosity
    verbosity = args.verbosity

    # get nodes that have been added or will be removed
    (old_ids, new_ids, rmap) = _get_update(args.to_state, args.remotes)

    if args.action == 'start':
        ids = new_ids if args.to_state else list(rmap.keys())

        beg_ms = millis()
        _start_protocol(args.protocol, rmap, ids)
        end_ms = millis()

        if verbosity != 'quiet':
            print('started {} in {} namespaces in {}'.format(
                args.protocol, len(ids), format_duration(end_ms - beg_ms)))
    elif args.action == 'stop':
        ids = old_ids if args.to_state else list(rmap.keys())

        beg_ms = millis()
        _stop_protocol(args.protocol, rmap, ids)
        end_ms = millis()

        if verbosity != 'quiet':
            print('stopped {} in {} namespaces in {}'.format(
                args.protocol, len(ids), format_duration(end_ms - beg_ms)))
    elif args.action == 'apply':
        beg_ms = millis()
        _stop_protocol(args.protocol, rmap, old_ids)
        _start_protocol(args.protocol, rmap, new_ids)
        end_ms = millis()

        if verbosity != 'quiet':
            print('applied {} in {} namespaces in {}'.format(
                args.protocol, len(rmap.keys()),
                format_duration(end_ms - beg_ms)))
    elif args.action == 'clear':
        beg_ms = millis()
        clear(args.remotes)
        end_ms = millis()

        if verbosity != 'quiet':
            print('cleared on {} remotes in {}'.format(
                len(args.remotes), format_duration(end_ms - beg_ms)))
    elif args.action == 'copy':
        beg_ms = millis()
        copy(args.remotes, args.source, args.destination)
        end_ms = millis()

        if verbosity != 'quiet':
            print('copied on {} remotes in {}'.format(
                len(args.remotes), format_duration(end_ms - beg_ms)))
    elif args.action == 'run':
        ids = new_ids if args.to_state else list(rmap.keys())

        for id in ids:
            remote = rmap[id]
            label = remote.address or 'local'
            cmd = f'ip netns exec ns-{id} {" ".join(args.command)} {label} {id}'
            _exec_verbose(remote, cmd)
    else:
        eprint('Unknown action: {}'.format(args.action))
        stop_all_terminals()
        exit(1)

    stop_all_terminals()