def test_fails_when_non_positive_buckets(): with shell.climb_git_root(): stdin = 'a' print(shell.run('pwd')) res = shell.run('bsv | bbucket 0', stdin=stdin, warn=True) assert 'NUM_BUCKETS must be positive, got: 0' == res['stderr'] assert res['exitcode'] == 1
def clone_source(): with shell.climb_git_root(): orig = os.getcwd() with shell.tempdir(cleanup=False): shell.run(f"rsync -avhc {orig}/ . --exclude '.git' --exclude '.tox' --exclude '.backups' --exclude '__pycache__'") shell.run('mkdir .git') return os.getcwd()
def test_fails_when_too_many_buckets(): with shell.climb_git_root(): stdin = 'a' res = shell.run('bsv | bbucket', int(1e8), stdin=stdin, warn=True) assert res['exitcode'] == 1 assert 'NUM_BUCKETS must be less than 1e8, got: 100000000' == res[ 'stderr']
def run(stdin, *args): with shell.climb_git_root(): stdinpath = 'stdin' stdoutpath = 'stdout' with open(stdinpath, 'w') as f: f.write(stdin) shell.run(*(('set -o pipefail; cat', stdinpath, '|') + args + ('>', stdoutpath)), stream=True) with open(stdoutpath) as f: return f.read()
def test_max_bytes(): stdin = 'a' * (2**16 - 2) assert len(stdin.strip()) == len(run(stdin, 'bsv | csv').strip()) stdin = 'a' * (2**16) with shell.climb_git_root(): res = shell.run('bsv', stdin=stdin, warn=True) assert 'fatal: cannot have columns with more than 2**16 - 1 bytes, column: 0, size: 65536, content: aaaaaaaaaa...' == res[ 'stderr'] assert res['exitcode'] == 1
def push(): with shell.tempdir(): with open('s4.conf', 'w') as f: f.write(conf) run('aws-ec2-scp -y s4.conf :.s4.conf', *ids) with shell.climb_git_root(): run('aws-ec2-rsync -y . :/mnt/s4', cluster_name) run('aws-ec2-ssh -yc scripts/install_archlinux.sh', *ids) state['ids'] = ids
def compile_buffer_sizes(name, buffers): with shell.climb_git_root(): shell.run('cp -f util/util.h util/util.h.bak') try: for i in buffers: shell.run(f'cat util/util.h.bak | sed -E "s/#define BUFFER_SIZE.*/#define BUFFER_SIZE {i}/" > util/util.h') print('compile:', name, i, shell.run('cat util/util.h | grep "define BUFFER_SIZE"'), flush=True) shell.run('make', name) shell.run(f'mv -f bin/{name} bin/{name}.{i}') finally: shell.run('cat util/util.h.bak > util/util.h') shell.run('rm -f util/util.h.bak')
def test_fails_when_too_many_columns(): with shell.climb_git_root(): stdin = 'a,' * (2**16 - 1) with shell.tempdir(cleanup=False): with open('input', 'w') as f: f.write(stdin) path = os.path.abspath('input') try: res = shell.run('cat', path, '| bin/_csv >/dev/null', warn=True) finally: shell.run('rm', path) assert res['exitcode'] == 1 assert 'fatal: line with more than 65535 columns' == res['stderr']
def runb(stdin, *args): with shell.climb_git_root(): stdinpath = f'stdin.{uuid.uuid4()}' stdoutpath = f'stdout.{uuid.uuid4()}' if isinstance(stdin, str): with open(stdinpath, 'w') as f: f.write(stdin) else: with open(stdinpath, 'wb') as f: f.write(stdin) shell.run(*(('set -o pipefail; cat', stdinpath, '|') + args + ('>', stdoutpath)), stream=True) with open(stdoutpath, 'rb') as f: return f.read()
import shell import os with shell.climb_git_root(): max_columns = int(shell.run('cat util/util.h | grep "define MAX_COLUMNS"').split()[-1]) def clone_source(): with shell.climb_git_root(): orig = os.getcwd() with shell.tempdir(cleanup=False): shell.run(f"rsync -avhc {orig}/ . --exclude '.git' --exclude '.tox' --exclude '.backups' --exclude '__pycache__'") shell.run('mkdir .git') return os.getcwd() def run(stdin, *args): with shell.climb_git_root(): stdinpath = 'stdin' stdoutpath = 'stdout' with open(stdinpath, 'w') as f: f.write(stdin) shell.run(*(('set -o pipefail; cat', stdinpath, '|') + args + ('>', stdoutpath)), stream=True) with open(stdoutpath) as f: return f.read() def runb(stdin, *args): with shell.climb_git_root(): stdinpath = 'stdin' stdoutpath = 'stdout' if isinstance(stdin, str): with open(stdinpath, 'w') as f: f.write(stdin)
def test_oversized_line(): with shell.climb_git_root(): stdin = 'a' * 8 res = shell.run('bin/_csv.8', stdin=stdin, warn=True) assert res['exitcode'] == 1 assert 'fatal: line longer than BUFFER_SIZE' == res['stderr']
def test_fails_when_non_positive_fields(): with shell.climb_git_root(): stdin = 'a,b,c' res = shell.run('bsv | bcut 0', stdin=stdin, warn=True) assert 'fatal: fields must be positive, got: 0' == res['stderr'] assert res['exitcode'] == 1
def test_fails_when_not_enough_columns(): with shell.climb_git_root(): stdin = 'a,b,c' res = shell.run('bsv | bcut 4', stdin=stdin, warn=True) assert 'fatal: line without 4 columns: a,b,c' == res['stderr'] assert res['exitcode'] == 1
def teardown_module(): with shell.climb_git_root(): shell.run('make clean', stream=True)
def setup_module(): with shell.cd(os.path.dirname(os.path.abspath(__file__))): with shell.climb_git_root(): shell.run('make -j', stream=True) os.environ['PATH'] += f':{os.getcwd()}/bin'