def test_fails_when_non_positive_buckets():
with shell.climb_git_root():
stdin = 'a'
print(shell.run('pwd'))
res = shell.run('bsv | bbucket 0', stdin=stdin, warn=True)
assert 'NUM_BUCKETS must be positive, got: 0' == res['stderr']
assert res['exitcode'] == 1
def clone_source():
with shell.climb_git_root():
orig = os.getcwd()
with shell.tempdir(cleanup=False):
shell.run(f"rsync -avhc {orig}/ . --exclude '.git' --exclude '.tox' --exclude '.backups' --exclude '__pycache__'")
shell.run('mkdir .git')
return os.getcwd()
def test_fails_when_too_many_buckets():
with shell.climb_git_root():
stdin = 'a'
res = shell.run('bsv | bbucket', int(1e8), stdin=stdin, warn=True)
assert res['exitcode'] == 1
assert 'NUM_BUCKETS must be less than 1e8, got: 100000000' == res[
'stderr']
def run(stdin, *args):
with shell.climb_git_root():
stdinpath = 'stdin'
stdoutpath = 'stdout'
with open(stdinpath, 'w') as f:
f.write(stdin)
shell.run(*(('set -o pipefail; cat', stdinpath, '|') + args + ('>', stdoutpath)), stream=True)
with open(stdoutpath) as f:
return f.read()
def test_max_bytes():
stdin = 'a' * (2**16 - 2)
assert len(stdin.strip()) == len(run(stdin, 'bsv | csv').strip())
stdin = 'a' * (2**16)
with shell.climb_git_root():
res = shell.run('bsv', stdin=stdin, warn=True)
assert 'fatal: cannot have columns with more than 2**16 - 1 bytes, column: 0, size: 65536, content: aaaaaaaaaa...' == res[
'stderr']
assert res['exitcode'] == 1
def push():
with shell.tempdir():
with open('s4.conf', 'w') as f:
f.write(conf)
run('aws-ec2-scp -y s4.conf :.s4.conf', *ids)
with shell.climb_git_root():
run('aws-ec2-rsync -y . :/mnt/s4', cluster_name)
run('aws-ec2-ssh -yc scripts/install_archlinux.sh', *ids)
state['ids'] = ids
def compile_buffer_sizes(name, buffers):
with shell.climb_git_root():
shell.run('cp -f util/util.h util/util.h.bak')
try:
for i in buffers:
shell.run(f'cat util/util.h.bak | sed -E "s/#define BUFFER_SIZE.*/#define BUFFER_SIZE {i}/" > util/util.h')
print('compile:', name, i, shell.run('cat util/util.h | grep "define BUFFER_SIZE"'), flush=True)
shell.run('make', name)
shell.run(f'mv -f bin/{name} bin/{name}.{i}')
finally:
shell.run('cat util/util.h.bak > util/util.h')
shell.run('rm -f util/util.h.bak')
def test_fails_when_too_many_columns():
with shell.climb_git_root():
stdin = 'a,' * (2**16 - 1)
with shell.tempdir(cleanup=False):
with open('input', 'w') as f:
f.write(stdin)
path = os.path.abspath('input')
try:
res = shell.run('cat', path, '| bin/_csv >/dev/null', warn=True)
finally:
shell.run('rm', path)
assert res['exitcode'] == 1
assert 'fatal: line with more than 65535 columns' == res['stderr']
def runb(stdin, *args):
with shell.climb_git_root():
stdinpath = f'stdin.{uuid.uuid4()}'
stdoutpath = f'stdout.{uuid.uuid4()}'
if isinstance(stdin, str):
with open(stdinpath, 'w') as f:
f.write(stdin)
else:
with open(stdinpath, 'wb') as f:
f.write(stdin)
shell.run(*(('set -o pipefail; cat', stdinpath, '|') + args +
('>', stdoutpath)),
stream=True)
with open(stdoutpath, 'rb') as f:
return f.read()
import shell
import os
with shell.climb_git_root():
max_columns = int(shell.run('cat util/util.h | grep "define MAX_COLUMNS"').split()[-1])
def clone_source():
with shell.climb_git_root():
orig = os.getcwd()
with shell.tempdir(cleanup=False):
shell.run(f"rsync -avhc {orig}/ . --exclude '.git' --exclude '.tox' --exclude '.backups' --exclude '__pycache__'")
shell.run('mkdir .git')
return os.getcwd()
def run(stdin, *args):
with shell.climb_git_root():
stdinpath = 'stdin'
stdoutpath = 'stdout'
with open(stdinpath, 'w') as f:
f.write(stdin)
shell.run(*(('set -o pipefail; cat', stdinpath, '|') + args + ('>', stdoutpath)), stream=True)
with open(stdoutpath) as f:
return f.read()
def runb(stdin, *args):
with shell.climb_git_root():
stdinpath = 'stdin'
stdoutpath = 'stdout'
if isinstance(stdin, str):
with open(stdinpath, 'w') as f:
f.write(stdin)
def test_oversized_line():
with shell.climb_git_root():
stdin = 'a' * 8
res = shell.run('bin/_csv.8', stdin=stdin, warn=True)
assert res['exitcode'] == 1
assert 'fatal: line longer than BUFFER_SIZE' == res['stderr']
def test_fails_when_non_positive_fields():
with shell.climb_git_root():
stdin = 'a,b,c'
res = shell.run('bsv | bcut 0', stdin=stdin, warn=True)
assert 'fatal: fields must be positive, got: 0' == res['stderr']
assert res['exitcode'] == 1
def test_fails_when_not_enough_columns():
with shell.climb_git_root():
stdin = 'a,b,c'
res = shell.run('bsv | bcut 4', stdin=stdin, warn=True)
assert 'fatal: line without 4 columns: a,b,c' == res['stderr']
assert res['exitcode'] == 1
def teardown_module():
with shell.climb_git_root():
shell.run('make clean', stream=True)
def setup_module():
with shell.cd(os.path.dirname(os.path.abspath(__file__))):
with shell.climb_git_root():
shell.run('make -j', stream=True)
os.environ['PATH'] += f':{os.getcwd()}/bin'
