def test_deleting_directory_in_fixed(sub_open, mount):
fdir = join(sub_open, 'new_test_file')
fdir2 = join(sub_open, 'new_test_file2')
fdir3 = join(sub_open, 'new_test_file3')
# Make sure we cannot delete existing files in fixed mode
assert not isdir(fdir)
mkdir(fdir)
assert isdir(fdir)
mount(fixed=True)
assert isdir(fdir)
with pytest.raises(PermissionError):
rmdir(fdir)
assert isdir(fdir)
del fdir
assert not isdir(fdir2)
mkdir(fdir2)
assert isdir(fdir2)
rename([fdir2], [fdir3])
assert not isdir(fdir2)
assert isdir(fdir3)
mount(fixed=True)
assert not isdir(fdir3)
def test_set_utime(sub_done, mount):
fname = join(sub_done, 'new_test_file')
fname2 = join(sub_done, 'new_test_file2')
open(fname, 'w').close()
old_st = os.lstat(fname)
os.utime(fname, (10, 123.5))
new_st = os.lstat(fname)
assert new_st.st_atime == 10.0
assert new_st.st_mtime == 123.5
assert old_st.st_atime != 10.0
assert old_st.st_mtime != 123.5
mount(fixed=True)
with pytest.raises(PermissionError):
os.utime(join(sub_done, 'dir', 'single_file_work'), (100, 1235))
open(fname2, 'w').close()
old_st = os.lstat(fname2)
os.utime(fname2, (10, 123.5))
new_st = os.lstat(fname2)
assert new_st.st_atime == 10.0
assert new_st.st_mtime == 123.5
assert old_st.st_atime != 10.0
assert old_st.st_mtime != 123.5
def test_renaming_file_in_fixed(sub_open, mount):
fname = join(sub_open, 'new_test_file')
fname2 = join(sub_open, 'new_test_file2')
fname3 = join(sub_open, 'new_test_file3')
# Make sure we cannot delete existing files in fixed mode
assert not isfile(fname)
with open(fname, 'w') as f:
f.write('Hello thomas\n')
assert isfile(fname)
mount(fixed=True)
assert isfile(fname)
with pytest.raises(PermissionError):
rename([fname], [fname2])
del fname
assert not isfile(fname2)
with open(fname2, 'w') as f:
f.write('Hello thomas\n')
assert isfile(fname2)
rename([fname2], [fname3])
assert not isfile(fname2)
assert isfile(fname3)
with open(fname3, 'r') as f:
assert f.read() == 'Hello thomas\n'
mount(fixed=True)
assert not isfile(fname3)
def _default_do_files(filename):
l = filename.split('.')
for i in range(1,len(l)+1):
basename = join('.', l[:i])
ext = join('.', l[i:])
if ext: ext = '.' + ext
yield ("default%s.do" % ext), basename, ext
def _default_do_files(filename):
l = filename.split('.')
for i in range(1, len(l) + 1):
basename = join('.', l[:i])
ext = join('.', l[i:])
if ext: ext = '.' + ext
yield ("default%s.do" % ext), basename, ext
def test_compiling_program(sub_done, mount, fixed):
url = 'https://attach.libremail.nl/__test_codegra.fs__.tar.gz'
fname = join(sub_done, '42.tar.gz')
fdir = join(sub_done, '42sh/')
urllib.request.urlretrieve(url, fname)
tar = tarfile.open(fname, "r:gz")
tar.extractall(sub_done)
tar.close()
print(subprocess.check_output(['make', '-C', fdir]))
assert isdir(fdir)
assert isfile(fname)
assert subprocess.check_output(
[join(fdir, '42sh'), '-c', 'echo hello from 42']
).decode('utf-8') == 'hello from 42\n'
mount()
if fixed:
assert not isdir(fdir)
assert not isfile(fname)
return
assert isfile(fname)
assert isdir(fdir)
assert subprocess.check_output(
[join(fdir, '42sh'), '-c', 'echo hello from 42']
).decode('utf-8') == 'hello from 42\n'
rm_rf(fdir)
assert not isdir(fdir)
mount()
assert isfile(fname)
assert not isdir(fdir)
def trainTree(tDir='.'):
tFile = open(join(tDir, 'trainData.csv'), 'r')
lFile = open(join(tDir, 'trainLabels.csv'), 'r')
try:
nFile = open(join(tDir, 'attrNames.csv'), 'r')
except:
nFile = None
print tDir + ':'
tree.train(tFile, lFile, nFile, maxEnt, maxPValue, maxSamples, maxAttrs)
def test_truncate_file(sub_done):
with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f:
old = f.read()
with open(join(sub_done, 'dir', 'single_file_work'), 'a') as f:
f.write('wow\n')
with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f:
assert f.read() == old + 'wow\n'
with open(join(sub_done, 'dir', 'single_file_work'), 'a') as f:
assert f.truncate(len(old))
with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f:
assert f.read() == old
with open(join(sub_done, 'dir', 'single_file_work'), 'a') as f:
assert f.truncate(len(old) * 3)
with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f:
new = f.read()
assert len(new) == len(old) * 3
new.startswith(old)
all(n == '\0' for n in new[len(old):])
fd = os.open(
join(sub_done, 'dir', 'single_file_work'), os.O_TRUNC | os.O_WRONLY
)
os.write(fd, b'hello\n')
os.close(fd)
with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f:
assert f.read() == 'hello\n'
def test_deleting_file_in_fixed(sub_open, mount):
fname = join(sub_open, 'new_test_file')
fname2 = join(sub_open, 'new_test_file2')
# Make sure we cannot delete existing files in fixed mode
assert not isfile(fname)
with open(fname, 'w') as f:
f.write('Hello thomas\n')
assert isfile(fname)
mount(fixed=True)
assert isfile(fname)
with open(fname, 'r') as f:
assert f.read() == 'Hello thomas\n'
with pytest.raises(PermissionError):
with open(fname, 'w') as f:
f.write('Hello thomas\n')
with pytest.raises(PermissionError):
rm(fname)
del fname
# Now make sure we can delete files that did not exist
assert not isfile(fname2)
with open(fname2, 'w') as f:
f.write('Hello thomas2\n')
with open(fname2, 'r') as f:
assert f.read() == 'Hello thomas2\n'
assert isfile(fname2)
rm(fname2)
assert not isfile(fname2)
# Make sure files created in fixed mode are not visible after a remount
assert not isfile(fname2)
with open(fname2, 'w') as f:
f.write('Hello thomas2\n')
with open(fname2, 'r') as f:
assert f.read() == 'Hello thomas2\n'
assert isfile(fname2)
mount(fixed=True)
assert not isfile(fname2)
with open(fname2, 'w') as f:
f.write('Hello thomas2\n')
with open(fname2, 'r') as f:
assert f.read() == 'Hello thomas2\n'
assert isfile(fname2)
mount(fixed=True)
assert not isfile(fname2)
def test_delete_files(mount_dir, sub_open, sub_done):
with open(join(sub_open, 'file1'), 'w') as f:
f.write('abc\n')
assert isfile(sub_open, 'file1')
with open(join(sub_open, 'file1'), 'r') as f:
assert f.read() == 'abc\n'
rm(sub_open, 'file1')
assert not isfile(sub_open, 'file1')
with pytest.raises(FileNotFoundError):
rm(sub_open, 'nonexisting')
def test_create_invalid_file(mount_dir):
with pytest.raises(PermissionError):
with open(join(mount_dir, 'file'), 'w+') as f:
f.write('hello\n')
with pytest.raises(PermissionError):
with open(
join(
mount_dir, [l for l in ls(mount_dir)
if isdir(mount_dir, l)][0], 'file'
), 'w+'
) as f:
f.write('hello\n')
def test_rename(mount_dir, sub_done, sub_open):
assert isdir(sub_done, 'dir')
assert not isdir(sub_done, 'dir33')
assert isdir(sub_done, 'dir')
assert not isdir(sub_done, 'dir33')
mkdir(sub_done, 'dir33')
mkdir(sub_done, 'dir', 'sub_dir')
assert isdir(sub_done, 'dir33')
with open(join(sub_done, 'dir33', 'new_file'), 'w') as f:
f.write('bye\n')
rename([sub_done, 'dir33'], [sub_done, 'dir', 'sub_dir', 'dir33'])
with open(join(sub_done, 'dir', 'sub_dir', 'dir33', 'new_file'), 'r') as f:
assert f.read() == 'bye\n'
def read_stamp(self, runid=None, st=None, st_deps=None):
# FIXME: make this formula more well-defined
if runid is None:
if st_deps == None:
try: st_deps = os.stat(self.tmpfilename('deps'))
except OSError: st_deps = False
if st_deps == False:
runid_suffix = ''
else:
runid_suffix = '+' + str(int(st_deps.st_mtime))
else:
runid_suffix = '+' + str(int(runid))
if st == None:
try: st = os.stat(self.name)
except OSError: st = False
if st == False:
return STAMP_MISSING + runid_suffix
if stat.S_ISDIR(st.st_mode):
return STAMP_DIR + runid_suffix
else:
# a "unique identifier" stamp for a regular file
return join('-', (st.st_ctime, st.st_mtime,
st.st_size, st.st_ino)) + runid_suffix
def _init_from_idname(self, id, name):
q = ('select %s from Files ' % join(', ', _file_cols))
if id != None:
q += 'where rowid=?'
l = [id]
elif name != None:
name = (name == ALWAYS) and ALWAYS or relpath(name, vars.BASE)
q += 'where name=?'
l = [name]
else:
raise Exception('name or id must be set')
d = db()
row = d.execute(q, l).fetchone()
if not row:
if not name:
raise Exception('No file with id=%r name=%r' % (id, name))
try:
_write('insert into Files (name) values (?)', [name])
except sqlite3.IntegrityError:
# some parallel redo probably added it at the same time; no
# big deal.
pass
row = d.execute(q, l).fetchone()
assert (row)
return self._init_from_cols(row)
def test_difficult_assigned_to(
mount, teacher_jwt, student_jwt, teacher_id, ta_id, assig_open,
student2_jwt
):
assig_id = open(join(assig_open, '.cg-assignment-id')).read().strip()
def submit(jwt):
req = requests.post(
'http://localhost:5000/api/v1/assignments/{}/submission'.
format(assig_id),
headers={'Authorization': 'Bearer ' + jwt},
files=dict(
file=open('./test_data/multiple_dir_archive.zip', 'rb')
)
)
req.raise_for_status()
return req.json()
def assign(sub, user_id):
requests.patch(
'http://localhost:5000/api/v1/submissions/{}/grader'.format(
sub["id"]
),
json={
'user_id': user_id
},
headers={
'Authorization': 'Bearer ' + teacher_jwt
},
).raise_for_status
sub2 = submit(student2_jwt)
old = submit(student_jwt)
new = submit(student_jwt)
print(sub2, old, new)
assign(sub2, ta_id)
assign(old, ta_id)
assign(new, teacher_id)
mount(assigned_to_me=True)
# We do not expect to see any submission. A submission is assigned to us,
# but it is not the newest of the student, so we do not show it.
print(ls(assig_open))
assert not any(['Student1' in l for l in ls(assig_open)])
assert any(['Student2' in l for l in ls(assig_open)])
mount(assigned_to_me=False)
assert ls(assig_open)
# Now we assign the newest submission to us, so we expect to see it.
requests.patch(
'http://localhost:5000/api/v1/submissions/{}/grader'.format(new["id"]),
json={'user_id': ta_id},
headers={'Authorization': 'Bearer ' + teacher_jwt},
)
mount(assigned_to_me=True)
print(ls(assig_open))
assert any(['Student1' in l for l in ls(assig_open)])
assert any(['Student2' in l for l in ls(assig_open)])
def _init_from_idname(self, id, name):
q = ('select %s from Files ' % join(', ', _file_cols))
if id != None:
q += 'where rowid=?'
l = [id]
elif name != None:
name = (name==ALWAYS) and ALWAYS or relpath(name, vars.BASE)
q += 'where name=?'
l = [name]
else:
raise Exception('name or id must be set')
d = db()
row = d.execute(q, l).fetchone()
if not row:
if not name:
raise Exception('File with id=%r not found and '
'name not given' % id)
try:
_write('insert into Files (name) values (?)', [name])
except sqlite3.IntegrityError:
# some parallel redo probably added it at the same time; no
# big deal.
pass
row = d.execute(q, l).fetchone()
assert(row)
return self._init_from_cols(row)
def _interpreter_locations(dodir):
dodir = os.path.realpath(dodir)
dirbits = dodir.split("/")
for i in range(len(dirbits), -1, -1):
d = join("/", dirbits[:i])
yield (d)
yield (d + "/do")
def _interpreter_locations(dodir):
dodir = os.path.realpath(dodir)
dirbits = dodir.split('/')
for i in range(len(dirbits), -1, -1):
d = join('/', dirbits[:i])
yield(d)
yield(os.path.join(d, "do"))
def test_force_create_file(sub_done):
with pytest.raises(FileExistsError):
fd = os.open(
join(sub_done, 'dir', 'single_file_work'),
os.O_CREAT | os.O_EXCL | os.O_WRONLY
)
os.write(fd, b'WRONG\n')
os.close(fd)
def test_write_to_directory(sub_done):
fdir = join(sub_done, 'new_dir')
fname = join(fdir, 'new_file')
mkdir(fdir)
assert isdir(fdir)
assert not isfile(fdir)
with pytest.raises(IsADirectoryError):
with open(fdir, 'w') as f:
f.write('hello\n')
open(fname, 'w').close()
with pytest.raises(NotADirectoryError):
ls(fname)
with pytest.raises(FileExistsError):
mkdir(fname)
def test_list_assignments(mount_dir):
course_dir = join(mount_dir, 'Programmeertalen')
assert isdir(mount_dir)
assert isdir(course_dir)
for assig in ['Haskell', 'Go', 'Python', 'Shell']:
assert isdir(course_dir, assig)
def _possible_do_files(t):
dirname, filename = os.path.split(t)
yield (dirname, "%s.do" % filename, "", filename, "")
yield (dirname, "%s.do" % filename, "", filename, "")
# It's important to try every possibility in a directory before resorting
# to a parent directory. Think about nested projects: I don't want
# ../../default.o.do to take precedence over ../default.do, because
# the former one might just be an artifact of someone embedding my project
# into theirs as a subdir. When they do, my rules should still be used
# for building my project in *all* cases.
dirbits = os.path.abspath(dirname).split("/")
for i in range(len(dirbits), -1, -1):
basedir = os.path.join(dirname, join("/", [".."] * (len(dirbits) - i)))
subdir = join("/", dirbits[i:])
for dofile, basename, ext in _default_do_files(filename):
yield (basedir, dofile, subdir, os.path.join(subdir, basename), ext)
def save(self):
cols = join(', ', ['%s=?' % i for i in _file_cols[2:]])
_write('update Files set '
' %s '
' where rowid=?' % cols, [
self.is_generated, self.is_override, self.checked_runid,
self.changed_runid, self.failed_runid, self.stamp,
self.csum, self.id
])
def save(self):
cols = join(', ', ['%s=?'%i for i in _file_cols[2:]])
_write('update Files set '
' %s '
' where rowid=?' % cols,
[self.is_generated, self.is_override,
self.checked_runid, self.changed_runid, self.failed_runid,
self.stamp, self.csum,
self.id])
def test_create_files(mount_dir, sub_open, sub_done):
with pytest.raises(PermissionError):
with open(join(sub_open, 'file1'), 'w') as f:
pass
assert not isfile(sub_open, 'file1')
with pytest.raises(PermissionError):
with open(join(sub_open, 'file1'), 'w') as f:
f.write('abc\n')
assert not isfile(sub_open, 'file1')
with open(join(sub_done, 'file1'), 'w') as f:
pass
assert isfile(sub_done, 'file1')
with open(join(sub_done, 'file2'), 'w') as f:
f.write('abc\n')
assert isfile(sub_done, 'file2')
def deps(self):
q = ('select Deps.mode, Deps.source, %s '
' from Files '
' join Deps on Files.rowid = Deps.source '
' where target=?' % join(', ', _file_cols[1:]))
for row in db().execute(q, [self.id]).fetchall():
mode = row[0]
cols = row[1:]
assert (mode in ('c', 'm'))
yield mode, File(cols=cols)
def deps(self):
q = ('select Deps.mode, Deps.source, %s '
' from Files '
' join Deps on Files.rowid = Deps.source '
' where target=?' % join(', ', _file_cols[1:]))
for row in db().execute(q, [self.id]).fetchall():
mode = row[0]
cols = row[1:]
assert(mode in ('c', 'm'))
yield mode,File(cols=cols)
def _possible_do_files(t):
dirname, filename = os.path.split(t)
yield (os.path.join(vars.BASE,
dirname), "%s.do" % filename, '', filename, '')
# It's important to try every possibility in a directory before resorting
# to a parent directory. Think about nested projects: I don't want
# ../../default.o.do to take precedence over ../default.do, because
# the former one might just be an artifact of someone embedding my project
# into theirs as a subdir. When they do, my rules should still be used
# for building my project in *all* cases.
t = os.path.normpath(os.path.join(vars.BASE, t))
dirname, filename = os.path.split(t)
dirbits = dirname.split('/')
for i in range(len(dirbits), -1, -1):
basedir = join('/', dirbits[:i])
subdir = join('/', dirbits[i:])
for dofile, basename, ext in _default_do_files(filename):
yield (basedir, dofile, subdir, os.path.join(subdir,
basename), ext)
def test_renaming_submission(sub_done, sub_done2, assig_done):
with pytest.raises(FileExistsError):
rename([sub_done], [sub_done2])
with pytest.raises(PermissionError):
rename([sub_done], [assig_done, 'NEW AND WRONG'])
open(join(sub_done, 'hello'), 'w').close()
with pytest.raises(PermissionError):
rename([sub_done, 'hello'], [sub_done2, 'hello'])
def test_quote_paths(sub_done, path):
dirs = path.split('/')[:-1]
if len(dirs):
mkdir(join(sub_done, *dirs))
f_path = join(sub_done, path)
with open(join(sub_done, f_path), 'w') as f:
assert f.write('abc') == 3
tmp_path = join(sub_done, 'abc')
rename([f_path], [tmp_path])
assert not isfile(f_path)
assert isfile(tmp_path)
rename([tmp_path], [f_path])
assert isfile(f_path)
assert not isfile(tmp_path)
rm(f_path)
if len(dirs):
rmdir(join(sub_done, *dirs))
def test_list_submissions(mount_dir, mount):
course_dir = join(mount_dir, 'Programmeertalen')
assert isdir(mount_dir)
assert isdir(course_dir)
for assig in ls(course_dir):
for sub in ls(course_dir, assig):
assert 'Student1' in sub or sub[0] == '.'
mount(assigned_to_me=True)
for assig in ls(course_dir):
for sub in ls(course_dir, assig):
assert 'Student1' in sub or sub[0] == '.'
def test_editing_file_in_fixed(sub_open, mount):
fname = join(sub_open, 'new_test_file')
# Make sure we cannot delete existing files in fixed mode
assert not isfile(fname)
with open(fname, 'w') as f:
f.write('Hello thomas\n')
assert isfile(fname)
mount(fixed=True)
assert isfile(fname)
with pytest.raises(PermissionError):
with open(fname, 'w') as f:
f.write('hello\n')
def test_non_exising_submission(assig_done):
with pytest.raises(FileNotFoundError):
open(join(assig_done, 'NON EXISTING', 'file'), 'x').close()
with pytest.raises(FileNotFoundError):
mkdir(assig_done, 'NON EXISTING', 'file')
with pytest.raises(FileNotFoundError):
rename(
[assig_done, 'NON EXISTING', 'file'],
[assig_done, 'NON EXISTING', 'file2']
)
with pytest.raises(FileNotFoundError):
ls(assig_done, 'NON EXISTING')
def test_feedback_file(sub_done, data):
f_file = join(sub_done, '.cg-feedback')
with open(f_file, 'r') as f:
assert f.read() == ''
with open(f_file, 'w') as f:
f.write(data)
with open(f_file, 'r') as f:
assert f.read() == data
with open(f_file, 'w') as f:
pass
with open(f_file, 'r') as f:
assert f.read() == ''
def test_bug_fsync(mount_dir, sub_done):
# The `fsync` function didn't return the value of `flush` so editing a
# teacher file didn't update the internal id of the file.
p = join(sub_done, 'dir', 'single_file_work')
assert isfile(p)
with open(p, 'r') as f:
old = f.read()
new = 'NEW CONTENT' + old
fd = os.open(p, os.O_WRONLY | os.O_TRUNC)
os.write(fd, bytes(new, 'utf8'))
os.fsync(fd)
os.close(fd)
with open(p, 'r') as f:
assert new == f.read()
def test_renaming_submission(sub_done, sub_done2, assig_done):
with pytest.raises(FileExistsError):
rename([sub_done], [sub_done2])
with pytest.raises(PermissionError):
rename([sub_done], [assig_done, 'NEW AND WRONG'])
open(join(sub_done, 'hello'), 'w').close()
with pytest.raises(PermissionError):
rename([sub_done, 'hello'], [sub_done2, 'hello'])
with pytest.raises(PermissionError):
new = [x for x in sub_done.split('/')]
new[-1] += new[-1]
rename([sub_done], ['/'.join(new)])
def relpath(t, base):
global _cwd
if not _cwd:
_cwd = os.getcwd()
t = os.path.normpath(os.path.join(_cwd, t))
tparts = t.split('/')
bparts = base.split('/')
for tp,bp in zip(tparts,bparts):
if tp != bp:
break
tparts.pop(0)
bparts.pop(0)
while bparts:
tparts.insert(0, '..')
bparts.pop(0)
return join('/', tparts)
def relpath(t, base):
global _cwd
if not _cwd:
_cwd = os.getcwd()
t = os.path.normpath(os.path.join(_cwd, t))
base = os.path.normpath(base)
tparts = t.split('/')
bparts = base.split('/')
for tp, bp in zip(tparts, bparts):
if tp != bp:
break
tparts.pop(0)
bparts.pop(0)
while bparts:
tparts.insert(0, '..')
bparts.pop(0)
return join('/', tparts)
def test_double_open(sub_done, mount, fixed):
fname = join(sub_done, 'new_test_file')
f = open(fname, 'wb')
f.write(b'hello')
ff = open(fname, 'r+b')
fff = open(fname, 'r+b')
f.flush()
assert ff.read() == b'hello'
f.close()
assert fff.read() == b'hello'
ff.close()
fff.close()
def __init__(self, stamp=None, csum=None, auto_detect=None, st=None, runid=None):
assert(stamp == None or isinstance(stamp, str))
assert(csum == None or isinstance(csum, str))
self.stamp = stamp
self.csum = csum
if auto_detect:
if len(auto_detect) == 40 and auto_detect.isalnum():
self.csum = auto_detect
else:
self.stamp = auto_detect
elif st != None:
if st == False:
self.stamp = STAMP_MISSING
elif stat.S_ISDIR(st.st_mode):
self.stamp = STAMP_DIR
else:
self.stamp = join('-', (st.st_ctime, st.st_mtime,
st.st_size, st.st_dev, st.st_ino))
if runid:
self.stamp = self.stamp + '+' + str(int(runid))
def files():
q = ('select %s from Files order by name' % join(', ', _file_cols))
for cols in db().execute(q).fetchall():
yield File(cols=cols)
