def test_deleting_directory_in_fixed(sub_open, mount): fdir = join(sub_open, 'new_test_file') fdir2 = join(sub_open, 'new_test_file2') fdir3 = join(sub_open, 'new_test_file3') # Make sure we cannot delete existing files in fixed mode assert not isdir(fdir) mkdir(fdir) assert isdir(fdir) mount(fixed=True) assert isdir(fdir) with pytest.raises(PermissionError): rmdir(fdir) assert isdir(fdir) del fdir assert not isdir(fdir2) mkdir(fdir2) assert isdir(fdir2) rename([fdir2], [fdir3]) assert not isdir(fdir2) assert isdir(fdir3) mount(fixed=True) assert not isdir(fdir3)
def test_set_utime(sub_done, mount): fname = join(sub_done, 'new_test_file') fname2 = join(sub_done, 'new_test_file2') open(fname, 'w').close() old_st = os.lstat(fname) os.utime(fname, (10, 123.5)) new_st = os.lstat(fname) assert new_st.st_atime == 10.0 assert new_st.st_mtime == 123.5 assert old_st.st_atime != 10.0 assert old_st.st_mtime != 123.5 mount(fixed=True) with pytest.raises(PermissionError): os.utime(join(sub_done, 'dir', 'single_file_work'), (100, 1235)) open(fname2, 'w').close() old_st = os.lstat(fname2) os.utime(fname2, (10, 123.5)) new_st = os.lstat(fname2) assert new_st.st_atime == 10.0 assert new_st.st_mtime == 123.5 assert old_st.st_atime != 10.0 assert old_st.st_mtime != 123.5
def test_renaming_file_in_fixed(sub_open, mount): fname = join(sub_open, 'new_test_file') fname2 = join(sub_open, 'new_test_file2') fname3 = join(sub_open, 'new_test_file3') # Make sure we cannot delete existing files in fixed mode assert not isfile(fname) with open(fname, 'w') as f: f.write('Hello thomas\n') assert isfile(fname) mount(fixed=True) assert isfile(fname) with pytest.raises(PermissionError): rename([fname], [fname2]) del fname assert not isfile(fname2) with open(fname2, 'w') as f: f.write('Hello thomas\n') assert isfile(fname2) rename([fname2], [fname3]) assert not isfile(fname2) assert isfile(fname3) with open(fname3, 'r') as f: assert f.read() == 'Hello thomas\n' mount(fixed=True) assert not isfile(fname3)
def _default_do_files(filename): l = filename.split('.') for i in range(1,len(l)+1): basename = join('.', l[:i]) ext = join('.', l[i:]) if ext: ext = '.' + ext yield ("default%s.do" % ext), basename, ext
def _default_do_files(filename): l = filename.split('.') for i in range(1, len(l) + 1): basename = join('.', l[:i]) ext = join('.', l[i:]) if ext: ext = '.' + ext yield ("default%s.do" % ext), basename, ext
def test_compiling_program(sub_done, mount, fixed): url = 'https://attach.libremail.nl/__test_codegra.fs__.tar.gz' fname = join(sub_done, '42.tar.gz') fdir = join(sub_done, '42sh/') urllib.request.urlretrieve(url, fname) tar = tarfile.open(fname, "r:gz") tar.extractall(sub_done) tar.close() print(subprocess.check_output(['make', '-C', fdir])) assert isdir(fdir) assert isfile(fname) assert subprocess.check_output( [join(fdir, '42sh'), '-c', 'echo hello from 42'] ).decode('utf-8') == 'hello from 42\n' mount() if fixed: assert not isdir(fdir) assert not isfile(fname) return assert isfile(fname) assert isdir(fdir) assert subprocess.check_output( [join(fdir, '42sh'), '-c', 'echo hello from 42'] ).decode('utf-8') == 'hello from 42\n' rm_rf(fdir) assert not isdir(fdir) mount() assert isfile(fname) assert not isdir(fdir)
def trainTree(tDir='.'): tFile = open(join(tDir, 'trainData.csv'), 'r') lFile = open(join(tDir, 'trainLabels.csv'), 'r') try: nFile = open(join(tDir, 'attrNames.csv'), 'r') except: nFile = None print tDir + ':' tree.train(tFile, lFile, nFile, maxEnt, maxPValue, maxSamples, maxAttrs)
def test_truncate_file(sub_done): with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f: old = f.read() with open(join(sub_done, 'dir', 'single_file_work'), 'a') as f: f.write('wow\n') with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f: assert f.read() == old + 'wow\n' with open(join(sub_done, 'dir', 'single_file_work'), 'a') as f: assert f.truncate(len(old)) with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f: assert f.read() == old with open(join(sub_done, 'dir', 'single_file_work'), 'a') as f: assert f.truncate(len(old) * 3) with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f: new = f.read() assert len(new) == len(old) * 3 new.startswith(old) all(n == '\0' for n in new[len(old):]) fd = os.open( join(sub_done, 'dir', 'single_file_work'), os.O_TRUNC | os.O_WRONLY ) os.write(fd, b'hello\n') os.close(fd) with open(join(sub_done, 'dir', 'single_file_work'), 'r') as f: assert f.read() == 'hello\n'
def test_deleting_file_in_fixed(sub_open, mount): fname = join(sub_open, 'new_test_file') fname2 = join(sub_open, 'new_test_file2') # Make sure we cannot delete existing files in fixed mode assert not isfile(fname) with open(fname, 'w') as f: f.write('Hello thomas\n') assert isfile(fname) mount(fixed=True) assert isfile(fname) with open(fname, 'r') as f: assert f.read() == 'Hello thomas\n' with pytest.raises(PermissionError): with open(fname, 'w') as f: f.write('Hello thomas\n') with pytest.raises(PermissionError): rm(fname) del fname # Now make sure we can delete files that did not exist assert not isfile(fname2) with open(fname2, 'w') as f: f.write('Hello thomas2\n') with open(fname2, 'r') as f: assert f.read() == 'Hello thomas2\n' assert isfile(fname2) rm(fname2) assert not isfile(fname2) # Make sure files created in fixed mode are not visible after a remount assert not isfile(fname2) with open(fname2, 'w') as f: f.write('Hello thomas2\n') with open(fname2, 'r') as f: assert f.read() == 'Hello thomas2\n' assert isfile(fname2) mount(fixed=True) assert not isfile(fname2) with open(fname2, 'w') as f: f.write('Hello thomas2\n') with open(fname2, 'r') as f: assert f.read() == 'Hello thomas2\n' assert isfile(fname2) mount(fixed=True) assert not isfile(fname2)
def test_delete_files(mount_dir, sub_open, sub_done): with open(join(sub_open, 'file1'), 'w') as f: f.write('abc\n') assert isfile(sub_open, 'file1') with open(join(sub_open, 'file1'), 'r') as f: assert f.read() == 'abc\n' rm(sub_open, 'file1') assert not isfile(sub_open, 'file1') with pytest.raises(FileNotFoundError): rm(sub_open, 'nonexisting')
def test_create_invalid_file(mount_dir): with pytest.raises(PermissionError): with open(join(mount_dir, 'file'), 'w+') as f: f.write('hello\n') with pytest.raises(PermissionError): with open( join( mount_dir, [l for l in ls(mount_dir) if isdir(mount_dir, l)][0], 'file' ), 'w+' ) as f: f.write('hello\n')
def test_rename(mount_dir, sub_done, sub_open): assert isdir(sub_done, 'dir') assert not isdir(sub_done, 'dir33') assert isdir(sub_done, 'dir') assert not isdir(sub_done, 'dir33') mkdir(sub_done, 'dir33') mkdir(sub_done, 'dir', 'sub_dir') assert isdir(sub_done, 'dir33') with open(join(sub_done, 'dir33', 'new_file'), 'w') as f: f.write('bye\n') rename([sub_done, 'dir33'], [sub_done, 'dir', 'sub_dir', 'dir33']) with open(join(sub_done, 'dir', 'sub_dir', 'dir33', 'new_file'), 'r') as f: assert f.read() == 'bye\n'
def read_stamp(self, runid=None, st=None, st_deps=None): # FIXME: make this formula more well-defined if runid is None: if st_deps == None: try: st_deps = os.stat(self.tmpfilename('deps')) except OSError: st_deps = False if st_deps == False: runid_suffix = '' else: runid_suffix = '+' + str(int(st_deps.st_mtime)) else: runid_suffix = '+' + str(int(runid)) if st == None: try: st = os.stat(self.name) except OSError: st = False if st == False: return STAMP_MISSING + runid_suffix if stat.S_ISDIR(st.st_mode): return STAMP_DIR + runid_suffix else: # a "unique identifier" stamp for a regular file return join('-', (st.st_ctime, st.st_mtime, st.st_size, st.st_ino)) + runid_suffix
def _init_from_idname(self, id, name): q = ('select %s from Files ' % join(', ', _file_cols)) if id != None: q += 'where rowid=?' l = [id] elif name != None: name = (name == ALWAYS) and ALWAYS or relpath(name, vars.BASE) q += 'where name=?' l = [name] else: raise Exception('name or id must be set') d = db() row = d.execute(q, l).fetchone() if not row: if not name: raise Exception('No file with id=%r name=%r' % (id, name)) try: _write('insert into Files (name) values (?)', [name]) except sqlite3.IntegrityError: # some parallel redo probably added it at the same time; no # big deal. pass row = d.execute(q, l).fetchone() assert (row) return self._init_from_cols(row)
def test_difficult_assigned_to( mount, teacher_jwt, student_jwt, teacher_id, ta_id, assig_open, student2_jwt ): assig_id = open(join(assig_open, '.cg-assignment-id')).read().strip() def submit(jwt): req = requests.post( 'http://localhost:5000/api/v1/assignments/{}/submission'. format(assig_id), headers={'Authorization': 'Bearer ' + jwt}, files=dict( file=open('./test_data/multiple_dir_archive.zip', 'rb') ) ) req.raise_for_status() return req.json() def assign(sub, user_id): requests.patch( 'http://localhost:5000/api/v1/submissions/{}/grader'.format( sub["id"] ), json={ 'user_id': user_id }, headers={ 'Authorization': 'Bearer ' + teacher_jwt }, ).raise_for_status sub2 = submit(student2_jwt) old = submit(student_jwt) new = submit(student_jwt) print(sub2, old, new) assign(sub2, ta_id) assign(old, ta_id) assign(new, teacher_id) mount(assigned_to_me=True) # We do not expect to see any submission. A submission is assigned to us, # but it is not the newest of the student, so we do not show it. print(ls(assig_open)) assert not any(['Student1' in l for l in ls(assig_open)]) assert any(['Student2' in l for l in ls(assig_open)]) mount(assigned_to_me=False) assert ls(assig_open) # Now we assign the newest submission to us, so we expect to see it. requests.patch( 'http://localhost:5000/api/v1/submissions/{}/grader'.format(new["id"]), json={'user_id': ta_id}, headers={'Authorization': 'Bearer ' + teacher_jwt}, ) mount(assigned_to_me=True) print(ls(assig_open)) assert any(['Student1' in l for l in ls(assig_open)]) assert any(['Student2' in l for l in ls(assig_open)])
def _init_from_idname(self, id, name): q = ('select %s from Files ' % join(', ', _file_cols)) if id != None: q += 'where rowid=?' l = [id] elif name != None: name = (name==ALWAYS) and ALWAYS or relpath(name, vars.BASE) q += 'where name=?' l = [name] else: raise Exception('name or id must be set') d = db() row = d.execute(q, l).fetchone() if not row: if not name: raise Exception('File with id=%r not found and ' 'name not given' % id) try: _write('insert into Files (name) values (?)', [name]) except sqlite3.IntegrityError: # some parallel redo probably added it at the same time; no # big deal. pass row = d.execute(q, l).fetchone() assert(row) return self._init_from_cols(row)
def _interpreter_locations(dodir): dodir = os.path.realpath(dodir) dirbits = dodir.split("/") for i in range(len(dirbits), -1, -1): d = join("/", dirbits[:i]) yield (d) yield (d + "/do")
def _interpreter_locations(dodir): dodir = os.path.realpath(dodir) dirbits = dodir.split('/') for i in range(len(dirbits), -1, -1): d = join('/', dirbits[:i]) yield(d) yield(os.path.join(d, "do"))
def test_force_create_file(sub_done): with pytest.raises(FileExistsError): fd = os.open( join(sub_done, 'dir', 'single_file_work'), os.O_CREAT | os.O_EXCL | os.O_WRONLY ) os.write(fd, b'WRONG\n') os.close(fd)
def test_write_to_directory(sub_done): fdir = join(sub_done, 'new_dir') fname = join(fdir, 'new_file') mkdir(fdir) assert isdir(fdir) assert not isfile(fdir) with pytest.raises(IsADirectoryError): with open(fdir, 'w') as f: f.write('hello\n') open(fname, 'w').close() with pytest.raises(NotADirectoryError): ls(fname) with pytest.raises(FileExistsError): mkdir(fname)
def test_list_assignments(mount_dir): course_dir = join(mount_dir, 'Programmeertalen') assert isdir(mount_dir) assert isdir(course_dir) for assig in ['Haskell', 'Go', 'Python', 'Shell']: assert isdir(course_dir, assig)
def _possible_do_files(t): dirname, filename = os.path.split(t) yield (dirname, "%s.do" % filename, "", filename, "") yield (dirname, "%s.do" % filename, "", filename, "") # It's important to try every possibility in a directory before resorting # to a parent directory. Think about nested projects: I don't want # ../../default.o.do to take precedence over ../default.do, because # the former one might just be an artifact of someone embedding my project # into theirs as a subdir. When they do, my rules should still be used # for building my project in *all* cases. dirbits = os.path.abspath(dirname).split("/") for i in range(len(dirbits), -1, -1): basedir = os.path.join(dirname, join("/", [".."] * (len(dirbits) - i))) subdir = join("/", dirbits[i:]) for dofile, basename, ext in _default_do_files(filename): yield (basedir, dofile, subdir, os.path.join(subdir, basename), ext)
def save(self): cols = join(', ', ['%s=?' % i for i in _file_cols[2:]]) _write('update Files set ' ' %s ' ' where rowid=?' % cols, [ self.is_generated, self.is_override, self.checked_runid, self.changed_runid, self.failed_runid, self.stamp, self.csum, self.id ])
def save(self): cols = join(', ', ['%s=?'%i for i in _file_cols[2:]]) _write('update Files set ' ' %s ' ' where rowid=?' % cols, [self.is_generated, self.is_override, self.checked_runid, self.changed_runid, self.failed_runid, self.stamp, self.csum, self.id])
def test_create_files(mount_dir, sub_open, sub_done): with pytest.raises(PermissionError): with open(join(sub_open, 'file1'), 'w') as f: pass assert not isfile(sub_open, 'file1') with pytest.raises(PermissionError): with open(join(sub_open, 'file1'), 'w') as f: f.write('abc\n') assert not isfile(sub_open, 'file1') with open(join(sub_done, 'file1'), 'w') as f: pass assert isfile(sub_done, 'file1') with open(join(sub_done, 'file2'), 'w') as f: f.write('abc\n') assert isfile(sub_done, 'file2')
def deps(self): q = ('select Deps.mode, Deps.source, %s ' ' from Files ' ' join Deps on Files.rowid = Deps.source ' ' where target=?' % join(', ', _file_cols[1:])) for row in db().execute(q, [self.id]).fetchall(): mode = row[0] cols = row[1:] assert (mode in ('c', 'm')) yield mode, File(cols=cols)
def deps(self): q = ('select Deps.mode, Deps.source, %s ' ' from Files ' ' join Deps on Files.rowid = Deps.source ' ' where target=?' % join(', ', _file_cols[1:])) for row in db().execute(q, [self.id]).fetchall(): mode = row[0] cols = row[1:] assert(mode in ('c', 'm')) yield mode,File(cols=cols)
def _possible_do_files(t): dirname, filename = os.path.split(t) yield (os.path.join(vars.BASE, dirname), "%s.do" % filename, '', filename, '') # It's important to try every possibility in a directory before resorting # to a parent directory. Think about nested projects: I don't want # ../../default.o.do to take precedence over ../default.do, because # the former one might just be an artifact of someone embedding my project # into theirs as a subdir. When they do, my rules should still be used # for building my project in *all* cases. t = os.path.normpath(os.path.join(vars.BASE, t)) dirname, filename = os.path.split(t) dirbits = dirname.split('/') for i in range(len(dirbits), -1, -1): basedir = join('/', dirbits[:i]) subdir = join('/', dirbits[i:]) for dofile, basename, ext in _default_do_files(filename): yield (basedir, dofile, subdir, os.path.join(subdir, basename), ext)
def test_renaming_submission(sub_done, sub_done2, assig_done): with pytest.raises(FileExistsError): rename([sub_done], [sub_done2]) with pytest.raises(PermissionError): rename([sub_done], [assig_done, 'NEW AND WRONG']) open(join(sub_done, 'hello'), 'w').close() with pytest.raises(PermissionError): rename([sub_done, 'hello'], [sub_done2, 'hello'])
def test_quote_paths(sub_done, path): dirs = path.split('/')[:-1] if len(dirs): mkdir(join(sub_done, *dirs)) f_path = join(sub_done, path) with open(join(sub_done, f_path), 'w') as f: assert f.write('abc') == 3 tmp_path = join(sub_done, 'abc') rename([f_path], [tmp_path]) assert not isfile(f_path) assert isfile(tmp_path) rename([tmp_path], [f_path]) assert isfile(f_path) assert not isfile(tmp_path) rm(f_path) if len(dirs): rmdir(join(sub_done, *dirs))
def test_list_submissions(mount_dir, mount): course_dir = join(mount_dir, 'Programmeertalen') assert isdir(mount_dir) assert isdir(course_dir) for assig in ls(course_dir): for sub in ls(course_dir, assig): assert 'Student1' in sub or sub[0] == '.' mount(assigned_to_me=True) for assig in ls(course_dir): for sub in ls(course_dir, assig): assert 'Student1' in sub or sub[0] == '.'
def test_editing_file_in_fixed(sub_open, mount): fname = join(sub_open, 'new_test_file') # Make sure we cannot delete existing files in fixed mode assert not isfile(fname) with open(fname, 'w') as f: f.write('Hello thomas\n') assert isfile(fname) mount(fixed=True) assert isfile(fname) with pytest.raises(PermissionError): with open(fname, 'w') as f: f.write('hello\n')
def test_non_exising_submission(assig_done): with pytest.raises(FileNotFoundError): open(join(assig_done, 'NON EXISTING', 'file'), 'x').close() with pytest.raises(FileNotFoundError): mkdir(assig_done, 'NON EXISTING', 'file') with pytest.raises(FileNotFoundError): rename( [assig_done, 'NON EXISTING', 'file'], [assig_done, 'NON EXISTING', 'file2'] ) with pytest.raises(FileNotFoundError): ls(assig_done, 'NON EXISTING')
def test_feedback_file(sub_done, data): f_file = join(sub_done, '.cg-feedback') with open(f_file, 'r') as f: assert f.read() == '' with open(f_file, 'w') as f: f.write(data) with open(f_file, 'r') as f: assert f.read() == data with open(f_file, 'w') as f: pass with open(f_file, 'r') as f: assert f.read() == ''
def test_bug_fsync(mount_dir, sub_done): # The `fsync` function didn't return the value of `flush` so editing a # teacher file didn't update the internal id of the file. p = join(sub_done, 'dir', 'single_file_work') assert isfile(p) with open(p, 'r') as f: old = f.read() new = 'NEW CONTENT' + old fd = os.open(p, os.O_WRONLY | os.O_TRUNC) os.write(fd, bytes(new, 'utf8')) os.fsync(fd) os.close(fd) with open(p, 'r') as f: assert new == f.read()
def test_renaming_submission(sub_done, sub_done2, assig_done): with pytest.raises(FileExistsError): rename([sub_done], [sub_done2]) with pytest.raises(PermissionError): rename([sub_done], [assig_done, 'NEW AND WRONG']) open(join(sub_done, 'hello'), 'w').close() with pytest.raises(PermissionError): rename([sub_done, 'hello'], [sub_done2, 'hello']) with pytest.raises(PermissionError): new = [x for x in sub_done.split('/')] new[-1] += new[-1] rename([sub_done], ['/'.join(new)])
def relpath(t, base): global _cwd if not _cwd: _cwd = os.getcwd() t = os.path.normpath(os.path.join(_cwd, t)) tparts = t.split('/') bparts = base.split('/') for tp,bp in zip(tparts,bparts): if tp != bp: break tparts.pop(0) bparts.pop(0) while bparts: tparts.insert(0, '..') bparts.pop(0) return join('/', tparts)
def relpath(t, base): global _cwd if not _cwd: _cwd = os.getcwd() t = os.path.normpath(os.path.join(_cwd, t)) base = os.path.normpath(base) tparts = t.split('/') bparts = base.split('/') for tp, bp in zip(tparts, bparts): if tp != bp: break tparts.pop(0) bparts.pop(0) while bparts: tparts.insert(0, '..') bparts.pop(0) return join('/', tparts)
def test_double_open(sub_done, mount, fixed): fname = join(sub_done, 'new_test_file') f = open(fname, 'wb') f.write(b'hello') ff = open(fname, 'r+b') fff = open(fname, 'r+b') f.flush() assert ff.read() == b'hello' f.close() assert fff.read() == b'hello' ff.close() fff.close()
def __init__(self, stamp=None, csum=None, auto_detect=None, st=None, runid=None): assert(stamp == None or isinstance(stamp, str)) assert(csum == None or isinstance(csum, str)) self.stamp = stamp self.csum = csum if auto_detect: if len(auto_detect) == 40 and auto_detect.isalnum(): self.csum = auto_detect else: self.stamp = auto_detect elif st != None: if st == False: self.stamp = STAMP_MISSING elif stat.S_ISDIR(st.st_mode): self.stamp = STAMP_DIR else: self.stamp = join('-', (st.st_ctime, st.st_mtime, st.st_size, st.st_dev, st.st_ino)) if runid: self.stamp = self.stamp + '+' + str(int(runid))
def files(): q = ('select %s from Files order by name' % join(', ', _file_cols)) for cols in db().execute(q).fetchall(): yield File(cols=cols)