def write_file() :
'''
Write content to a file (overwrites original; creates new file)
Request
pathname: Linux-style path and name of file
content: base64 encoded content
Response
length: Length written to file
pathname: same as in request
'''
pathname = get_pathname()
b64content = request.args.get('content')
if b64content is None :
return json_error('Please supply content using HTTP GET')
try :
content = base64.decodebytes(b64content.encode())
except binascii.Error :
return json_error('Content cannot be base64 decoded')
directory = os.path.dirname(pathname)
if not os.path.exists(directory) :
return json_error('Directory %s does not exist' % repr(directory))
try :
f = open(pathname, 'wb')
nbytes = f.write(content)
f.close()
except PermissionError :
return json_error('Permission Denied')
except Exception :
return json_error('Exception: %s' % repr(traceback.format_exc()))
return json_success('Written %d bytes to %s' % (nbytes, repr(pathname)),
length=nbytes, pathname=pathname)
def status() :
'''
Display file status (permission, user, etc), wrapper for os.stat
Request
pathname: Linux-style path of file or directory
Response
mode: file mode in integer
uid: user id in integer
gid: group id in integer
size: size in integer
atime: access time in float
mtime: modify time in float
ctime: change time in float
'''
pathname = get_pathname()
parent = request.args.get('parent', '0') == '1'
if not os.path.exists(pathname) :
return json_error('pathname does not exist')
try :
stat = os.stat(pathname)
except Exception as e :
return json_error(e.strerror)
return json_success(mode=stat.st_mode,
uid=stat.st_uid,
gid=stat.st_gid,
size=stat.st_size,
atime=stat.st_atime,
mtime=stat.st_mtime,
ctime=stat.st_ctime)
def copy_tree() :
'''
Copy a tree using shutil.copytree
Call convention: (src, dst, ignore=shutil.ignore_patterns(*ignore))
Request
src: A Linux-style path of file or directory
dst: A Linux-style path of file or directory
ignore: list of json strings, passed to ignore_patterns(); optional
Response
(none)
'''
src = get_pathname('src')
dst = get_pathname('dst')
ignore = request.args.get('ignore', None)
try :
print(repr(ignore))
if ignore is not None :
ignore = shutil.ignore_patterns(*json.loads(ignore))
shutil.copytree(src, dst, ignore=ignore)
except PermissionError :
return json_error('Permission Denied')
except FileExistsError :
return json_error('File exists')
except FileNotFoundError :
return json_error('No such file or directory')
except Exception :
return json_error('Exception: %s' % repr(traceback.format_exc()))
return json_success()
def globbing() : ''' Return a list of paths matching a pathname pattern Request pathname: A pattern that is passed into glob.glob Response content: json list of direcotry content ''' pathname = get_pathname(escape=glob.escape) return json_success(content=glob.glob(pathname))
def os_path_exists() : ''' Return whether a path is a directory Request pathname: A Linux-style path of file or directory Response exists: true or false indicating result ''' pathname = get_pathname() return json_success(exists=os.path.exists(pathname))
def is_mount() : ''' Return whether a path is a directory Request pathname: A Linux-style path of file or directory Response ismount: true or false indicating result ''' pathname = get_pathname() return json_success(ismount=os.path.ismount(pathname))
def list_assignments(db, course_id):
'''
GET /api/assignments/<course_id>
List all assignments for a course (students+instructors)
'''
user = get_user(db)
course = find_course(db, course_id)
check_course_user(db, course, user)
assignments = course.assignments
return json_success(assignments=list(map(lambda x: x.id, assignments)))
def download_assignment(db, course_id, assignment_id):
'''
GET /api/assignment/<course_id>/<assignment_id>
Download a copy of an assignment (students+instructors)
'''
user = get_user(db)
course = find_course(db, course_id)
check_course_user(db, course, user)
assignment = find_assignment(db, course, assignment_id)
list_only = request.args.get('list_only', 'false') == 'true'
return json_success(files=json_files_pack(assignment.files, list_only))
def add_new_book():
filename = "books_2.json"
with open(filename, 'r') as json_data:
temp = json.load(json_data)
counter = 0
for obj in temp:
Books().create(obj)
counter += 1
return json_success(results=counter)
def add_course(db, course_id):
'''
POST /api/course/<course_id>
Add a course (anyone)
'''
user = get_user(db)
if db.query(Course).filter(Course.id == course_id).one_or_none():
raise JsonError('Course already exists')
course = Course(course_id, user)
db.add(course)
db.commit()
return json_success()
def list_courses(db):
'''
GET /api/courses
List all available courses the user is taking or teaching (anyone)
'''
user = get_user(db)
courses = set()
for i in user.teaching:
courses.add(i.id)
for i in user.taking:
courses.add(i.id)
return json_success(courses=sorted(courses))
def submit_assignment(db, course_id, assignment_id):
'''
POST /api/submission/<course_id>/<assignment_id>
Submit a copy of an assignment (students+instructors)
'''
user = get_user(db)
course = find_course(db, course_id)
check_course_user(db, course, user)
assignment = find_assignment(db, course, assignment_id)
submission = Submission(user, assignment)
json_files_unpack(request.form.get('files'), submission.files)
db.commit()
return json_success()
def list_directory() :
'''
List a directory
Request
pathname: Linux-style path of directory
Response
content: json list of direcotry content
'''
pathname = get_pathname()
if not os.path.exists(pathname) :
return json_error('pathname does not exist')
if not os.path.isdir(pathname) :
return json_error('pathname is not directory')
return json_success(content=os.listdir(pathname))
def os_walk() : ''' Return a list of paths matching a pathname pattern Request pathname: A Linux-style path of file or directory Response content: json list of os.walk content ''' # Maybe not safe when FS_PREFIX != '/' pathname = get_pathname() ans = [] for p, d, f in os.walk(pathname): ans.append((remove_pathname(p), d, f)) return json_success(content=ans)
def release_assignment(db, course_id, assignment_id):
'''
POST /api/assignment/<course_id>/<assignment_id>
Release an assignment (instructors only)
'''
user = get_user(db)
course = find_course(db, course_id)
check_course_instructor(db, course, user)
if db.query(Assignment).filter(Assignment.id == assignment_id,
Assignment.course == course).one_or_none():
raise JsonError('Assignment already exists')
assignment = Assignment(assignment_id, course)
json_files_unpack(request.form.get('files'), assignment.files)
db.commit()
return json_success()
def download_submission(db, course_id, assignment_id, student_id):
'''
GET /api/submission/<course_id>/<assignment_id>/<student_id>
Download a student's submitted assignment (instructors only)
TODO: maybe allow student to see their own submissions?
'''
user = get_user(db)
course = find_course(db, course_id)
check_course_instructor(db, course, user)
assignment = find_assignment(db, course, assignment_id)
student = find_course_user(db, course, student_id)
submission = find_student_latest_submission(db, assignment, student)
list_only = request.args.get('list_only', 'false') == 'true'
return json_success(files=json_files_pack(submission.files, list_only),
timestamp=strftime(submission.timestamp))
def copy_file() :
'''
Copy a file
Request
src: A Linux-style path of file or directory
dst: A Linux-style path of file or directory
Response
(none)
'''
src = get_pathname('src')
dst = get_pathname('dst')
try :
shutil.copy(src, dst)
except PermissionError :
return json_error('Permission Denied')
except Exception :
return json_error('Exception: %s' % repr(traceback.format_exc()))
return json_success()
def list_submissions(db, course_id, assignment_id):
'''
GET /api/submissions/<course_id>/<assignment_id>
List all submissions for an assignment from all students
(instructors only)
'''
user = get_user(db)
course = find_course(db, course_id)
check_course_instructor(db, course, user)
assignment = find_assignment(db, course, assignment_id)
submissions = []
for submission in assignment.submissions:
submissions.append({
'student_id': submission.student.id,
'timestamp': strftime(submission.timestamp),
# TODO: "notebooks": [],
})
return json_success(submissions=submissions)
def remove_tree() :
'''
Copy a tree using shutil.copytree
Call convention: (src, dst, ignore=shutil.ignore_patterns(*ignore))
Request
pathname: A Linux-style path of file or directory
Response
(none)
'''
pathname = get_pathname('pathname')
try :
shutil.rmtree(pathname)
except PermissionError :
return json_error('Permission Denied')
except FileNotFoundError :
return json_error('No such file or directory')
except Exception :
return json_error('Exception: %s' % repr(traceback.format_exc()))
return json_success()
def download_feedback(db, course_id, assignment_id, student_id):
'''
GET /api/feedback/<course_id>/<assignment_id>/<student_id>
Download feedback on a student's assignment
(instructors+students, students restricted to their own submissions)
'''
user = get_user(db)
course = find_course(db, course_id)
if user.id != student_id:
check_course_instructor(db, course, user)
assignment = find_assignment(db, course, assignment_id)
student = find_course_user(db, course, student_id)
if 'timestamp' not in request.args:
raise JsonError('Please supply timestamp')
timestamp = strptime(request.args.get('timestamp'))
submission = find_student_submission(db, assignment, student, timestamp)
list_only = request.args.get('list_only', 'false') == 'true'
return json_success(files=json_files_pack(submission.feedbacks, list_only),
timestamp=strftime(submission.timestamp))
def upload_feedback(db, course_id, assignment_id, student_id):
'''
POST /api/feedback/<course_id>/<assignment_id>/<student_id>
Upload feedback on a student's assignment (instructors only)
'''
user = get_user(db)
course = find_course(db, course_id)
check_course_instructor(db, course, user)
assignment = find_assignment(db, course, assignment_id)
student = find_course_user(db, course, student_id)
if 'timestamp' not in request.form:
raise JsonError('Please supply timestamp')
timestamp = strptime(request.form.get('timestamp'))
submission = find_student_submission(db, assignment, student, timestamp)
submission.feedbacks.clear()
# TODO: does this automatically remove the files?
json_files_unpack(request.form.get('files'), submission.feedbacks)
db.commit()
return json_success()
def read_file() :
'''
Read the content of a file
Request
pathname: Linux-style path and name of file
Response
content: base64 encoded content of file
'''
pathname = get_pathname()
if not os.path.exists(pathname) :
return json_error('File %s does not exist' % repr(pathname))
try :
f = open(pathname, 'rb')
content = f.read()
f.close()
except PermissionError :
return json_error('Permission Denied')
except Exception :
return json_error('Exception: %s' % repr(traceback.format_exc()))
return json_success(content=base64.encodebytes(content).decode())
def list_student_submission(db, course_id, assignment_id, student_id):
'''
GET /api/submissions/<course_id>/<assignment_id>/<student_id>
List all submissions for an assignment from a particular student
(instructors+students, students restricted to their own submissions)
'''
user = get_user(db)
course = find_course(db, course_id)
if user.id != student_id:
check_course_instructor(db, course, user)
assignment = find_assignment(db, course, assignment_id)
student = find_course_user(db, course, student_id)
submissions = []
for submission in find_student_submissions(db, assignment, student):
submissions.append({
'student_id': submission.student.id,
'timestamp': strftime(submission.timestamp),
# TODO: "notebooks": [],
})
return json_success(submissions=submissions)
def make_directory() :
'''
Make a directory
Request
pathname: Linux-style path of directory
parent: (optional) 0 or 1, whether mkdir parent directories
Response
(none)
'''
pathname = get_pathname()
parent = request.args.get('parent', '0') == '1'
if os.path.exists(pathname) :
return json_error('File exists')
try :
if parent :
os.makedirs(pathname)
else :
os.mkdir(pathname)
except Exception as e :
return json_error(e.strerror)
return json_success()
def change_mode() :
'''
Change mode of file or directory
Request
pathname: Linux-style path of file or directory
mode: file mode in integer
Response
(none)
'''
pathname = get_pathname()
parent = request.args.get('parent', '0') == '1'
if not os.path.exists(pathname) :
return json_error('pathname does not exist')
try :
mode = int(request.args.get('mode'))
except ValueError :
return json_error('Please supply mode in integer using HTTP GET')
try :
os.chmod(pathname, mode)
except Exception as e :
return json_error(e.strerror)
return json_success()
def add_authors():
conn = sqlite3.connect('nlp_learn.db')
filename = "goodreads_book_authors.json"
with open(filename, 'r') as json_data:
temp = json.load(json_data)
commit_counter = 0
total_data_count = 0
for row in temp:
query = "INSERT INTO authors VALUES ({0}, '{1}')".format(
int(row["author_id"]), row["name"].replace("'", ""))
print(query)
conn.execute(query)
commit_counter += 1
if commit_counter == 1000:
total_data_count += commit_counter
commit_counter = 0
conn.commit()
conn.commit()
conn.close()
return json_success(results=total_data_count)
def train_book_model():
""" Fetching all the documnets i.e. book titles - Start """
documents = fetch_all_titles()
""" Fetching all the documnets i.e. book titles - Complete"""
np.random.seed(2018)
""" Downloading package wordnet - Start"""
nltk.download('wordnet')
""" Downloading package wordnet - Complete """
processed_docs = documents['title'].map(preprocess)
dictionary = gensim.corpora.Dictionary(processed_docs)
dictionary.filter_extremes(no_below=15, no_above=0.5, keep_n=100000)
bow_docs = [dictionary.doc2bow(doc) for doc in processed_docs]
""" Running LDA using Bag of Words """
lda_model = gensim.models.LdaMulticore(bow_docs,
num_topics=25,
id2word=dictionary,
passes=2,
workers=2)
lda_model.save('trained_model/lda_train_bow.model')
""" LDA Training - Complete """
return json_success(results="Training Complete, files saved!")
