def grading_list_collector(): sha_tag, dt_tag = extract_commit_datetime(tag) if sha_tag is None: logger.debug("This assignment hasn't been graded yet") self.ungraded.append(os.getcwd()) return True sha_head, dt_head = extract_commit_datetime("head") if sha_head == sha_tag: logger.debug( "SHA's for commits matched\n\tGRADED MOST RECENT SUBMISSION" ) self.graded.append(os.getcwd()) elif dt_tag < dt_head: logger.debug( "Instructor feedback was tagged then more work was submitted" ) self.new_student_work_since_grading.append(os.getcwd()) else: print_error("This directory has graded feedback, "\ "but the most recent commit is prior to the instructor's"\ " feedback commit & tag. This might indicate a problem"\ " with a timezone on the server\n\t"+\ os.getcwd()) return True
def git_do_core(self, root_dir, git_cmds): """Searches for git repos in & under <root_dir>, and then invokes each of the commands listed in <git_cmds> on the repo.""" logger.info("Walking through " + root_dir) cwd_prev = os.getcwd() for current_dir, dirs, files in os.walk(root_dir): for dir in dirs: if dir == ".git": os.chdir(current_dir) # make the directory more readable by truncating # the common root local_dir = current_dir.replace(root_dir, "") logger.info( Fore.LIGHTGREEN_EX + "Found repo at "\ + local_dir + Style.RESET_ALL + "\n") for git_cmd in git_cmds: if callable(git_cmd): if not git_cmd(): break # we're done here elif isinstance(git_cmd, basestring): call_shell(git_cmd, exit_on_fail=False) else: print_error( "This is neither a string nor a function:\n\t"\ +str(git_cmd)) #print "="*20 + "\n" # note that we don't restore the current # working dir between git commands! os.chdir(cwd_prev)
def print_errors(self): """If there are any UserErrorDescs in the students_with_errors list then print them out. Otherwise print nothing""" if self.students_with_errors: print "The following student accounts had problems:" for error in self.students_with_errors: print_error(str(error.student) + "\n\tERROR:" + error.error_desc)
def print_errors(self): """If there are any UserErrorDescs in the students_with_errors list then print them out. Otherwise print nothing""" if self.students_with_errors: print "The following student accounts had problems:" for error in self.students_with_errors: print_error( str(error.student) + "\n\tERROR:" + error.error_desc)
def connect_to_gitlab(env): """make an API request to create the glc.user object. This is mandatory if you use the username/password authentication.""" glc = gitlab.Gitlab(env[EnvOptions.SERVER], \ email=env[EnvOptions.USERNAME], \ password=env[EnvOptions.PASSWORD]) # print 'Connecting to GitLab server' try: glc.auth() except gitlab.GitlabError as exc: print_error('Unable to connect to the GitLab server', exc) exit() return glc
def connect_to_gitlab(env): """make an API request to create the glc.user object. This is mandatory if you use the username/password authentication.""" glc = gitlab.Gitlab(env[EnvOptions.SERVER], \ email=env[EnvOptions.USERNAME], \ password=env[EnvOptions.PASSWORD]) # print 'Connecting to GitLab server' try: glc.auth() except gitlab.GitlabError as exc: print_error( 'Unable to connect to the GitLab server', exc ) exit() return glc
def call_shell(cmd, exit_on_fail = True): """Invokes git in a command-line shell""" logger.info( 'About to do:' + cmd) sz_stdout, sz_stderr, ret = run_command_capture_output(cmd) # git sometimes uses '1' to indicate that something didn't have # a problem, but didn't do anything, either if ret != 0 and ret != 1: print_error("Problem executing '"+cmd+"'\n\tIn: " + os.getcwd() +\ "\n\tReturn code:"+str(ret)) if exit_on_fail: exit() else: logger.debug('\nGit Command:\n\t' + cmd + ' - SUCCEEDED\n')
def call_shell(cmd, exit_on_fail=True): """Invokes git in a command-line shell""" logger.info('About to do:' + cmd) sz_stdout, sz_stderr, ret = run_command_capture_output(cmd) # git sometimes uses '1' to indicate that something didn't have # a problem, but didn't do anything, either if ret != 0 and ret != 1: print_error("Problem executing '"+cmd+"'\n\tIn: " + os.getcwd() +\ "\n\tReturn code:"+str(ret)) if exit_on_fail: exit() else: logger.debug('\nGit Command:\n\t' + cmd + ' - SUCCEEDED\n')
def add_student_to_project(self, glc, project_id, student): """Tells the GitLab server to give the student 'reporter' level permissions on the project""" user_permission = {'project_id': project_id, \ 'access_level': 20, \ 'user_id': student.glid} # access_level: # 10 = guest, 20 = reporter, 30 = developer, 40 = master # Anything else causes an error (e.g., 31 != developer) try: #membership = glc.project_members.create(user_permission) glc.project_members.create(user_permission) except gitlab.exceptions.GitlabCreateError as exc: print_error("ERROR: unable to add " + student.first_name + " " \ + student.last_name + " to the project!") print_error(str(exc.response_code) + ": " + exc.error_message) return False return True
def grading_list_collector(): sha_tag, dt_tag = extract_commit_datetime(tag) if sha_tag is None: logger.debug( "This assignment hasn't been graded yet" ) self.ungraded.append(os.getcwd()) return True sha_head, dt_head = extract_commit_datetime("head") if sha_head == sha_tag: logger.debug( "SHA's for commits matched\n\tGRADED MOST RECENT SUBMISSION" ) self.graded.append(os.getcwd()) elif dt_tag < dt_head: logger.debug( "Instructor feedback was tagged then more work was submitted") self.new_student_work_since_grading.append(os.getcwd()) else: print_error("This directory has graded feedback, "\ "but the most recent commit is prior to the instructor's"\ " feedback commit & tag. This might indicate a problem"\ " with a timezone on the server\n\t"+\ os.getcwd()) return True
def commit_feedback(): """ Go through all the directories and if we find a file that matches the pattern try to commit it and tag it. """ # The expectation is that there's a single file that # matches and either it's already been committed & tagged, # or else that it's not yet in the repo (in which case, # commit and tag it) # # The full outline of what happens when is listed after # the code to determine if the tag exists and if any # matching files still need to be committed # git_tag_cmd = "git tag -a -m INSTRUCTOR_FEEDBACK "+ tag path_to_repo = os.getcwd() regex = re.compile(pattern, flags=re.IGNORECASE) # First figure out if the tag already exists: logger.debug("Looking for tag \"" + tag + "\" in " + os.getcwd() ) git_cmd = "git tag -l " + tag sz_stdout, sz_stderr, ret = run_command_capture_output(git_cmd, True) if sz_stdout == "": tagged = False else: tagged = True # Next, figure out if any matching files need to be committed: logger.debug("Looking for untracked and/or committed, modified files") git_cmd = "git status --porcelain" sz_stdout, sz_stderr, ret = run_command_capture_output(git_cmd, True) modified_staged = list() modified_not_staged = list() untracked = list() untracked_subdirs = list() for line in sz_stdout.splitlines(): # line format: file:///C:/Program%20Files/Git/mingw64/share/doc/git-doc/git-status.html#_short_format # [index][working tree]<space>filename # examples of lines: # M File.txt # present in repo, but not staged #M NewFile.txt # modified, added to index #A SubDir/FooFile.txt # added to index #?? ExtraFile.txt # untracked # # Note that git does NOT include the contents of untracked # subdirs in this output, so if a new file is put into a new # subdir (say, SubDir2\Grade.txt) git status will list #?? SubDir2 # note that Grade.txt is NOT listed # Thus, we actually do need to traverse the file system to # find new files # does this line's file match the pattern? both_codes = line[0:2] filename = line[3:] match = re.search(regex, filename) # If there's a new, untracked subdir # then we'll need to os.walk it to find # any matching files # (otherwise we can skip that) if both_codes == "??" and \ filename[len(filename)-1:] == '/': untracked_subdirs.append(os.path.join(path_to_repo, filename) ) if match: code_index = line[0] code_working = line[1] if both_codes == "??": untracked.append(filename) continue if both_codes == "!!": print_error(filename + " (in "+os.getcwd()+"):"\ "\n\tWARNIG: This matched the pattern but it"\ " also matches something in .gitignore\n"\ "(This will NOT be committed now)\n") continue codes_changed = "M ARC" if codes_changed.find(code_index) != -1: # changed in the index if code_working == " ": modified_staged.append(filename) # code_working & _index will never both be blank # (that would mean no changes) elif code_working == "M": modified_not_staged.append(filename) # find matching file(s) in untracked subdirs: # Skip this unless there's an untracked directory # (these can contain more stuff, and git doesn't scan through # the untracked dir) if untracked_subdirs: for subdir in untracked_subdirs: # walk through the subdir # (starting the walk here avoids any of the # files that git told us about) for root, dirs, files in os.walk(subdir): for name in files: match = re.search(regex, name) if match: path = os.path.join(root, name) local_dir = path.replace(path_to_repo, "") # remove the leading / if local_dir[0] == os.sep: local_dir = local_dir[1:] logger.debug( "found a match at " + local_dir ) untracked.append( local_dir ) #print_list(path_to_repo, modified_staged, Fore.CYAN, "modified, staged files:") #print_list(path_to_repo, modified_not_staged, Fore.YELLOW, "modified, unstaged files:") #print_list(path_to_repo, untracked, Fore.RED, "untracked files:") if modified_staged: need_commit = True else: need_commit = False files_to_add = modified_not_staged + untracked # The two 'expected' cases are listed at the top # Here's the full outline: # if not tagged: # if file absent: # note and skip # if file present but untracked: # add, commit, tag, done # if file committed and unchanged: # tag it? <ERROR> # # if tagged: # file should be in repo (else error) # if file not updated: # note and skip # if file has been updated: # update existing tag to have number after it # commit changes # tag the current commit with the desired tag if not tagged: # if file absent: if not need_commit and not files_to_add: # note and skip self.no_feedback_ever.append(os.getcwd() ) return # if file present but untracked: else: # add, commit, tag, done git_cmd = "git add " + " ".join(files_to_add) call_shell(git_cmd) call_shell("git commit -m Adding_Instructor_Feedback") sz_stdout, sz_stderr, ret = run_command_capture_output( git_tag_cmd ) self.new_feedback.append(os.getcwd()) return # if file committed and unchanged: # tag it? <ERROR> # we're not checking for previously committed files # so we don't handle this case # It *shouldn't* happen, anyways, so hopefully it won't # (It might happen if the teacher commits their # feedback manually) if tagged: # file should be in repo (else error) # if file not updated: if not need_commit and not files_to_add: # note and skip self.current_feedback_not_changed.append(os.getcwd() ) # if file has been updated: else: # update existing tag to have number after its renumber_current_tag(tag) git_cmd = "git add " + " ".join(files_to_add) call_shell(git_cmd) # commit changes call_shell("git commit -m Adding_Instructor_Feedback") # tag the current commit with the desired tag: sz_stdout, sz_stderr, ret = run_command_capture_output( git_tag_cmd ) self.current_feedback_updated.append(os.getcwd())
def download_homework(self, glc, env): if not self.assignments: print_error( self.section + " doesn't have any assignments "\ " to download") exit() new_student_projects = list() updated_student_projects = list() unchanged_student_projects = list() # After this, one of three things is true: # 1) hw_to_download is a list of all the homework project # 2) hw_to_download is a list of exactly one homework project # 3) EnvOptions.HOMEWORK_NAME didn't match anything and we exit # # 'homework project' is a copy of the HomeworkDesc named tuple (name,id) if env[EnvOptions.HOMEWORK_NAME].lower() == 'all': # we're going to make a new list with all the projects hw_name = 'all' hw_to_download = list(self.assignments) else: # we're going to make a new list that should match just one # homework assignment hw_name = self.homework_to_project_name( env[EnvOptions.HOMEWORK_NAME]) hw_to_download = [item for item in self.assignments \ if item.name == env[EnvOptions.HOMEWORK_NAME]] if not hw_to_download: # if list is empty print_error( env[EnvOptions.HOMEWORK_NAME] + " (internal name: " +\ hw_name + " ) doesn't match any of the have any assignments"\ " in section " + env[EnvOptions.SECTION]) exit() # First make sure that we can at least create the 'root' # directory for saving all the homework projects: dest_dir = env[EnvOptions.STUDENT_WORK_DIR] if not os.path.isdir(dest_dir): os.makedirs(dest_dir) # throws an exception on fail # Next, go through all the projects and see which # (if any) were forked from the target project foundAny = False whichPage = 1 cProject = 0 try: # If we need to clone (download) any projects we'll put them # in the temp_dir, then move the dir containing .git (and # all subdirs) to the place where we want them to end up temp_dir_root = tempfile.mkdtemp() # this is an absolute path projects = True while projects: logger.debug("About to retrieve page " + str(whichPage)) projects = glc.projects.all(page=whichPage, per_page=10) whichPage = whichPage + 1 if projects: foundAny = True if not foundAny and not projects: # if projects list is empty print "There are no projects present on the GitLab server" return logger.info("Found the following projects:\n") for project in projects: # For testing, to selectively download a single project: #if project.id != 61: # continue logger.info(project.name_with_namespace + " ID: " + str(project.id)) # See if the project matches any of the # 1/many projects that we're trying to download forked_project = None if hasattr(project, 'forked_from_project'): for hw in hw_to_download: if project.forked_from_project['id'] == hw.id: forked_project = hw break if forked_project is None: logger.debug( "\tNOT a forked project (or not forked from "\ + env[EnvOptions.HOMEWORK_NAME] + ")\n" ) continue logger.info( "\tProject was forked from " + forked_project.name \ + " (ID:"+str(forked_project.id)+")" ) owner_name = project.path_with_namespace.split('/')[0] student = Student(username=owner_name, id=project.owner.id) sys.stdout.write( '.') # This will slowly update the screen, so the # user doesn't just wait till we're all done :) # figure out the dir (path) for this project student_dest_dir = student.generate_hw_dir( env, \ forked_project.name) #cProject = cProject + 1 #if cProject == 4: # return # exit early for testing purposes if os.path.isdir(student_dest_dir): # if there's already a .git repo there then refresh (pull) it # instead of cloning it repo_exists = False for root, dirs, files in os.walk(student_dest_dir): for dir in dirs: if dir == '.git': git_dir = os.path.join(root, dir) logger.debug("Found an existing repo at " + git_dir) cwd_prev = os.getcwd() os.chdir(root) # in order to know if the pull actually # changes anything we'll need to compare # the SHA ID's of the HEAD commit before & after sz_std_out, sz_std_err, ret_code = run_command_capture_output("git show-ref --head --heads HEAD"\ , False) sha_pre_pull = sz_std_out.strip().split( )[0].strip() # Update the repo call_shell("git pull") sz_std_out, sz_std_err, ret_code = run_command_capture_output("git show-ref --head --heads HEAD"\ , False) sha_post_pull = sz_std_out.strip().split( )[0].strip() os.chdir(cwd_prev) hw_info = StudentHomeworkUpdateDesc(student, \ student_dest_dir, project, \ datetime.datetime.now()) if sha_pre_pull == sha_post_pull: unchanged_student_projects.append( hw_info) else: updated_student_projects.append( hw_info) # remember that we've updated it: repo_exists = True if repo_exists: break if repo_exists: break if repo_exists: continue # don't try to clone it again else: logger.debug( "local copy doesn't exist (yet), so clone it") # clone the repo into the project # The ssh connection string should look like: # git@ubuntu:root/bit142_assign_1.git temp_dir = os.path.join(temp_dir_root, "TEMP") os.makedirs(temp_dir) git_clone_repo(env[EnvOptions.SERVER_IP_ADDR], \ project, temp_dir) # next, go find the .git dir: found_git_dir = False for root, dirs, files in os.walk(temp_dir): for dir in dirs: if dir == '.git': new_git_dir = root logger.debug("Found the git dir inside " + new_git_dir) found_git_dir = True if found_git_dir: break if found_git_dir: break if not found_git_dir: raise Exception( "Despite cloning new repo, couldn't find git dir!" ) shutil.copytree(new_git_dir, student_dest_dir) shutil.rmtree(temp_dir, onerror=rmtree_remove_readonly_files) # add the repo into the list of updated projects new_student_projects.append( \ StudentHomeworkUpdateDesc(student, \ student_dest_dir, project, \ datetime.datetime.now()) ) # return the list of updated projects sys.stdout.flush() # make sure we actually get rid of our temp directory: finally: shutil.rmtree(temp_dir_root, onerror=rmtree_remove_readonly_files) return new_student_projects, updated_student_projects, unchanged_student_projects
def create_homework(self, glc, env): """ Attempt to create a homework assignment for a course by creating a project in the GitLab server, adding a record of the assignment to the course data file, and giving all current students access to the project""" proj_name = self.homework_to_project_name( \ env[EnvOptions.HOMEWORK_NAME]) project_data = {'name': proj_name, \ 'issues_enabled': False, \ 'wall_enabled': False, \ 'merge_requests_enabled': False, \ 'wiki_enabled': False, \ 'snippets_enabled': False, \ 'visibility_level': False, \ 'builds_enabled': False, \ 'public_builds': False, \ 'public': False, \ } # print(project_data) # If the user's account already we'll get an error here: try: project = glc.projects.create(project_data) print "Created : " + project.name_with_namespace # Remember that we created the assignment # This will be serialized to disk (in the section's data file) # at the end of this command self.assignments.append(HomeworkDesc(env[EnvOptions.HOMEWORK_NAME], \ proj_name, project.id)) self.assignments.sort() except gitlab.exceptions.GitlabCreateError as exc: # If the project already exists, look up it's info # and then proceed to add students to it if exc.error_message['name'][0].find("already been taken") >= 0: proj_path = env[EnvOptions.USERNAME] + "/" + proj_name project = glc.projects.get(proj_path) logger.debug("Found existing project " + project.name_with_namespace) else: # For anything else, just exit here print_error("Unable to create project " + proj_name) print_error( str(exc.response_code) + ": " + str(exc.error_message)) exit() #print project # add each student to the project as a reporter for student in self.roster.students_no_errors: if self.add_student_to_project(glc, project.id, student): print 'Added ' + student.first_name + " " + student.last_name else: print_error('ERROR: Unable to add ' + student.first_name + \ " " + student.last_name) cwd_prev = os.getcwd() dest_dir = os.path.join(env[EnvOptions.TEMP_DIR], project.name) try: # This is the part where we add the local, 'starter' repo to the # GitLab repo. # This should be idempotent: # If you already have a GitLab repo, and... # 1) If the local, starter repo and the GitLab repo are the # same then no changes will be made to the GitLab repo # 2) If the local, starter repo is different from the GitLab # repo then we'll update the existing GitLab repo git_clone_repo(env[EnvOptions.SERVER_IP_ADDR],\ project, env[EnvOptions.TEMP_DIR]) starter_project_name = "STARTER" # next, move into the directory # (so that subsequent commands affect the new repo) os.chdir(dest_dir) # TODO: Above line should be os.path.join # Next, add a 'remote' reference in the newly-cloned repo # to the starter project on our local machine: call_shell("git remote add "+starter_project_name+" "\ +env[EnvOptions.HOMEWORK_DIR]) # Get all the files from the starter project: call_shell("git fetch " + starter_project_name) # Merge the starter files into our new project: call_shell("git merge " + starter_project_name + "/master") # Clean up (remove) the remove reference # TODO: Do we actually need to do this? Refs don't get pushed, # and we delete the whole thing in the next step... call_shell("git remote remove " + starter_project_name) # Push the changes back up to GitLab call_shell("git push") finally: # Clear the temporary directory os.chdir(cwd_prev) shutil.rmtree(dest_dir, onerror=rmtree_remove_readonly_files)
def commit_feedback(): """ Go through all the directories and if we find a file that matches the pattern try to commit it and tag it. """ # The expectation is that there's a single file that # matches and either it's already been committed & tagged, # or else that it's not yet in the repo (in which case, # commit and tag it) # # The full outline of what happens when is listed after # the code to determine if the tag exists and if any # matching files still need to be committed # git_tag_cmd = "git tag -a -m INSTRUCTOR_FEEDBACK " + tag path_to_repo = os.getcwd() regex = re.compile(pattern, flags=re.IGNORECASE) # First figure out if the tag already exists: logger.debug("Looking for tag \"" + tag + "\" in " + os.getcwd()) git_cmd = "git tag -l " + tag sz_stdout, sz_stderr, ret = run_command_capture_output( git_cmd, True) if sz_stdout == "": tagged = False else: tagged = True # Next, figure out if any matching files need to be committed: logger.debug( "Looking for untracked and/or committed, modified files") git_cmd = "git status --porcelain" sz_stdout, sz_stderr, ret = run_command_capture_output( git_cmd, True) modified_staged = list() modified_not_staged = list() untracked = list() untracked_subdirs = list() for line in sz_stdout.splitlines(): # line format: file:///C:/Program%20Files/Git/mingw64/share/doc/git-doc/git-status.html#_short_format # [index][working tree]<space>filename # examples of lines: # M File.txt # present in repo, but not staged #M NewFile.txt # modified, added to index #A SubDir/FooFile.txt # added to index #?? ExtraFile.txt # untracked # # Note that git does NOT include the contents of untracked # subdirs in this output, so if a new file is put into a new # subdir (say, SubDir2\Grade.txt) git status will list #?? SubDir2 # note that Grade.txt is NOT listed # Thus, we actually do need to traverse the file system to # find new files # does this line's file match the pattern? both_codes = line[0:2] filename = line[3:] match = re.search(regex, filename) # If there's a new, untracked subdir # then we'll need to os.walk it to find # any matching files # (otherwise we can skip that) if both_codes == "??" and \ filename[len(filename)-1:] == '/': untracked_subdirs.append( os.path.join(path_to_repo, filename)) if match: code_index = line[0] code_working = line[1] if both_codes == "??": untracked.append(filename) continue if both_codes == "!!": print_error(filename + " (in "+os.getcwd()+"):"\ "\n\tWARNIG: This matched the pattern but it"\ " also matches something in .gitignore\n"\ "(This will NOT be committed now)\n") continue codes_changed = "M ARC" if codes_changed.find(code_index) != -1: # changed in the index if code_working == " ": modified_staged.append(filename) # code_working & _index will never both be blank # (that would mean no changes) elif code_working == "M": modified_not_staged.append(filename) # find matching file(s) in untracked subdirs: # Skip this unless there's an untracked directory # (these can contain more stuff, and git doesn't scan through # the untracked dir) if untracked_subdirs: for subdir in untracked_subdirs: # walk through the subdir # (starting the walk here avoids any of the # files that git told us about) for root, dirs, files in os.walk(subdir): for name in files: match = re.search(regex, name) if match: path = os.path.join(root, name) local_dir = path.replace(path_to_repo, "") # remove the leading / if local_dir[0] == os.sep: local_dir = local_dir[1:] logger.debug("found a match at " + local_dir) untracked.append(local_dir) #print_list(path_to_repo, modified_staged, Fore.CYAN, "modified, staged files:") #print_list(path_to_repo, modified_not_staged, Fore.YELLOW, "modified, unstaged files:") #print_list(path_to_repo, untracked, Fore.RED, "untracked files:") if modified_staged: need_commit = True else: need_commit = False files_to_add = modified_not_staged + untracked # The two 'expected' cases are listed at the top # Here's the full outline: # if not tagged: # if file absent: # note and skip # if file present but untracked: # add, commit, tag, done # if file committed and unchanged: # tag it? <ERROR> # # if tagged: # file should be in repo (else error) # if file not updated: # note and skip # if file has been updated: # update existing tag to have number after it # commit changes # tag the current commit with the desired tag if not tagged: # if file absent: if not need_commit and not files_to_add: # note and skip self.no_feedback_ever.append(os.getcwd()) return # if file present but untracked: else: # add, commit, tag, done git_cmd = "git add " + " ".join(files_to_add) call_shell(git_cmd) call_shell("git commit -m Adding_Instructor_Feedback") sz_stdout, sz_stderr, ret = run_command_capture_output( git_tag_cmd) self.new_feedback.append(os.getcwd()) return # if file committed and unchanged: # tag it? <ERROR> # we're not checking for previously committed files # so we don't handle this case # It *shouldn't* happen, anyways, so hopefully it won't # (It might happen if the teacher commits their # feedback manually) if tagged: # file should be in repo (else error) # if file not updated: if not need_commit and not files_to_add: # note and skip self.current_feedback_not_changed.append(os.getcwd()) # if file has been updated: else: # update existing tag to have number after its renumber_current_tag(tag) git_cmd = "git add " + " ".join(files_to_add) call_shell(git_cmd) # commit changes call_shell("git commit -m Adding_Instructor_Feedback") # tag the current commit with the desired tag: sz_stdout, sz_stderr, ret = run_command_capture_output( git_tag_cmd) self.current_feedback_updated.append(os.getcwd())