def open_assignment(conf, args): """Adds each student in the roster to their respective homework repositories as Developers so they can pull/commit/push their work. """ hw_name = args.name host = conf.gitlab_host namespace = conf.namespace token = conf.gitlab_token semester = conf.semester roster = get_filtered_roster(conf.roster, args.section, args.student) count = 0 for student in roster: username = student["username"] student_section = student["section"] full_name = StudentRepo.name(semester, student_section, hw_name, username) try: repo = StudentRepo(host, namespace, full_name, token) if "id" not in student: student["id"] = Repo.get_user_id(username, host, token) repo.add_member(student["id"], Access.developer) count += 1 except RepoError: logging.warn("Could not add {} to {}.".format(username, full_name)) except HTTPError: raise print("Granted access to {} repositories.".format(count))
def manage_repos(conf, args, action): """Performs an action (lambda) on all student repos """ hw_name = args.name dry_run = args.dry_run host = conf.gitlab_host namespace = conf.namespace token = conf.gitlab_token semester = conf.semester roster = get_filtered_roster(conf.roster, args.section, args.student) count = 0 for student in roster: username = student["username"] student_section = student["section"] if "id" not in student: logging.warning( "Student {} does not have a gitlab account.".format(username)) continue full_name = StudentRepo.name(semester, student_section, hw_name, username) try: repo = StudentRepo(host, namespace, full_name, token) if not dry_run: action(repo) count += 1 except HTTPError: raise print("Changed {} repositories.".format(count))
def manage_repos(conf, backend, args, action): """Performs an action (lambda) on all student repos """ hw_name = args.name dry_run = args.dry_run namespace = conf.namespace semester = conf.semester backend_conf = conf.backend roster = get_filtered_roster(conf.roster, args.section, args.student) count = 0 for student in progress.iterate(roster): username = student["username"] student_section = student["section"] if "id" not in student: logging.warning("Student %s does not have a gitlab account.", username) continue full_name = backend.student_repo.build_name(semester, student_section, hw_name, username) repo = backend.student_repo(backend_conf, namespace, full_name) if not dry_run: if action(repo, student): count += 1 else: count += 1 print("Changed {} repositories.".format(count))
def open_all_assignments(conf, backend, args): """Adds each student in the roster to their respective homework repositories as Developers so they can pull/commit/push their work. """ hw_name = args.name namespace = conf.namespace semester = conf.semester backend_conf = conf.backend roster = get_filtered_roster(conf.roster, args.section, args.student) count = 0 for student in progress.iterate(roster): username = student["username"] student_section = student["section"] full_name = backend.student_repo.build_name(semester, student_section, hw_name, username) try: repo = backend.student_repo(backend_conf, namespace, full_name) if "id" not in student: student["id"] = backend.repo.get_user_id( username, backend_conf) open_assignment(repo, student, backend.access.developer) count += 1 except UserInAssignerGroup: logging.info( "%s already has access via group membership, skipping...", username) except RepoError: logging.warning("Could not add %s to %s.", username, full_name) print("Granted access to {} repositories.".format(count))
def list_students(conf, args): """List students in the roster """ output = PrettyTable(["#", "Name", "Username", "Section"]) for idx, student in enumerate( get_filtered_roster(conf.roster, args.section, None)): output.add_row((idx + 1, student["name"], student["username"], student["section"])) print(output)
def checkout_students(conf: Config, backend: BackendBase, args: argparse.Namespace) -> None: """Interactively prompts for student info and grabs the most recent CI artifact, which contains their autograded score """ roster = get_filtered_roster(conf.roster, args.section, None) while True: query = input("Enter student ID or name, or 'q' to quit: ") if "quit".startswith(query): break student = student_search(roster, query) if not student: continue score = handle_scoring(conf, backend, args, student) logger.info("Uploaded score of %d", (score))
def score_assignments(conf: Config, backend: BackendBase, args: argparse.Namespace) -> None: """Goes through each student repository and grabs the most recent CI artifact, which contains their autograded score """ student = args.student roster = get_filtered_roster(conf.roster, args.section, student) scores = [] for student in progress.iterate(roster): score = handle_scoring(conf, backend, args, student) if score is not None: scores.append(score) print("Scored {} repositories.".format(len(scores))) print_statistics(scores)
def _push(conf, backend, args): hw_name = args.name hw_path = args.path namespace = conf.namespace semester = conf.semester backend_conf = conf.backend branch = args.branch force = args.force push_unlocked = args.push_unlocked path = os.path.join(hw_path, hw_name) roster = get_filtered_roster(conf.roster, args.section, args.student) for student in progress.iterate(roster): username = student["username"] student_section = student["section"] full_name = backend.student_repo.build_name(semester, student_section, hw_name, username) try: repo = backend.student_repo(backend_conf, namespace, full_name) repo_dir = os.path.join(path, username) repo.add_local_copy(repo_dir) if repo.is_locked() or push_unlocked: info = repo.repo.remote().push(branch, force=force, set_upstream=True) for line in info: logging.debug("%s: flags: %s, branch: %s, summary: %s", full_name, line.flags, line.local_ref, line.summary) if line.flags & line.ERROR: logging.warning("%s: push to %s failed: %s", full_name, line.local_ref, line.summary) else: logging.warning( "%s: repo is not locked (run 'assigner lock %s' first)", full_name, hw_name) except NoSuchPathError: logging.warning("Local repo for %s does not exist; skipping...", username) except RepoError as e: logging.warning(e)
def manage_users(conf, args, level): """Creates a folder for the assignment in the CWD (or <path>, if specified) and clones each students' repository into subfolders. """ hw_name = args.name dry_run = args.dry_run if dry_run: raise NotImplementedError("'--dry-run' is not implemented") host = conf.gitlab_host namespace = conf.namespace token = conf.gitlab_token semester = conf.semester roster = get_filtered_roster(conf.roster, args.section, args.student) count = 0 for student in roster: username = student["username"] student_section = student["section"] if "id" not in student: logging.warning( "Student {} does not have a gitlab account.".format(username)) continue full_name = StudentRepo.name(semester, student_section, hw_name, username) try: repo = StudentRepo(host, namespace, full_name, token) repo.edit_member(student["id"], level) count += 1 except HTTPError: raise print("Changed {} repositories.".format(count))
def integrity_check(conf: Config, backend: BackendBase, args: argparse.Namespace) -> None: """Checks that none of the grading files were modified in the timeframe during which students could push to their repository """ student = args.student files_to_check = set(args.files) roster = get_filtered_roster(conf.roster, args.section, None) for student in progress.iterate(roster): username = student["username"] student_section = student["section"] full_name = backend.student_repo.build_name(conf.semester, student_section, args.name, username) try: repo = backend.student_repo(conf.backend, conf.namespace, full_name) check_repo_integrity(repo, files_to_check) except RepoError as e: logger.debug(e) logger.warning("Unable to find repo for %s with URL %s", username, full_name)
def get(conf, args): """Creates a folder for the assignment in the CWD (or <path>, if specified) and clones each students' repository into subfolders. """ hw_name = args.name hw_path = args.path host = conf.gitlab_host namespace = conf.namespace token = conf.gitlab_token semester = conf.semester roster = get_filtered_roster(conf.roster, args.section, args.student) path = os.path.join(hw_path, hw_name) os.makedirs(path, mode=0o700, exist_ok=True) count = 0 for student in roster: username = student["username"] student_section = student["section"] full_name = StudentRepo.name(semester, student_section, hw_name, username) try: repo = StudentRepo(host, namespace, full_name, token) repo.clone_to(os.path.join(path, username)) count += 1 except RepoError as e: logging.warn(str(e)) except HTTPError as e: if e.response.status_code == 404: logging.warn("Repository {} does not exist.".format(full_name)) else: raise print("Cloned {} repositories.".format(count))
def assign(conf, args): """Creates homework repositories for an assignment for each student in the roster. """ hw_name = args.name if args.branch: branch = args.branch else: branch = ["master"] dry_run = args.dry_run force = args.force host = conf.gitlab_host namespace = conf.namespace token = conf.gitlab_token semester = conf.semester roster = get_filtered_roster(conf.roster, args.section, args.student) actual_count = 0 # Represents the number of repos actually pushed to student_count = len(roster) with tempfile.TemporaryDirectory() as tmpdirname: print("Assigning '{}' to {} student{} in {}.".format( hw_name, student_count, "s" if student_count != 1 else "", "section " + args.section if args.section else "all sections")) base = BaseRepo(host, namespace, hw_name, token) if not dry_run: try: base.clone_to(tmpdirname, branch) except RepoError as e: logging.error( "Could not clone base repo (have you pushed at least one commit to it?)" ) logging.debug(e) return if force: logging.warning("Repos will be overwritten.") for i, student in enumerate(roster): username = student["username"] student_section = student["section"] full_name = StudentRepo.name(semester, student_section, hw_name, username) repo = StudentRepo(host, namespace, full_name, token) print("{}/{} - {}".format(i + 1, student_count, full_name)) if not repo.already_exists(): if not dry_run: repo = StudentRepo.new(base, semester, student_section, username, token) repo.push(base, branch) for b in branch: repo.protect(b) actual_count += 1 logging.debug("Assigned.") elif force: logging.info("{}: Already exists.".format(full_name)) logging.info("{}: Deleting...".format(full_name)) if not dry_run: repo.delete() # Gitlab will throw a 400 if you delete and immediately recreate a repo. # We retry w/ exponential backoff up to 5 times wait = 0.1 retries = 0 while True: try: repo = StudentRepo.new(base, semester, student_section, username, token) logger.debug("Success!") break except HTTPError as e: if retries >= 5 or e.response.status_code != 400: logger.debug("Critical Failure!") raise logger.debug("Failed, retrying...") # Delay and try again time.sleep(wait * 2**retries) retries += 1 repo.push(base, branch) for b in branch: repo.protect(b) actual_count += 1 logging.debug("Assigned.") elif args.branch: logging.info("{}: Already exists.".format(full_name)) # If we have an explicit branch, push anyways if not dry_run: repo.push(base, branch) for b in branch: repo.protect(b) actual_count += 1 logging.debug("Assigned.") else: logging.warning("Skipping...") i += 1 print("Assigned '{}' to {} student{}.".format( hw_name, actual_count, "s" if actual_count != 1 else "")) if actual_count == 0: logging.warning( "Consider using --force if you want to override existing repos.")
def _push(conf, backend, args): backend_conf = conf.backend namespace = conf.namespace semester = conf.semester hw_name = args.name hw_path = args.path message = args.message branch = args.branch add = args.add remove = args.remove update = args.update allow_empty = args.allow_empty # Default behavior: commit changes to all tracked files if (add == []) and (remove == []): logging.debug("Nothing explicitly added or removed; defaulting to git add --update") update = True path = os.path.join(hw_path, hw_name) roster = get_filtered_roster(conf.roster, args.section, args.student) for student in progress.iterate(roster): username = student["username"] student_section = student["section"] full_name = backend.student_repo.build_name(semester, student_section, hw_name, username) has_changes = False try: repo = backend.student_repo(backend_conf, namespace, full_name) repo_dir = os.path.join(path, username) repo.add_local_copy(repo_dir) logging.debug("%s: checking out branch %s", full_name, branch) repo.get_head(branch).checkout() index = repo.get_index() if update: # Stage modified and deleted files for commit # This exactly mimics the behavior of git add --update # (or the effect of doing git commit -a) for change in index.diff(None): has_changes = True if change.deleted_file: logging.debug("%s: git rm %s", full_name, change.b_path) index.remove([change.b_path]) else: logging.debug("%s: git add %s", full_name, change.b_path) index.add([change.b_path]) if add: has_changes = True logging.debug("%s: adding %s", full_name, add) index.add(add) if remove: has_changes = True logging.debug("%s: removing %s", full_name, remove) index.remove(remove) if has_changes or allow_empty: logging.debug("%s: committing changes with message %s", full_name, message) index.commit(message) else: logging.warning("%s: No changes in repo; skipping commit.", full_name) except NoSuchPathError: logging.warning("Local repo for %s does not exist; skipping...", username) except RepoError as e: logging.warning(e) except HTTPError as e: if e.response.status_code == 404: logging.warning("Repository %s does not exist.", full_name) else: raise
def status(conf, args): """Retrieves and prints the status of repos""" hw_name = args.name if not hw_name: raise ValueError("Missing assignment name.") host = conf.gitlab_host namespace = conf.namespace token = conf.gitlab_token semester = conf.semester roster = get_filtered_roster(conf.roster, args.section, args.student) sort_key = args.sort if sort_key: roster.sort(key=lambda s: s[sort_key]) output = PrettyTable([ "#", "Sec", "SID", "Name", "Status", "Branches", "HEAD", "Last Commit Author", "Last Push Time" ]) output.align["Name"] = "l" output.align["Last Commit Author"] = "l" progress = ProgressBar(max_value=len(roster)) for i, student in enumerate(roster): progress.update(i) name = student["name"] username = student["username"] student_section = student["section"] full_name = StudentRepo.name(semester, student_section, hw_name, username) row = [i + 1, student_section, username, name, "", "", "", "", ""] try: repo = StudentRepo(host, namespace, full_name, token) if not repo.already_exists(): row[4] = "Not Assigned" output.add_row(row) continue if "id" not in student: try: student["id"] = Repo.get_user_id(username, host, token) except RepoError: row[4] = "No Gitlab user" output.add_row(row) continue members = repo.list_members() if student["id"] not in [s["id"] for s in members]: row[4] = "Not Opened" output.add_row(row) continue if repo.info["archived"]: row[4] = 'Archived' else: level = Access([ s["access_level"] for s in members if s["id"] == student["id"] ][0]) row[4] = "Open" if level is Access.developer else "Locked" branches = repo.list_branches() if branches: row[5] = ", ".join([b["name"] for b in branches]) head = repo.get_last_HEAD_commit() if head: row[6] = head["short_id"] row[7] = head["author_name"] created_at = head["created_at"] # Fix UTC offset format in GitLab's datetime created_at = created_at[:-6] + created_at[-6:].replace(':', '') row[8] = datetime.strptime( created_at, "%Y-%m-%dT%H:%M:%S.%f%z").astimezone().strftime("%c") output.add_row(row) except HTTPError: raise progress.finish() print(output)
def status(conf, backend, args): """Retrieves and prints the status of repos""" hw_name = args.name if not hw_name: raise ValueError("Missing assignment name.") namespace = conf.namespace semester = conf.semester backend_conf = conf.backend roster = get_filtered_roster(conf.roster, args.section, args.student) sort_key = args.sort if sort_key: roster.sort(key=lambda s: s[sort_key]) output = PrettyTable([ "#", "Sec", "SID", "Name", "Status", "Branches", "HEAD", "Last Commit Author", "Last Push Time", ]) output.align["Name"] = "l" output.align["Last Commit Author"] = "l" for i, student in progress.enumerate(roster): name = student["name"] username = student["username"] student_section = student["section"] full_name = backend.student_repo.build_name(semester, student_section, hw_name, username) row = [i + 1, student_section, username, name, "", "", "", "", ""] repo = backend.student_repo(backend_conf, namespace, full_name) if not repo.already_exists(): row[4] = "Not Assigned" output.add_row(row) continue if "id" not in student: try: student["id"] = backend.repo.get_user_id( username, backend_conf) except RepoError: row[4] = "No Gitlab user" output.add_row(row) continue members = repo.list_members() if student["id"] not in [s["id"] for s in members]: row[4] = "Not Opened" output.add_row(row) continue if repo.info["archived"]: row[4] = "Archived" else: level = backend.access([ s["access_level"] for s in members if s["id"] == student["id"] ][0]) row[4] = "Open" if level is backend.access.developer else "Locked" branches = repo.list_branches() if branches: row[5] = "\n".join([b["name"] for b in branches]) head = repo.get_last_HEAD_commit() if head: row[6] = head["short_id"] row[7] = head["author_name"] created_at = head["created_at"] row[8] = datetime.strptime( created_at, "%Y-%m-%dT%H:%M:%S.%f%z").astimezone().strftime("%c") output.add_row(row) print(output)
def _get(conf, backend, args): """ Creates a folder for the assignment in the CWD (or <path>, if specified) and clones each students' repository into subfolders. """ hw_name = args.name hw_path = args.path namespace = conf.namespace semester = conf.semester backend_conf = conf.backend branch = args.branch force = args.force attempts = args.attempts roster = get_filtered_roster(conf.roster, args.section, args.student) path = os.path.join(hw_path, hw_name) os.makedirs(path, mode=0o700, exist_ok=True) output = PrettyTable(["#", "Sec", "SID", "Name", "Change"], print_empty=False) output.align["Name"] = "l" output.align["Change"] = "l" for i, student in progress.enumerate(roster): username = student["username"] student_section = student["section"] full_name = backend.student_repo.build_name(semester, student_section, hw_name, username) try: repo = backend.student_repo(backend_conf, namespace, full_name) repo_dir = os.path.join(path, username) row = str(i + 1) sec = student["section"] sid = student["username"] name = student["name"] try: logging.debug("Attempting to use local repo %s...", repo_dir) repo.add_local_copy(repo_dir) logging.debug("Local repo exists, fetching...") results = repo.repo.remote().fetch() for result in results: logging.debug("fetch result: name: %s flags: %s note: %s", result.ref.name, result.flags, result.note) # see: # http://gitpython.readthedocs.io/en/stable/reference.html#git.remote.FetchInfo if result.flags & result.NEW_HEAD: output.add_row([ row, sec, sid, name, "{}: new branch at {}".format( result.ref.name, str(result.ref.commit)[:8]) ]) row = sec = sid = name = "" # don't print user info more than once elif result.old_commit is not None: output.add_row([ row, sec, sid, name, "{}: {} -> {}".format(result.ref.name, str(result.old_commit)[:8], str(result.ref.commit)[:8]) ]) row = sec = sid = name = "" logging.debug("Pulling specified branches...") for b in branch: try: repo.get_head(b).checkout(force=force) repo.pull(b) except GitCommandError as e: logging.debug(e) logging.warning( "Local changes to %s/%s would be overwritten by pull", username, b) logging.warning(" (use --force to overwrite)") except (NoSuchPathError, InvalidGitRepositoryError): logging.debug("Local repo does not exist; cloning...") repo.clone_to(repo_dir, branch, attempts) output.add_row([row, sec, sid, name, "Cloned a new copy"]) # Check out first branch specified; this is probably what people expect # If there's just one branch, it's already checked out by the loop above if len(branch) > 1: repo.get_head(branch[0]).checkout() except RetryableGitError as e: logging.warning(e) except RepoError as e: logging.warning(e) out_str = output.get_string() if out_str != "": print(out_str) else: print("No changes since last call to get")
def assign(conf, backend, args): """Creates homework repositories for an assignment for each student in the roster. """ hw_name = args.name if args.branch: branch = args.branch else: branch = ["master"] dry_run = args.dry_run force = args.force namespace = conf.namespace semester = conf.semester backend_conf = conf.backend roster = get_filtered_roster(conf.roster, args.section, args.student) actual_count = 0 # Represents the number of repos actually pushed to student_count = len(roster) with tempfile.TemporaryDirectory() as tmpdirname: print("Assigning '{}' to {} student{} in {}.".format( hw_name, student_count, "s" if student_count != 1 else "", "section " + args.section if args.section else "all sections")) template = backend.template_repo(backend_conf, namespace, hw_name) if not dry_run: try: template.clone_to(tmpdirname, branch) except BranchNotFound as e: logging.error("Cound not find branch %s in base repository", e.args[0]) logging.debug(e) return except RepoError as e: logging.error( "Could not clone template repo (have you pushed at least one commit to it?)" ) logging.debug(e) return if force: logging.warning("Repos will be overwritten.") for i, student in progress.enumerate(roster): username = student["username"] student_section = student["section"] full_name = backend.student_repo.build_name( semester, student_section, hw_name, username) repo = backend.student_repo(backend_conf, namespace, full_name) if not repo.already_exists(): if not dry_run: repo = backend.student_repo.new(template, semester, student_section, username) repo.push(template, branch) for b in branch: repo.protect(b) actual_count += 1 logging.debug("Assigned.") elif force: logging.info("%s: Already exists, deleting...", full_name) if not dry_run: repo.delete() # Gitlab will throw a 400 if you delete and immediately # recreate a repo. We retry w/ exponential backoff up # to 5 times wait = 0.1 retries = 0 while True: try: repo = backend.student_repo.new( template, semester, student_section, username) logger.debug("Success!") break except RepositoryAlreadyExists as e: if retries >= 5: logger.debug("Critical Failure!") raise logger.debug("Failed, retrying...") logger.debug(e) # Delay and try again time.sleep(wait * 2**retries) retries += 1 repo.push(template, branch) for b in branch: repo.protect(b) actual_count += 1 logging.debug("Assigned.") elif args.branch: logging.info("%s: Already exists.", full_name) # If we have an explicit branch, push anyways if not dry_run: repo.push(template, branch) for b in branch: repo.protect(b) actual_count += 1 logging.debug("Assigned.") else: logging.info("%s: Already exists, skipping...", full_name) i += 1 if args.open: open_assignment(repo, student, backend.access.developer) print("Assigned '{}' to {} student{}.".format( hw_name, actual_count, "s" if actual_count != 1 else "")) if actual_count == 0: logging.warning( "Consider using --force if you want to override existing repos.")
def _push(conf, backend, args): backend_conf = conf.backend namespace = conf.namespace semester = conf.semester hw_name = args.name hw_path = args.path message = args.message branch = args.branch add = args.add remove = args.remove update = args.update allow_empty = args.allow_empty gpg_sign = args.gpg_sign # Default behavior: commit changes to all tracked files if (add == []) and (remove == []): logging.debug( "Nothing explicitly added or removed; defaulting to git add --update" ) update = True path = os.path.join(hw_path, hw_name) roster = get_filtered_roster(conf.roster, args.section, args.student) for student in progress.iterate(roster): username = student["username"] student_section = student["section"] full_name = backend.student_repo.build_name(semester, student_section, hw_name, username) has_changes = False try: repo = backend.student_repo(backend_conf, namespace, full_name) repo_dir = os.path.join(path, username) repo.add_local_copy(repo_dir) logging.debug("%s: checking out branch %s", full_name, branch) repo.get_head(branch).checkout() index = repo.get_index() if update: # Stage modified and deleted files for commit # This exactly mimics the behavior of git add --update # (or the effect of doing git commit -a) for change in index.diff(None): has_changes = True if change.deleted_file: logging.debug("%s: git rm %s", full_name, change.b_path) index.remove([change.b_path]) else: logging.debug("%s: git add %s", full_name, change.b_path) index.add([change.b_path]) if add: has_changes = True logging.debug("%s: adding %s", full_name, add) index.add(add) if remove: has_changes = True logging.debug("%s: removing %s", full_name, remove) index.remove(remove) if has_changes or allow_empty: logging.debug("%s: committing changes with message %s", full_name, message) if gpg_sign: # The GitPython interface does not support signed commits, and # launching via repo.git.commit will launch an inaccessible # interactive prompt in the background index.write(ignore_extension_data=True) subprocess.check_call( ["git", "commit", "-S", "-m", '"{}"'.format(message)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=repo_dir) else: index.commit(message) else: logging.warning("%s: No changes in repo; skipping commit.", full_name) except NoSuchPathError: logging.warning("Local repo for %s does not exist; skipping...", username) except RepoError as e: logging.warning(e)