def creat_new_assignment(reference_assignment: canvasapi.assignment,
course: canvasapi.canvas.Course,
students_dict: typing.Dict[int, Student],
assignment_group_id):
"""
--creates a new assignment inside the course
:param reference_assignment: the new assignment inherits its name from the refrence assignment
:param course: the course in which the assignment is created
:param students_dict: student's dictionary is used to generate grades and upload the the new assignment
:param assignment_group_id
:side effects: the new assignment will not show up in the assignments view until the view is updated
"""
assignment = {}
name = ("Peer Review For " + reference_assignment.name)
number_of_times_name_used = 0
assignments = course.get_assignments()
for a in assignments:
if name in a.name:
number_of_times_name_used = number_of_times_name_used + 1
if number_of_times_name_used > 0:
name = ("Peer Review For " + reference_assignment.name
) + "(" + str(number_of_times_name_used) + ")"
assignment['name'] = name
assignment['published'] = True
assignment['points_possible'] = 100
assignment['grading_type'] = "percent"
assignment['assignment_group_id'] = assignment_group_id
new_assignment = course.create_assignment(assignment)
new_assignment.submissions_bulk_update(
grade_data=make_grade_dictionary(students_dict))
def extract_modules(c: canvasapi.canvas.Course) -> [canvasapi.canvas.File]:
# Some courses put files on the home page, but the Course.show_front_page endpoint seems to be broken
yield from file_extractor(c, extract_file_ids(getattr(c, 'syllabus_body', "")))
for mod in c.get_modules():
for mod_item in mod.get_module_items():
mod_type = getattr(mod_item, "type", None)
if mod_type == 'File':
yield c.get_file(mod_item.content_id)
elif mod_type == 'Page':
page_html = c.get_page(mod_item.page_url).body
yield from file_extractor(c, extract_file_ids(page_html))
elif mod_type in ['ExternalUrl', 'ExternalTool']:
# Not sure what to do with these. Ignoring for now
pass
elif mod_type in ['Assignment']:
# These can be picked up by the AssignmentBuilder, so ignoring here
pass
elif mod_type in ['SubHeader']:
# Useless
pass
else:
logger.warning(f"Unknown module type: {mod_type}")
logger.warning(mod_item)
def organize_by_file(
course: canvasapi.canvas.Course) -> (canvasapi.canvas.File, str):
folders = {folder.id: folder.full_name for folder in course.get_folders()}
for file in course.get_files():
folder = folders[file.folder_id] + "/"
folder = folder.lstrip("course files/")
yield (file, folder)
def organize_by_module(
course: canvasapi.canvas.Course) -> (canvasapi.canvas.File, str):
for m_idx, module in enumerate(course.get_modules()):
print(f" Module {Fore.CYAN}{module.name}{Style.RESET_ALL}")
for item in module.get_module_items():
if item.type == "File":
yield (course.get_file(item.content_id), '%d ' % m_idx +
re.sub(file_regex, "_",
module.name.replace("（", "(").replace("）", ")")))
def extract_assignments(c: canvasapi.canvas.Course):
for ass in c.get_assignments():
# `ass.description` might be outdated so we have to make a separate request
html_to_parse = c.get_assignment(ass.id).description
if html_to_parse is None:
continue
# Sometimes file links might be broken, sometimes the Canvas API just returns nonsense
# yield from map(c.get_file, extract_file_ids(html_to_parse))
yield from file_extractor(c, extract_file_ids(html_to_parse))
def organize_by_module(
course: canvasapi.canvas.Course) -> (canvasapi.canvas.File, str):
for module in course.get_modules():
print(module.name)
for item in module.get_module_items():
if item.type == "File":
yield (course.get_file(item.content_id),
re.sub(file_regex, "_",
module.name.replace("（", "(").replace("）", ")")))
def check_filelist_cache(course: canvasapi.canvas.Course):
if course.id not in course_files:
if 'files' in [tab.id for tab in course.get_tabs()]:
course_files[course.id] = {
file.id: file
for file in course.get_files()
}
else:
course_files[course.id] = None
return course_files[course.id] != None
def process_course(course: canvasapi.canvas.Course) -> [(str, str)]:
name = course.name.replace("（", "(").replace("）", ")")
print(f"{Fore.CYAN}Course {name} {course.course_code}{Style.RESET_ALL}")
folders = {folder.id: folder.full_name for folder in course.get_folders()}
for file in course.get_files():
folder = folders[file.folder_id] + "/"
if folder.startswith("course files/"):
folder = folder[len("course files/"):]
directory = f"{BASE_DIR}/{name}/{folder}"
path = f"{directory}{file.display_name}"
json_key = f"{name}/{folder}{file}"
d, reason = do_download(file)
if pathlib.Path(path).exists():
if json_key in checkpoint:
if checkpoint[json_key]["updated_at"] == file.updated_at:
d = False
reason = "already downloaded"
else:
if json_key in checkpoint:
del checkpoint[json_key]
do_checkpoint()
if file.url == "":
d = False
reason = "file not available"
if d:
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
try:
pathlib.Path(path).unlink()
except:
pass
print(
f" {Fore.GREEN}Download {file.display_name} ({file.size // 1024 / 1000}MB){Style.RESET_ALL}"
)
download_file(file.url, " Downloading", path)
checkpoint[json_key] = {"updated_at": file.updated_at}
new_files_list.append(path)
else:
print(
f" {Style.DIM}Ignore {file.display_name}: {reason}{Style.RESET_ALL}"
)
do_checkpoint()
def organize_by_module(
course: canvasapi.canvas.Course) -> (canvasapi.canvas.File, str):
for m_idx, module in enumerate(course.get_modules()):
print(f" Module {Fore.CYAN}{module.name}{Style.RESET_ALL}")
for item in module.get_module_items():
if item.type == "File":
module_name = MODULE_FOLDER_TEMPLATE
module_name = module_name.replace(
"{NAME}",
re.sub(file_regex, "_",
module.name.replace("（", "(").replace("）", ")")))
if CONSOLIDATE_MODULE_SPACE:
module_name = " ".join(module_name.split())
module_name = module_name.replace(
"{IDX}", str(m_idx + MODULE_FOLDER_IDX_BEGIN_WITH))
yield (course.get_file(item.content_id), module_name)
def organize_by_module(
course: canvasapi.canvas.Course) -> (canvasapi.canvas.File, str):
for module in course.get_modules():
module_item_count = module.items_count
module_item_position = module.position - 1 # it begins with 1
module_name = config.MODULE_FOLDER_TEMPLATE
module_name = module_name.replace(
"{NAME}",
re.sub(file_regex, "_",
module.name.replace("（", "(").replace("）", ")")))
if config.CONSOLIDATE_MODULE_SPACE:
module_name = " ".join(module_name.split())
module_name = module_name.replace(
"{IDX}",
str(module_item_position + config.MODULE_FOLDER_IDX_BEGIN_WITH))
print(
f" Module {Fore.CYAN}{module_name} ({module_item_count} items){Style.RESET_ALL}"
)
for item in module.get_module_items():
if item.type == "File":
yield get_file_in_course(course, item.content_id), module_name
elif item.type in ["Page", "Discussion", "Assignment"]:
page_url = item.html_url
elif item.type == "ExternalUrl":
page_url = item.external_url
elif item.type == "SubHeader":
pass
else:
if config.VERBOSE_MODE:
print(
f" {Fore.YELLOW}Unsupported item type: {item.type}{Style.RESET_ALL}"
)
def organize_by_file(
course: canvasapi.canvas.Course) -> (canvasapi.canvas.File, str):
folders = {folder.id: folder.full_name for folder in course.get_folders()}
for file in get_files_in_course(course):
folder = folders[file.folder_id] + "/"
if folder.startswith("course files/"):
folder = folder[len("course files/"):]
yield (file, folder)
def get_rubric(course: canvasapi.canvas.Course,
assignment_id: int) -> canvasapi.rubric:
"""
:param course: canvas course object
:return: the rubric associated with the assignment is returned
"""
# r = get_assignment(course,assignment_id).rubric
# rubric = course.get_rubrics(rubric_association_id=assignment_id, include=["peer_assessments"], style="full")[0]
# rubric_id = rubric.id
# rubric = course.get_rubric(rubric_id, include=["peer_assessments"], style="full")
# return rubric
assignment = get_assignment(course, assignment_id)
try:
rubric_id = assignment.rubric_settings['id']
rubric = course.get_rubric(rubric_id,
include=["peer_assessments"],
style="full")
return rubric
except AttributeError:
return None
def process_course(course: canvasapi.canvas.Course) -> [(str, str)]:
name = parse_course_folder_name(course)
print(f"{Fore.CYAN}Course {course.course_code}{Style.RESET_ALL}")
folders = {folder.id: folder.full_name for folder in course.get_folders()}
reasons_of_not_download = {}
for file in course.get_files():
folder = folders[file.folder_id] + "/"
folder = folder.lstrip("course files/")
directory = os.path.join(BASE_DIR, name, folder)
path = os.path.join(directory, file.display_name)
json_key = f"{name}/{folder}{file}"
d, reason = do_download(file)
update_flag = False
if pathlib.Path(path).exists():
if json_key in checkpoint:
if checkpoint[json_key]["updated_at"] == file.updated_at:
d = False
reason = "already downloaded"
else:
update_flag = True
else:
if json_key in checkpoint:
del checkpoint[json_key]
do_checkpoint()
if file.url == "":
d = False
reason = "file not available"
if d:
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
try:
pathlib.Path(path).unlink()
except:
pass
print(
f" {Fore.GREEN}{'Update' if update_flag else 'New'}: {file.display_name} ({file.size // 1024 / 1000}MB){Style.RESET_ALL}")
download_file(file.url, " Downloading", path)
if OVERRIDE_FILE_TIME:
c_time = datetime.strptime(
file.created_at, '%Y-%m-%dT%H:%M:%S%z').timestamp()
m_time = datetime.strptime(
file.updated_at, '%Y-%m-%dT%H:%M:%S%z').timestamp()
a_time = time.time()
if is_windows():
setctime(path, c_time)
os.utime(path, (a_time, m_time))
checkpoint[json_key] = {"updated_at": file.updated_at}
new_files_list.append(path)
else:
if VERBOSE_MODE:
print(
f" {Style.DIM}Ignore {file.display_name}: {reason}{Style.RESET_ALL}")
else:
prev_cnt = reasons_of_not_download.get(reason, 0)
reasons_of_not_download[reason] = prev_cnt + 1
do_checkpoint()
for (reason, cnt) in reasons_of_not_download.items():
print(f" {Style.DIM}{cnt} files ignored: {reason}{Style.RESET_ALL}")
def process_course(course: canvasapi.canvas.Course):
global checkpoint
name = parse_course_folder_name(course)
print(
f"Course {Fore.CYAN}{course.course_code} (ID: {course.id}){Style.RESET_ALL}"
)
reasons_of_not_download = {}
organize_mode = config.ORGANIZE_BY
if course.id in config.CUSTOM_ORGANIZE:
organize_mode = config.CUSTOM_ORGANIZE[course.id]
for (file, folder) in get_file_list(course, organize_mode):
directory = os.path.join(config.BASE_DIR, name, folder).rstrip()
filename = replaceIllegalChar(file.display_name, file_regex)
path = os.path.join(directory, filename)
json_key = f"{name}/{folder}{file}"
if type(file) == Link:
if config.ENABLE_LINK:
Path(directory).mkdir(parents=True, exist_ok=True)
path += '.url' if is_windows() else '.html'
download_link(file.url, path)
current_link_list.append(path)
if config.OVERRIDE_FILE_TIME:
# cannot be implemented
# apply_datetime_attr(path, file.created_at, file.updated_at)
pass
elif config.VERBOSE_MODE:
print(
f" {Style.DIM}Ignore {file.display_name}: {'ENABLE_LINK disabled'}{Style.RESET_ALL}"
)
continue
can_download, reason, update_flag = check_download_rule(
file, path, json_key)
if can_download:
Path(directory).mkdir(parents=True, exist_ok=True)
try:
Path(path).unlink()
except:
pass
print(
f" {Fore.GREEN}{'Update' if update_flag else 'New'}: {file.display_name} ({file.size // 1024 / 1000}MB){Style.RESET_ALL}"
)
try:
download_file(file.url,
" Downloading",
path,
file.size,
verbose=config.VERBOSE_MODE)
if config.OVERRIDE_FILE_TIME:
apply_datetime_attr(path, file.created_at, file.updated_at)
checkpoint[json_key] = CheckpointItem(file.updated_at_date,
file.id, config.SESSION)
new_files_list.append(path)
except Exception as e:
print(
f" {Fore.YELLOW}Failed to download: {e}{Style.RESET_ALL}"
)
failure_file_list.append(path)
else:
if config.VERBOSE_MODE:
print(
f" {Style.DIM}Ignore {file.display_name}: {reason}{Style.RESET_ALL}"
)
else:
prev_cnt = reasons_of_not_download.get(reason, 0)
reasons_of_not_download[reason] = prev_cnt + 1
if update_flag:
updated_files_list.append(path)
current_file_list.append(path)
checkpoint.dump()
if config.ENABLE_VIDEO:
for page in course.get_pages():
for (result, msg) in resolve_video(page.show_latest_revision()):
if result:
filename = msg.split("/")[-2]
json_key = f"{name}/{page.title}-{filename}"
path = os.path.join(config.BASE_DIR, name,
f"{page.title}-{filename}")
path = replaceIllegalChar(path)
if not Path(path).exists():
quoted_path = shlex.quote(path)
ffmpeg_commands.append(
f"{config.FFMPEG_PATH} -i '{msg}' -c copy -bsf:a aac_adtstoasc {quoted_path}"
)
else:
prev_cnt = reasons_of_not_download.get(msg, 0)
reasons_of_not_download[msg] = prev_cnt + 1
for (reason, cnt) in reasons_of_not_download.items():
print(f" {Style.DIM}{cnt} files ignored: {reason}{Style.RESET_ALL}")
def get_assignment(course: canvasapi.canvas.Course, assignment_id: int):
"""
:param course: canvas course object
:return: a canvas assignment is returned given its id
"""
return course.get_assignment(assignment_id)
def get_assignments(course: canvasapi.canvas.Course) -> [canvasapi.assignment]:
"""
:param course: canvas course object
:return: a list of canvas assignment objects
"""
return course.get_assignments()
def get_assignment_groups(course: canvasapi.canvas.Course):
"""
returns assignment groups for a given course
"""
return course.get_assignment_groups()
def get_file_in_course(course: canvasapi.canvas.Course, file_id: str):
if check_filelist_cache(course):
return course_files[course.id][file_id]
else:
return course.get_file(file_id)
def get_students(course: canvasapi.canvas.Course) -> [canvasapi.canvas.User]:
"""
:param course: canvas course object
:return: returns a list of students in a canvas course
"""
return course.get_users(enrollment_type='student')
def process_course(course: canvasapi.canvas.Course):
name = parse_course_folder_name(course)
print(
f"Course {Fore.CYAN}{course.course_code} (ID: {course.id}){Style.RESET_ALL}"
)
reasons_of_not_download = {}
organize_mode = config.ORGANIZE_BY
if course.id in config.CUSTOM_ORGANIZE:
organize_mode = config.CUSTOM_ORGANIZE[course.id]
for (file, folder) in get_file_list(course, organize_mode):
directory = os.path.join(config.BASE_DIR, name, folder)
filename = replaceIlligalChar(file.display_name, file_regex)
path = os.path.join(directory, filename)
json_key = f"{name}/{folder}{file}"
can_download, reason, update_flag = check_download_rule(
file, path, json_key)
if can_download:
Path(directory).mkdir(parents=True, exist_ok=True)
try:
Path(path).unlink()
except:
pass
print(
f" {Fore.GREEN}{'Update' if update_flag else 'New'}: {file.display_name} ({file.size // 1024 / 1000}MB){Style.RESET_ALL}"
)
try:
download_file(file.url,
" Downloading",
path,
file.size,
verbose=config.VERBOSE_MODE)
if config.OVERRIDE_FILE_TIME:
c_time = datetime.strptime(
file.created_at, r'%Y-%m-%dT%H:%M:%S%z').timestamp()
m_time = datetime.strptime(
file.updated_at, r'%Y-%m-%dT%H:%M:%S%z').timestamp()
a_time = time.time()
if is_windows():
setctime(path, c_time)
os.utime(path, (a_time, m_time))
checkpoint[json_key] = {
"updated_at": file.updated_at,
"id": file.id,
"session": config.SESSION
}
new_files_list.append(path)
except Exception as e:
print(
f" {Fore.YELLOW}Failed to download: {e}{Style.RESET_ALL}"
)
failure_file_list.append(path)
else:
if config.VERBOSE_MODE:
print(
f" {Style.DIM}Ignore {file.display_name}: {reason}{Style.RESET_ALL}"
)
else:
prev_cnt = reasons_of_not_download.get(reason, 0)
reasons_of_not_download[reason] = prev_cnt + 1
if update_flag:
updated_files_list.append(path)
current_file_list.append(path)
do_checkpoint()
if config.ENABLE_VIDEO:
for page in course.get_pages():
for (result, msg) in resolve_video(page.show_latest_revision()):
if result == True:
filename = msg.split("/")[-2]
json_key = f"{name}/{page.title}-{filename}"
path = os.path.join(config.BASE_DIR, name,
f"{page.title}-{filename}")
path = replaceIlligalChar(path)
if not Path(path).exists():
quoted_path = shlex.quote(path)
ffmpeg_commands.append(
f"{config.FFMPEG_PATH} -i '{msg}' -c copy -bsf:a aac_adtstoasc {quoted_path}"
)
else:
prev_cnt = reasons_of_not_download.get(msg, 0)
reasons_of_not_download[msg] = prev_cnt + 1
for (reason, cnt) in reasons_of_not_download.items():
print(f" {Style.DIM}{cnt} files ignored: {reason}{Style.RESET_ALL}")
def process_course(course: canvasapi.canvas.Course):
name = parse_course_folder_name(course)
print(
f"{Fore.CYAN}Course {course.course_code} (ID: {course.id}){Style.RESET_ALL}"
)
reasons_of_not_download = {}
for (file, folder) in get_file_list(course, ORGANIZE_BY):
directory = os.path.join(BASE_DIR, name, folder)
path = os.path.join(directory, file.display_name)
json_key = f"{name}/{folder}{file}"
d, reason = do_download(file)
update_flag = False
if pathlib.Path(path).exists():
if json_key in checkpoint:
if checkpoint[json_key]["updated_at"] == file.updated_at:
d = False
reason = "already downloaded"
else:
update_flag = True
else:
if json_key in checkpoint:
del checkpoint[json_key]
do_checkpoint()
if file.url == "":
d = False
reason = "file not available"
if d:
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
try:
pathlib.Path(path).unlink()
except:
pass
print(
f" {Fore.GREEN}{'Update' if update_flag else 'New'}: {file.display_name} ({file.size // 1024 / 1000}MB){Style.RESET_ALL}"
)
download_file(file.url, " Downloading", path)
if OVERRIDE_FILE_TIME:
c_time = datetime.strptime(file.created_at,
'%Y-%m-%dT%H:%M:%S%z').timestamp()
m_time = datetime.strptime(file.updated_at,
'%Y-%m-%dT%H:%M:%S%z').timestamp()
a_time = time.time()
if is_windows():
setctime(path, c_time)
os.utime(path, (a_time, m_time))
checkpoint[json_key] = {"updated_at": file.updated_at}
new_files_list.append(path)
else:
if VERBOSE_MODE:
print(
f" {Style.DIM}Ignore {file.display_name}: {reason}{Style.RESET_ALL}"
)
else:
prev_cnt = reasons_of_not_download.get(reason, 0)
reasons_of_not_download[reason] = prev_cnt + 1
do_checkpoint()
if ENABLE_VIDEO:
for page in course.get_pages():
for (result, msg) in resolve_video(page.show_latest_revision()):
if result == True:
filename = msg.split("/")[-2]
json_key = f"{name}/{page.title}-{filename}"
path = os.path.join(BASE_DIR, name,
f"{page.title}-{filename}")
if not Path(path).exists():
quoted_path = shlex.quote(path)
ffmpeg_commands.append(
f"{FFMPEG_PATH} -i '{msg}' -c copy -bsf:a aac_adtstoasc {quoted_path}"
)
else:
prev_cnt = reasons_of_not_download.get(msg, 0)
reasons_of_not_download[msg] = prev_cnt + 1
for (reason, cnt) in reasons_of_not_download.items():
print(f" {Style.DIM}{cnt} files ignored: {reason}{Style.RESET_ALL}")