def process_course(course: canvasapi.canvas.Course):
name = parse_course_folder_name(course)
print(
f"Course {Fore.CYAN}{course.course_code} (ID: {course.id}){Style.RESET_ALL}"
)
reasons_of_not_download = {}
organize_mode = config.ORGANIZE_BY
if course.id in config.CUSTOM_ORGANIZE:
organize_mode = config.CUSTOM_ORGANIZE[course.id]
for (file, folder) in get_file_list(course, organize_mode):
directory = os.path.join(config.BASE_DIR, name, folder)
filename = replaceIlligalChar(file.display_name, file_regex)
path = os.path.join(directory, filename)
json_key = f"{name}/{folder}{file}"
can_download, reason, update_flag = check_download_rule(
file, path, json_key)
if can_download:
Path(directory).mkdir(parents=True, exist_ok=True)
try:
Path(path).unlink()
except:
pass
print(
f" {Fore.GREEN}{'Update' if update_flag else 'New'}: {file.display_name} ({file.size // 1024 / 1000}MB){Style.RESET_ALL}"
)
try:
download_file(file.url,
" Downloading",
path,
file.size,
verbose=config.VERBOSE_MODE)
if config.OVERRIDE_FILE_TIME:
c_time = datetime.strptime(
file.created_at, r'%Y-%m-%dT%H:%M:%S%z').timestamp()
m_time = datetime.strptime(
file.updated_at, r'%Y-%m-%dT%H:%M:%S%z').timestamp()
a_time = time.time()
if is_windows():
setctime(path, c_time)
os.utime(path, (a_time, m_time))
checkpoint[json_key] = {
"updated_at": file.updated_at,
"id": file.id,
"session": config.SESSION
}
new_files_list.append(path)
except Exception as e:
print(
f" {Fore.YELLOW}Failed to download: {e}{Style.RESET_ALL}"
)
failure_file_list.append(path)
else:
if config.VERBOSE_MODE:
print(
f" {Style.DIM}Ignore {file.display_name}: {reason}{Style.RESET_ALL}"
)
else:
prev_cnt = reasons_of_not_download.get(reason, 0)
reasons_of_not_download[reason] = prev_cnt + 1
if update_flag:
updated_files_list.append(path)
current_file_list.append(path)
do_checkpoint()
if config.ENABLE_VIDEO:
for page in course.get_pages():
for (result, msg) in resolve_video(page.show_latest_revision()):
if result == True:
filename = msg.split("/")[-2]
json_key = f"{name}/{page.title}-{filename}"
path = os.path.join(config.BASE_DIR, name,
f"{page.title}-{filename}")
path = replaceIlligalChar(path)
if not Path(path).exists():
quoted_path = shlex.quote(path)
ffmpeg_commands.append(
f"{config.FFMPEG_PATH} -i '{msg}' -c copy -bsf:a aac_adtstoasc {quoted_path}"
)
else:
prev_cnt = reasons_of_not_download.get(msg, 0)
reasons_of_not_download[msg] = prev_cnt + 1
for (reason, cnt) in reasons_of_not_download.items():
print(f" {Style.DIM}{cnt} files ignored: {reason}{Style.RESET_ALL}")
def process_course(course: canvasapi.canvas.Course):
global checkpoint
name = parse_course_folder_name(course)
print(
f"Course {Fore.CYAN}{course.course_code} (ID: {course.id}){Style.RESET_ALL}"
)
reasons_of_not_download = {}
organize_mode = config.ORGANIZE_BY
if course.id in config.CUSTOM_ORGANIZE:
organize_mode = config.CUSTOM_ORGANIZE[course.id]
for (file, folder) in get_file_list(course, organize_mode):
directory = os.path.join(config.BASE_DIR, name, folder).rstrip()
filename = replaceIllegalChar(file.display_name, file_regex)
path = os.path.join(directory, filename)
json_key = f"{name}/{folder}{file}"
if type(file) == Link:
if config.ENABLE_LINK:
Path(directory).mkdir(parents=True, exist_ok=True)
path += '.url' if is_windows() else '.html'
download_link(file.url, path)
current_link_list.append(path)
if config.OVERRIDE_FILE_TIME:
# cannot be implemented
# apply_datetime_attr(path, file.created_at, file.updated_at)
pass
elif config.VERBOSE_MODE:
print(
f" {Style.DIM}Ignore {file.display_name}: {'ENABLE_LINK disabled'}{Style.RESET_ALL}"
)
continue
can_download, reason, update_flag = check_download_rule(
file, path, json_key)
if can_download:
Path(directory).mkdir(parents=True, exist_ok=True)
try:
Path(path).unlink()
except:
pass
print(
f" {Fore.GREEN}{'Update' if update_flag else 'New'}: {file.display_name} ({file.size // 1024 / 1000}MB){Style.RESET_ALL}"
)
try:
download_file(file.url,
" Downloading",
path,
file.size,
verbose=config.VERBOSE_MODE)
if config.OVERRIDE_FILE_TIME:
apply_datetime_attr(path, file.created_at, file.updated_at)
checkpoint[json_key] = CheckpointItem(file.updated_at_date,
file.id, config.SESSION)
new_files_list.append(path)
except Exception as e:
print(
f" {Fore.YELLOW}Failed to download: {e}{Style.RESET_ALL}"
)
failure_file_list.append(path)
else:
if config.VERBOSE_MODE:
print(
f" {Style.DIM}Ignore {file.display_name}: {reason}{Style.RESET_ALL}"
)
else:
prev_cnt = reasons_of_not_download.get(reason, 0)
reasons_of_not_download[reason] = prev_cnt + 1
if update_flag:
updated_files_list.append(path)
current_file_list.append(path)
checkpoint.dump()
if config.ENABLE_VIDEO:
for page in course.get_pages():
for (result, msg) in resolve_video(page.show_latest_revision()):
if result:
filename = msg.split("/")[-2]
json_key = f"{name}/{page.title}-{filename}"
path = os.path.join(config.BASE_DIR, name,
f"{page.title}-{filename}")
path = replaceIllegalChar(path)
if not Path(path).exists():
quoted_path = shlex.quote(path)
ffmpeg_commands.append(
f"{config.FFMPEG_PATH} -i '{msg}' -c copy -bsf:a aac_adtstoasc {quoted_path}"
)
else:
prev_cnt = reasons_of_not_download.get(msg, 0)
reasons_of_not_download[msg] = prev_cnt + 1
for (reason, cnt) in reasons_of_not_download.items():
print(f" {Style.DIM}{cnt} files ignored: {reason}{Style.RESET_ALL}")
def process_course(course: canvasapi.canvas.Course):
name = parse_course_folder_name(course)
print(
f"{Fore.CYAN}Course {course.course_code} (ID: {course.id}){Style.RESET_ALL}"
)
reasons_of_not_download = {}
for (file, folder) in get_file_list(course, ORGANIZE_BY):
directory = os.path.join(BASE_DIR, name, folder)
path = os.path.join(directory, file.display_name)
json_key = f"{name}/{folder}{file}"
d, reason = do_download(file)
update_flag = False
if pathlib.Path(path).exists():
if json_key in checkpoint:
if checkpoint[json_key]["updated_at"] == file.updated_at:
d = False
reason = "already downloaded"
else:
update_flag = True
else:
if json_key in checkpoint:
del checkpoint[json_key]
do_checkpoint()
if file.url == "":
d = False
reason = "file not available"
if d:
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
try:
pathlib.Path(path).unlink()
except:
pass
print(
f" {Fore.GREEN}{'Update' if update_flag else 'New'}: {file.display_name} ({file.size // 1024 / 1000}MB){Style.RESET_ALL}"
)
download_file(file.url, " Downloading", path)
if OVERRIDE_FILE_TIME:
c_time = datetime.strptime(file.created_at,
'%Y-%m-%dT%H:%M:%S%z').timestamp()
m_time = datetime.strptime(file.updated_at,
'%Y-%m-%dT%H:%M:%S%z').timestamp()
a_time = time.time()
if is_windows():
setctime(path, c_time)
os.utime(path, (a_time, m_time))
checkpoint[json_key] = {"updated_at": file.updated_at}
new_files_list.append(path)
else:
if VERBOSE_MODE:
print(
f" {Style.DIM}Ignore {file.display_name}: {reason}{Style.RESET_ALL}"
)
else:
prev_cnt = reasons_of_not_download.get(reason, 0)
reasons_of_not_download[reason] = prev_cnt + 1
do_checkpoint()
if ENABLE_VIDEO:
for page in course.get_pages():
for (result, msg) in resolve_video(page.show_latest_revision()):
if result == True:
filename = msg.split("/")[-2]
json_key = f"{name}/{page.title}-{filename}"
path = os.path.join(BASE_DIR, name,
f"{page.title}-{filename}")
if not Path(path).exists():
quoted_path = shlex.quote(path)
ffmpeg_commands.append(
f"{FFMPEG_PATH} -i '{msg}' -c copy -bsf:a aac_adtstoasc {quoted_path}"
)
else:
prev_cnt = reasons_of_not_download.get(msg, 0)
reasons_of_not_download[msg] = prev_cnt + 1
for (reason, cnt) in reasons_of_not_download.items():
print(f" {Style.DIM}{cnt} files ignored: {reason}{Style.RESET_ALL}")