def _select_sections_to_download(self, sections: [{}],
excluded: [int]) -> [int]:
"""
Asks the user for the sections that should be downloaded.
@param sections: All available sections
@param excluded sections currently excluded
"""
choices = []
defaults = []
for i, section in enumerate(sections):
section_id = section.get("id")
choices.append(('%5i\t%s' % (section_id, section.get("name"))))
if ResultsHandler.should_download_section(section_id, excluded):
defaults.append(i)
Log.special('Which of the sections should be downloaded?')
Log.info(
'[You can select with the space bar and confirm your selection with the enter key]'
)
print('')
selected_sections = cutie.select_multiple(options=choices,
ticked_indices=defaults)
dont_download_section_ids = []
for i, section in enumerate(sections):
if i not in selected_sections:
dont_download_section_ids.append(section.get("id"))
return dont_download_section_ids
def _select_courses_to_download(self, courses: [Course]):
"""
Asks the user for the courses that should be downloaded.
@param courses: All available courses
"""
download_course_ids = self.config_helper.get_download_course_ids()
dont_download_course_ids = self.config_helper.get_dont_download_course_ids(
)
print('')
Log.info(
'To avoid downloading all the Moodle courses you are enrolled in, you can select which ones you want'
+ ' to download here. ')
print('')
choices = []
defaults = []
for i, course in enumerate(courses):
choices.append(('%5i\t%s' % (course.id, course.fullname)))
if ResultsHandler.should_download_course(course.id,
download_course_ids,
dont_download_course_ids):
defaults.append(i)
Log.special('Which of the courses should be downloaded?')
Log.info(
'[You can select with the space bar and confirm your selection with the enter key]'
)
print('')
selected_courses = cutie.select_multiple(options=choices,
ticked_indices=defaults)
download_course_ids = []
for i, course in enumerate(courses):
if i in selected_courses:
download_course_ids.append(course.id)
self.config_helper.set_property('download_course_ids',
download_course_ids)
self.config_helper.remove_property('dont_download_course_ids')
return download_course_ids
def filter_courses(
changes: [Course],
config_helper: ConfigHelper,
cookie_handler: CookieHandler = None,
courses_list: [Course] = None,
) -> [Course]:
"""
Filters the changes course list from courses that
should not get downloaded
@param config_helper: ConfigHelper to obtain all the diffrent filter configs
@param cookie_handler: CookieHandler to check if the cookie is valid
@param courses_list: A list of all courses that are available online
@return: filtered changes course list
"""
download_course_ids = config_helper.get_download_course_ids()
download_public_course_ids = config_helper.get_download_public_course_ids()
dont_download_course_ids = config_helper.get_dont_download_course_ids()
download_submissions = config_helper.get_download_submissions()
download_descriptions = config_helper.get_download_descriptions()
download_links_in_descriptions = config_helper.get_download_links_in_descriptions()
download_databases = config_helper.get_download_databases()
download_quizzes = config_helper.get_download_quizzes()
download_lessons = config_helper.get_download_lessons()
download_workshops = config_helper.get_download_workshops()
exclude_file_extensions = config_helper.get_exclude_file_extensions()
download_also_with_cookie = config_helper.get_download_also_with_cookie()
if cookie_handler is not None:
download_also_with_cookie = cookie_handler.test_cookies()
filtered_changes = []
for course in changes:
if not ResultsHandler.should_download_course(
course.id, download_course_ids + download_public_course_ids, dont_download_course_ids
):
# Filter courses that should not be downloaded
continue
if courses_list is not None:
not_online = True
# Filter courses that are not available online
for online_course in courses_list:
if online_course.id == course.id:
not_online = False
break
if not_online:
Log.warning(f'The Moodle course with id {course.id} is no longer available online.')
logging.warning('The Moodle course with id %d is no longer available online.', course.id)
continue
course_files = []
for file in course.files:
# Filter Files based on options
if (
# Filter Assignment Submission Files
(download_submissions or (not (file.module_modname.endswith('assign') and file.deleted)))
# Filter Description Files (except the forum posts)
and (
download_descriptions
or file.content_type != 'description'
or (file.module_modname == 'forum' and file.content_type == 'description' and file.content_filename != 'Forum intro')
)
# Filter Database Files
and (download_databases or file.content_type != 'database_file')
# Filter Quiz Files
and (download_quizzes or (not (file.module_modname.endswith('quiz') and file.deleted)))
# Filter Lesson Files
and (download_lessons or (not (file.module_modname.endswith('lesson') and file.deleted)))
# Filter Workshops Files
and (download_workshops or (not (file.module_modname.endswith('workshop') and file.deleted)))
# Filter Files that requiere a Cookie
and (download_also_with_cookie or (not file.module_modname.startswith('cookie_mod-')))
# Exclude files whose file extension is blacklisted
and (not (determine_ext(file.content_filename) in exclude_file_extensions))
# Exclude files that are in excluded sections
and (ResultsHandler.should_download_section(file.section_id, course.excluded_sections))
):
course_files.append(file)
course.files = course_files
# Filter Description URLs
course_files = []
for file in course.files:
if not file.content_type == 'description-url':
course_files.append(file)
elif download_links_in_descriptions:
add_description_url = True
for test_file in course.files:
if file.content_fileurl == test_file.content_fileurl:
if test_file.content_type != 'description-url':
# If a URL in a description also exists as a real link in the course,
# then ignore this URL
add_description_url = False
break
elif file.module_id > test_file.module_id:
# Always use the link from the older description.
add_description_url = False
break
if add_description_url:
course_files.append(file)
course.files = course_files
if len(course.files) > 0:
filtered_changes.append(course)
return filtered_changes
def fetch_state(self) -> [Course]:
"""
Gets the current status of the configured Moodle account and compares
it with the last known status for changes. It does not change the
known state, nor does it download the files.
@return: List with detected changes
"""
logging.debug('Fetching current Moodle State...')
token = self.config_helper.get_token()
privatetoken = self.config_helper.get_privatetoken()
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
use_http = self.config_helper.get_use_http()
request_helper = RequestHelper(
moodle_domain,
moodle_path,
token,
self.skip_cert_verify,
self.log_responses_to,
use_http,
)
first_contact_handler = FirstContactHandler(request_helper)
results_handler = ResultsHandler(request_helper, moodle_domain, moodle_path)
download_course_ids = self.config_helper.get_download_course_ids()
download_public_course_ids = self.config_helper.get_download_public_course_ids()
dont_download_course_ids = self.config_helper.get_dont_download_course_ids()
download_submissions = self.config_helper.get_download_submissions()
download_databases = self.config_helper.get_download_databases()
download_forums = self.config_helper.get_download_forums()
download_quizzes = self.config_helper.get_download_quizzes()
download_lessons = self.config_helper.get_download_lessons()
download_workshops = self.config_helper.get_download_workshops()
download_also_with_cookie = self.config_helper.get_download_also_with_cookie()
courses = []
filtered_courses = []
cookie_handler = None
print('\rDownloading account information\033[K', end='')
userid, version = self.config_helper.get_userid_and_version()
if userid is None or version is None:
userid, version = first_contact_handler.fetch_userid_and_version()
else:
first_contact_handler.version = version
assignments_handler = AssignmentsHandler(request_helper, version)
databases_handler = DatabasesHandler(request_helper, version)
forums_handler = ForumsHandler(request_helper, version)
quizzes_handler = QuizzesHandler(request_helper, version)
lessons_handler = LessonsHandler(request_helper, version)
workshops_handler = WorkshopsHandler(request_helper, version)
pages_handler = PagesHandler(request_helper, version)
folders_handler = FoldersHandler(request_helper, version)
results_handler.setVersion(version)
if download_also_with_cookie:
# generate a new cookie if necessary
cookie_handler = CookieHandler(request_helper, version, self.storage_path)
cookie_handler.check_and_fetch_cookies(privatetoken, userid)
courses_list = first_contact_handler.fetch_courses(userid)
courses = []
# Filter unselected courses
for course in courses_list:
if ResultsHandler.should_download_course(course.id, download_course_ids, dont_download_course_ids):
courses.append(course)
public_courses_list = first_contact_handler.fetch_courses_info(download_public_course_ids)
for course in public_courses_list:
courses.append(course)
assignments = assignments_handler.fetch_assignments(courses)
if download_submissions:
assignments = assignments_handler.fetch_submissions(userid, assignments)
databases = databases_handler.fetch_databases(courses)
if download_databases:
databases = databases_handler.fetch_database_files(databases)
forums = forums_handler.fetch_forums(courses)
if download_forums:
last_timestamps_per_forum = self.recorder.get_last_timestamps_per_forum()
forums = forums_handler.fetch_forums_posts(forums, last_timestamps_per_forum)
quizzes = quizzes_handler.fetch_quizzes(courses)
if download_quizzes:
quizzes = quizzes_handler.fetch_quizzes_files(userid, quizzes)
lessons = lessons_handler.fetch_lessons(courses)
if download_lessons:
lessons = lessons_handler.fetch_lessons_files(userid, lessons)
workshops = workshops_handler.fetch_workshops(courses)
if download_workshops:
workshops = workshops_handler.fetch_workshops_files(userid, workshops)
pages = pages_handler.fetch_pages(courses)
folders = folders_handler.fetch_folders(courses)
courses = self.add_options_to_courses(courses)
index = 0
for course in courses:
index += 1
# to limit the output to one line
limits = shutil.get_terminal_size()
shorted_course_name = course.fullname
if len(course.fullname) > 17:
shorted_course_name = course.fullname[:15] + '..'
into = '\rDownloading course information'
status_message = into + ' %3d/%3d [%-17s|%6s]' % (index, len(courses), shorted_course_name, course.id)
if len(status_message) > limits.columns:
status_message = status_message[0 : limits.columns]
print(status_message + '\033[K', end='')
course_fetch_addons = {
'assign': assignments.get(course.id, {}),
'data': databases.get(course.id, {}),
'forum': forums.get(course.id, {}),
'quiz': quizzes.get(course.id, {}),
'lesson': lessons.get(course.id, {}),
'workshop': workshops.get(course.id, {}),
'page': pages.get(course.id, {}),
'folder': folders.get(course.id, {}),
}
results_handler.set_fetch_addons(course_fetch_addons)
course.files = results_handler.fetch_files(course)
filtered_courses.append(course)
print('')
logging.debug('Checking for changes...')
changes = self.recorder.changes_of_new_version(filtered_courses)
# Filter changes
changes = self.add_options_to_courses(changes)
changes = self.filter_courses(changes, self.config_helper, cookie_handler, courses_list + public_courses_list)
return changes
def filter_courses(changes: [Course],
config_helper: ConfigHelper,
cookie_handler: CookieHandler = None) -> [Course]:
"""
Filters the changes course list from courses that
should not get downloaded
@param config_helper: ConfigHelper to obtain all the diffrent filter configs
@param cookie_handler: CookieHandler to check if the cookie is valid
@return: filtered changes course list
"""
download_course_ids = config_helper.get_download_course_ids()
download_public_course_ids = config_helper.get_download_public_course_ids(
)
dont_download_course_ids = config_helper.get_dont_download_course_ids()
download_submissions = config_helper.get_download_submissions()
download_descriptions = config_helper.get_download_descriptions()
download_links_in_descriptions = config_helper.get_download_links_in_descriptions(
)
download_databases = config_helper.get_download_databases()
download_also_with_cookie = config_helper.get_download_also_with_cookie(
)
if cookie_handler is not None:
download_also_with_cookie = cookie_handler.test_cookies()
filtered_changes = []
for course in changes:
if not download_submissions:
course_files = []
for file in course.files:
if not (file.module_modname.endswith('assign')
and file.deleted):
course_files.append(file)
course.files = course_files
if not download_descriptions:
course_files = []
for file in course.files:
if file.content_type != 'description':
course_files.append(file)
course.files = course_files
course_files = []
for file in course.files:
if not file.content_type == 'description-url':
course_files.append(file)
elif download_links_in_descriptions:
add_description_url = True
for test_file in course.files:
if file.content_fileurl == test_file.content_fileurl:
if test_file.content_type != 'description-url':
# If a URL in a description also exists as a real link in the course,
# then ignore this URL
add_description_url = False
break
elif file.module_id > test_file.module_id:
# Always use the link from the older description.
add_description_url = False
break
if add_description_url:
course_files.append(file)
course.files = course_files
if not download_databases:
course_files = []
for file in course.files:
if file.content_type != 'database_file':
course_files.append(file)
course.files = course_files
if not download_also_with_cookie:
course_files = []
for file in course.files:
if not file.module_modname.startswith('cookie_mod-'):
course_files.append(file)
course.files = course_files
if (ResultsHandler.should_download_course(
course.id, download_course_ids +
download_public_course_ids, dont_download_course_ids)
and len(course.files) > 0):
filtered_changes.append(course)
return filtered_changes
def fetch_state(self) -> [Course]:
"""
Gets the current status of the configured Moodle account and compares
it with the last known status for changes. It does not change the
known state, nor does it download the files.
@return: List with detected changes
"""
logging.debug('Fetching current Moodle State...')
token = self.config_helper.get_token()
privatetoken = self.config_helper.get_privatetoken()
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
request_helper = RequestHelper(moodle_domain, moodle_path, token,
self.skip_cert_verify,
self.log_responses_to)
first_contact_handler = FirstContactHandler(request_helper)
results_handler = ResultsHandler(request_helper, moodle_domain,
moodle_path)
download_course_ids = self.config_helper.get_download_course_ids()
dont_download_course_ids = self.config_helper.get_dont_download_course_ids(
)
download_submissions = self.config_helper.get_download_submissions()
download_databases = self.config_helper.get_download_databases()
download_forums = self.config_helper.get_download_forums()
download_also_with_cookie = self.config_helper.get_download_also_with_cookie(
)
courses = []
filtered_courses = []
cookie_handler = None
try:
print('\rDownloading account information\033[K', end='')
userid, version = first_contact_handler.fetch_userid_and_version()
assignments_handler = AssignmentsHandler(request_helper, version)
databases_handler = DatabasesHandler(request_helper, version)
forums_handler = ForumsHandler(request_helper, version)
results_handler.setVersion(version)
if download_also_with_cookie:
# generate a new cookie if necessary
cookie_handler = CookieHandler(request_helper, version,
self.storage_path)
cookie_handler.check_and_fetch_cookies(privatetoken, userid)
courses_list = first_contact_handler.fetch_courses(userid)
courses = []
# Filter unselected courses
for course in courses_list:
if ResultsHandler.should_download_course(
course.id, download_course_ids,
dont_download_course_ids):
courses.append(course)
assignments = assignments_handler.fetch_assignments(courses)
if download_submissions:
assignments = assignments_handler.fetch_submissions(
userid, assignments)
databases = databases_handler.fetch_databases(courses)
if download_databases:
databases = databases_handler.fetch_database_files(databases)
forums = forums_handler.fetch_forums(courses)
if download_forums:
last_timestamps_per_forum = self.recorder.get_last_timestamps_per_forum(
)
forums = forums_handler.fetch_forums_posts(
forums, last_timestamps_per_forum)
index = 0
for course in courses:
index += 1
# to limit the output to one line
limits = shutil.get_terminal_size()
shorted_course_name = course.fullname
if len(course.fullname) > 17:
shorted_course_name = course.fullname[:15] + '..'
into = '\rDownloading course information'
status_message = into + ' %3d/%3d [%-17s|%6s]' % (
index, len(courses), shorted_course_name, course.id)
if len(status_message) > limits.columns:
status_message = status_message[0:limits.columns]
print(status_message + '\033[K', end='')
course_assignments = assignments.get(course.id, {})
course_databases = databases.get(course.id, {})
course_forums = forums.get(course.id, {})
results_handler.set_fetch_addons(course_assignments,
course_databases,
course_forums)
course.files = results_handler.fetch_files(course.id)
filtered_courses.append(course)
print('')
except (RequestRejectedError, ValueError, RuntimeError) as error:
raise RuntimeError(
'Error while communicating with the Moodle System! (%s)' %
(error))
logging.debug('Checking for changes...')
changes = self.recorder.changes_of_new_version(filtered_courses)
# Filter changes
changes = self.filter_courses(changes, self.config_helper,
cookie_handler)
changes = self.add_options_to_courses(changes)
return changes
def _set_options_of_courses(self, courses: [Course]):
"""
Let the user set special options for every single course
"""
download_course_ids = self.config_helper.get_download_course_ids()
dont_download_course_ids = self.config_helper.get_dont_download_course_ids(
)
self.section_seperator()
Log.info(
'You can set special settings for every single course.\n' +
'You can set these options:\n' +
' - A different name for the course\n' +
' - If a directory structure should be created for the course' +
' [create_directory_structure (cfs)].')
print('')
while True:
choices = []
choices_courses = []
options_of_courses = self.config_helper.get_options_of_courses()
choices.append('None')
for course in courses:
if ResultsHandler.should_download_course(
course.id, download_course_ids,
dont_download_course_ids):
current_course_settings = options_of_courses.get(
str(course.id), None)
# create default settings
if current_course_settings is None:
current_course_settings = {
'original_name': course.fullname,
'overwrite_name_with': None,
'create_directory_structure': True,
}
# create list of options
overwrite_name_with = current_course_settings.get(
'overwrite_name_with', None)
create_directory_structure = current_course_settings.get(
'create_directory_structure', True)
if overwrite_name_with is not None and overwrite_name_with != course.fullname:
choices.append(
('%5i\t%s (%s) cfs=%s' %
(course.id, overwrite_name_with, course.fullname,
create_directory_structure)))
else:
choices.append(
('%5i\t%s cfs=%s' % (course.id, course.fullname,
create_directory_structure)))
choices_courses.append(course)
print('')
Log.special(
'For which of the following course do you want to change the settings?'
)
print('[Confirm your selection with the Enter key]')
print('')
selected_course = cutie.select(options=choices)
if selected_course == 0:
break
else:
self._change_settings_of(choices_courses[selected_course - 1],
options_of_courses)
def filter_courses(
changes: [Course],
download_course_ids: [int],
dont_download_course_ids: [int],
download_submissions: bool,
download_descriptions: bool,
download_links_in_descriptions: bool,
download_databases: bool,
) -> [Course]:
"""
Filters the changes course list from courses that
should not get downloaded
@param download_course_ids: list of course ids
that should be downloaded
@param dont_download_course_ids: list of course ids
that should not be downloaded
@param download_submissions: boolean if submissions
should be downloaded
@param download_descriptions: boolean if descriptions
should be downloaded
@param download_links_in_descriptions: boolean if links in descriptions should be downloaded
@param download_databases: boolean if databases should be downloaded
@return: filtered changes course list
"""
filtered_changes = []
for course in changes:
if not download_submissions:
course_files = []
for file in course.files:
if file.content_type != 'submission_file':
course_files.append(file)
course.files = course_files
if not download_descriptions:
course_files = []
for file in course.files:
if file.content_type != 'description':
course_files.append(file)
course.files = course_files
if not download_links_in_descriptions:
course_files = []
for file in course.files:
if not file.module_modname.endswith('-description'):
course_files.append(file)
course.files = course_files
if not download_databases:
course_files = []
for file in course.files:
if file.content_type != 'database_file':
course_files.append(file)
course.files = course_files
if (ResultsHandler.should_download_course(
course.id, download_course_ids, dont_download_course_ids)
and len(course.files) > 0):
filtered_changes.append(course)
return filtered_changes