def __init__(
self,
config_helper: ConfigHelper,
storage_path: str,
skip_cert_verify: bool = False,
log_responses: bool = False,
):
self.config_helper = config_helper
self.storage_path = storage_path
self.recorder = StateRecorder(Path(storage_path) / 'moodle_state.db')
self.skip_cert_verify = skip_cert_verify
self.log_responses_to = None
if log_responses:
self.log_responses_to = str(Path(storage_path) / 'responses.log')
class MoodleService:
def __init__(
self,
config_helper: ConfigHelper,
storage_path: str,
skip_cert_verify: bool = False,
log_responses: bool = False,
):
self.config_helper = config_helper
self.storage_path = storage_path
self.recorder = StateRecorder(Path(storage_path) / 'moodle_state.db')
self.skip_cert_verify = skip_cert_verify
self.log_responses_to = None
if log_responses:
self.log_responses_to = str(Path(storage_path) / 'responses.log')
def interactively_acquire_token(
self, use_stored_url: bool = False, username: str = None, password: str = None
) -> str:
"""
Walks the user through executing a login into the Moodle-System to get
the Token and saves it.
@return: The Token for Moodle.
"""
automated = False
stop_automatic_generation = False
if username is not None and password is not None:
automated = True
if not automated:
print('[The following Credentials are not saved, it is only used temporarily to generate a login token.]')
moodle_token = None
while moodle_token is None:
if stop_automatic_generation and automated:
break
if not use_stored_url:
moodle_url = input('URL of Moodle: ')
use_http = False
if moodle_url.startswith('http://'):
Log.error(
'Warning: You have entered an insecure URL! Are you sure that the Moodle is'
+ ' not accessible via `https://`? All your data will be transferred'
+ ' insecurely! If your Moodle is accessible via `https://`, then run'
+ ' the process again using `https://` to protect your data.'
)
use_http = True
elif not moodle_url.startswith('https://'):
Log.error('The url of your moodle must start with `https://`')
continue
moodle_uri = urlparse(moodle_url)
moodle_domain, moodle_path = self._split_moodle_uri(moodle_uri)
else:
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
use_http = self.config_helper.get_use_http()
if username is not None:
moodle_username = username
stop_automatic_generation = True
else:
moodle_username = input('Username for Moodle: ')
if password is not None:
moodle_password = password
else:
moodle_password = getpass('Password for Moodle [no output]: ')
try:
moodle_token, moodle_privatetoken = login_helper.obtain_login_token(
moodle_username,
moodle_password,
moodle_domain,
moodle_path,
self.skip_cert_verify,
use_http,
)
except RequestRejectedError as error:
Log.error('Login Failed! (%s) Please try again.' % (error))
except (ValueError, RuntimeError) as error:
Log.error('Error while communicating with the Moodle System! (%s) Please try again.' % (error))
except ConnectionError as error:
Log.error(str(error))
if automated is True and moodle_token is None:
sys.exit(1)
# Saves the created token and the successful Moodle parameters.
self.config_helper.set_property('token', moodle_token)
if moodle_privatetoken is not None:
self.config_helper.set_property('privatetoken', moodle_privatetoken)
self.config_helper.set_property('moodle_domain', moodle_domain)
self.config_helper.set_property('moodle_path', moodle_path)
if use_http is True:
self.config_helper.set_property('use_http', use_http)
return moodle_token
def interactively_acquire_sso_token(self, use_stored_url: bool = False) -> str:
"""
Walks the user through the receiving of a SSO token for the
Moodle-System and saves it.
@return: The Token for Moodle.
"""
if not use_stored_url:
moodle_url = input('URL of Moodle: ')
moodle_uri = urlparse(moodle_url)
moodle_domain, moodle_path = self._split_moodle_uri(moodle_uri)
else:
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
use_http = self.config_helper.get_use_http()
scheme = 'https://'
if use_http:
scheme = 'http://'
atomatic_procedure_warning = (
'Between version 3.81 and 3.82 a change was added to'
+ ' Moodle so that automatic copying of the SSO token'
+ ' might not work.'
)
try:
version = RequestHelper(
moodle_domain,
moodle_path,
skip_cert_verify=self.skip_cert_verify,
use_http=use_http,
).get_simple_moodle_version()
if StrictVersion(version) > StrictVersion("3.8.1"):
Log.warning(atomatic_procedure_warning + '\nYou can still try it, your version is: ' + str(version))
except ConnectionError:
Log.warning(
atomatic_procedure_warning
+ '\nThe version of your Moodle could not be detected, you can still try the automatic procedure.'
)
print(' If you want to copy the login-token manual, you will be guided through the manual copy process.')
do_automatic = cutie.prompt_yes_or_no('Do you want to try to receive the SSO token automatically?')
print('Please log into Moodle on this computer and then visit the following address in your web browser: ')
if do_automatic:
print(
scheme
+ moodle_domain
+ moodle_path
+ 'admin/tool/mobile/launch.php?service='
+ 'moodle_mobile_app&passport=12345&'
+ 'urlscheme=http%3A%2F%2Flocalhost'
)
moodle_token = sso_token_receiver.receive_token()
else:
print(
scheme
+ moodle_domain
+ moodle_path
+ 'admin/tool/mobile/launch.php?service='
+ 'moodle_mobile_app&passport=12345&urlscheme=moodledownloader'
)
print(
'If you open the link in the browser, no web page should'
+ ' load, instead an error will occur. Open the'
+ ' developer console (press F12) and go to the Network Tab,'
+ ' if there is no error, reload the web page.'
)
print(
'Copy the link address of the website that could not be'
+ ' loaded (right click, then click on Copy, then click'
+ ' on copy link address).'
)
print(
'The script expects a URL that looks something like this:'
+ '`moodledownloader://token=$apptoken`.'
+ ' Where $apptoken looks random. In reality it is a Base64'
+ ' encoded hash and the token we need to access moodle.'
)
token_address = input('Then insert the address here: ')
moodle_token, moodle_privatetoken = sso_token_receiver.extract_token(token_address)
if moodle_token is None:
raise ValueError('Invalid URL!')
# Saves the created token and the successful Moodle parameters.
self.config_helper.set_property('token', moodle_token)
if moodle_privatetoken is not None:
self.config_helper.set_property('privatetoken', moodle_privatetoken)
self.config_helper.set_property('moodle_domain', moodle_domain)
self.config_helper.set_property('moodle_path', moodle_path)
return moodle_token
def fetch_state(self) -> [Course]:
"""
Gets the current status of the configured Moodle account and compares
it with the last known status for changes. It does not change the
known state, nor does it download the files.
@return: List with detected changes
"""
logging.debug('Fetching current Moodle State...')
token = self.config_helper.get_token()
privatetoken = self.config_helper.get_privatetoken()
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
use_http = self.config_helper.get_use_http()
request_helper = RequestHelper(
moodle_domain,
moodle_path,
token,
self.skip_cert_verify,
self.log_responses_to,
use_http,
)
first_contact_handler = FirstContactHandler(request_helper)
results_handler = ResultsHandler(request_helper, moodle_domain, moodle_path)
download_course_ids = self.config_helper.get_download_course_ids()
download_public_course_ids = self.config_helper.get_download_public_course_ids()
dont_download_course_ids = self.config_helper.get_dont_download_course_ids()
download_submissions = self.config_helper.get_download_submissions()
download_databases = self.config_helper.get_download_databases()
download_forums = self.config_helper.get_download_forums()
download_quizzes = self.config_helper.get_download_quizzes()
download_lessons = self.config_helper.get_download_lessons()
download_workshops = self.config_helper.get_download_workshops()
download_also_with_cookie = self.config_helper.get_download_also_with_cookie()
courses = []
filtered_courses = []
cookie_handler = None
print('\rDownloading account information\033[K', end='')
userid, version = self.config_helper.get_userid_and_version()
if userid is None or version is None:
userid, version = first_contact_handler.fetch_userid_and_version()
else:
first_contact_handler.version = version
assignments_handler = AssignmentsHandler(request_helper, version)
databases_handler = DatabasesHandler(request_helper, version)
forums_handler = ForumsHandler(request_helper, version)
quizzes_handler = QuizzesHandler(request_helper, version)
lessons_handler = LessonsHandler(request_helper, version)
workshops_handler = WorkshopsHandler(request_helper, version)
pages_handler = PagesHandler(request_helper, version)
folders_handler = FoldersHandler(request_helper, version)
results_handler.setVersion(version)
if download_also_with_cookie:
# generate a new cookie if necessary
cookie_handler = CookieHandler(request_helper, version, self.storage_path)
cookie_handler.check_and_fetch_cookies(privatetoken, userid)
courses_list = first_contact_handler.fetch_courses(userid)
courses = []
# Filter unselected courses
for course in courses_list:
if ResultsHandler.should_download_course(course.id, download_course_ids, dont_download_course_ids):
courses.append(course)
public_courses_list = first_contact_handler.fetch_courses_info(download_public_course_ids)
for course in public_courses_list:
courses.append(course)
assignments = assignments_handler.fetch_assignments(courses)
if download_submissions:
assignments = assignments_handler.fetch_submissions(userid, assignments)
databases = databases_handler.fetch_databases(courses)
if download_databases:
databases = databases_handler.fetch_database_files(databases)
forums = forums_handler.fetch_forums(courses)
if download_forums:
last_timestamps_per_forum = self.recorder.get_last_timestamps_per_forum()
forums = forums_handler.fetch_forums_posts(forums, last_timestamps_per_forum)
quizzes = quizzes_handler.fetch_quizzes(courses)
if download_quizzes:
quizzes = quizzes_handler.fetch_quizzes_files(userid, quizzes)
lessons = lessons_handler.fetch_lessons(courses)
if download_lessons:
lessons = lessons_handler.fetch_lessons_files(userid, lessons)
workshops = workshops_handler.fetch_workshops(courses)
if download_workshops:
workshops = workshops_handler.fetch_workshops_files(userid, workshops)
pages = pages_handler.fetch_pages(courses)
folders = folders_handler.fetch_folders(courses)
courses = self.add_options_to_courses(courses)
index = 0
for course in courses:
index += 1
# to limit the output to one line
limits = shutil.get_terminal_size()
shorted_course_name = course.fullname
if len(course.fullname) > 17:
shorted_course_name = course.fullname[:15] + '..'
into = '\rDownloading course information'
status_message = into + ' %3d/%3d [%-17s|%6s]' % (index, len(courses), shorted_course_name, course.id)
if len(status_message) > limits.columns:
status_message = status_message[0 : limits.columns]
print(status_message + '\033[K', end='')
course_fetch_addons = {
'assign': assignments.get(course.id, {}),
'data': databases.get(course.id, {}),
'forum': forums.get(course.id, {}),
'quiz': quizzes.get(course.id, {}),
'lesson': lessons.get(course.id, {}),
'workshop': workshops.get(course.id, {}),
'page': pages.get(course.id, {}),
'folder': folders.get(course.id, {}),
}
results_handler.set_fetch_addons(course_fetch_addons)
course.files = results_handler.fetch_files(course)
filtered_courses.append(course)
print('')
logging.debug('Checking for changes...')
changes = self.recorder.changes_of_new_version(filtered_courses)
# Filter changes
changes = self.add_options_to_courses(changes)
changes = self.filter_courses(changes, self.config_helper, cookie_handler, courses_list + public_courses_list)
return changes
def add_options_to_courses(self, courses: [Course]):
"""
Updates a array of courses with its options
"""
options_of_courses = self.config_helper.get_options_of_courses()
for course in courses:
options = options_of_courses.get(str(course.id), None)
if options is not None:
course.overwrite_name_with = options.get('overwrite_name_with', None)
course.create_directory_structure = options.get('create_directory_structure', True)
course.excluded_sections = options.get("excluded_sections", [])
return courses
@staticmethod
def filter_courses(
changes: [Course],
config_helper: ConfigHelper,
cookie_handler: CookieHandler = None,
courses_list: [Course] = None,
) -> [Course]:
"""
Filters the changes course list from courses that
should not get downloaded
@param config_helper: ConfigHelper to obtain all the diffrent filter configs
@param cookie_handler: CookieHandler to check if the cookie is valid
@param courses_list: A list of all courses that are available online
@return: filtered changes course list
"""
download_course_ids = config_helper.get_download_course_ids()
download_public_course_ids = config_helper.get_download_public_course_ids()
dont_download_course_ids = config_helper.get_dont_download_course_ids()
download_submissions = config_helper.get_download_submissions()
download_descriptions = config_helper.get_download_descriptions()
download_links_in_descriptions = config_helper.get_download_links_in_descriptions()
download_databases = config_helper.get_download_databases()
download_quizzes = config_helper.get_download_quizzes()
download_lessons = config_helper.get_download_lessons()
download_workshops = config_helper.get_download_workshops()
exclude_file_extensions = config_helper.get_exclude_file_extensions()
download_also_with_cookie = config_helper.get_download_also_with_cookie()
if cookie_handler is not None:
download_also_with_cookie = cookie_handler.test_cookies()
filtered_changes = []
for course in changes:
if not ResultsHandler.should_download_course(
course.id, download_course_ids + download_public_course_ids, dont_download_course_ids
):
# Filter courses that should not be downloaded
continue
if courses_list is not None:
not_online = True
# Filter courses that are not available online
for online_course in courses_list:
if online_course.id == course.id:
not_online = False
break
if not_online:
Log.warning(f'The Moodle course with id {course.id} is no longer available online.')
logging.warning('The Moodle course with id %d is no longer available online.', course.id)
continue
course_files = []
for file in course.files:
# Filter Files based on options
if (
# Filter Assignment Submission Files
(download_submissions or (not (file.module_modname.endswith('assign') and file.deleted)))
# Filter Description Files (except the forum posts)
and (
download_descriptions
or file.content_type != 'description'
or (file.module_modname == 'forum' and file.content_type == 'description' and file.content_filename != 'Forum intro')
)
# Filter Database Files
and (download_databases or file.content_type != 'database_file')
# Filter Quiz Files
and (download_quizzes or (not (file.module_modname.endswith('quiz') and file.deleted)))
# Filter Lesson Files
and (download_lessons or (not (file.module_modname.endswith('lesson') and file.deleted)))
# Filter Workshops Files
and (download_workshops or (not (file.module_modname.endswith('workshop') and file.deleted)))
# Filter Files that requiere a Cookie
and (download_also_with_cookie or (not file.module_modname.startswith('cookie_mod-')))
# Exclude files whose file extension is blacklisted
and (not (determine_ext(file.content_filename) in exclude_file_extensions))
# Exclude files that are in excluded sections
and (ResultsHandler.should_download_section(file.section_id, course.excluded_sections))
):
course_files.append(file)
course.files = course_files
# Filter Description URLs
course_files = []
for file in course.files:
if not file.content_type == 'description-url':
course_files.append(file)
elif download_links_in_descriptions:
add_description_url = True
for test_file in course.files:
if file.content_fileurl == test_file.content_fileurl:
if test_file.content_type != 'description-url':
# If a URL in a description also exists as a real link in the course,
# then ignore this URL
add_description_url = False
break
elif file.module_id > test_file.module_id:
# Always use the link from the older description.
add_description_url = False
break
if add_description_url:
course_files.append(file)
course.files = course_files
if len(course.files) > 0:
filtered_changes.append(course)
return filtered_changes
@staticmethod
def _split_moodle_uri(moodle_uri: str):
"""
Splits a given Moodle-Uri into the domain and the installation path
@return: moodle_domain, moodle_path as strings
"""
moodle_domain = moodle_uri.netloc
moodle_path = moodle_uri.path
if not moodle_path.endswith('/'):
moodle_path = moodle_path + '/'
if moodle_path == '':
moodle_path = '/'
return moodle_domain, moodle_path
def __init__(self, config_helper: ConfigHelper, storage_path: str):
self.config_helper = config_helper
self.storage_path = storage_path
self.state_recorder = StateRecorder(
Path(storage_path) / 'moodle_state.db')
class OfflineService:
def __init__(self, config_helper: ConfigHelper, storage_path: str):
self.config_helper = config_helper
self.storage_path = storage_path
self.state_recorder = StateRecorder(
Path(storage_path) / 'moodle_state.db')
def interactively_manage_database(self):
RESET_SEQ = '\033[0m'
COLOR_SEQ = '\033[1;%dm'
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
stored_files = self.state_recorder.get_stored_files()
stored_files = MoodleService.filter_courses(stored_files,
self.config_helper)
if len(stored_files) <= 0:
return
print('This management tool will navigate you through a menu to' +
' selectively remove file entries from the database so' +
' that these files can be downloaded again.')
course_options = []
courses = []
for course in stored_files:
for course_file in course.files:
if not os.path.exists(course_file.saved_to):
course_options.append(COLOR_SEQ % BLUE + course.fullname +
RESET_SEQ)
courses.append(course)
break
print('Choose one of the courses:')
print('[Confirm your selection with the Enter key]')
print('')
selected_course_id = cutie.select(options=course_options)
selected_course = courses[selected_course_id]
section_options = []
sections = []
for course_file in selected_course.files:
if not os.path.exists(course_file.saved_to) and (
course_file.section_name not in sections):
section_options.append(COLOR_SEQ % MAGENTA +
course_file.section_name + RESET_SEQ)
sections.append(course_file.section_name)
print('From which sections you want to select files.')
print(
'[You can select with the space bar and confirm your selection with the enter key]'
)
print('')
selected_sections_ids = cutie.select_multiple(options=section_options,
minimal_count=1)
selected_sections = []
for selected_sections_id in selected_sections_ids:
if selected_sections_id < len(sections):
selected_sections.append(sections[selected_sections_id])
file_options = []
files = []
for course_file in selected_course.files:
if not os.path.exists(course_file.saved_to) and (
course_file.section_name in selected_sections):
file_options.append(COLOR_SEQ % CYAN +
course_file.content_filename + RESET_SEQ)
files.append(course_file)
print(
'Which of the files should be removed form the database, so that they will be redownloaded?'
)
print(
'[You can select with the space bar and confirm your selection with the enter key]'
)
print('')
selected_files = cutie.select_multiple(options=file_options)
files_to_delete = []
for file_index in selected_files:
if file_index < len(files) and isinstance(files[file_index], File):
files_to_delete.append(files[file_index])
self.state_recorder.batch_delete_files_from_db(files_to_delete)
class OfflineService:
def __init__(self, config_helper: ConfigHelper, storage_path: str):
self.config_helper = config_helper
self.storage_path = storage_path
self.state_recorder = StateRecorder(Path(storage_path) / 'moodle_state.db')
def interactively_manage_database(self):
RESET_SEQ = '\033[0m'
COLOR_SEQ = '\033[1;%dm'
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
stored_files = self.state_recorder.get_stored_files()
stored_files = MoodleService.filter_courses(stored_files, self.config_helper)
if len(stored_files) <= 0:
return
course_options = []
courses = []
for course in stored_files:
for course_file in course.files:
if not os.path.exists(course_file.saved_to):
course_options.append(COLOR_SEQ % BLUE + course.fullname + RESET_SEQ)
courses.append(course)
break
print(
'This management tool will navigate you through a menu to'
+ ' selectively remove file entries from the database so'
+ ' that these files can be downloaded again.'
)
Log.warning(
'Only files that are missing locally but stored in the local'
+ ' database are displayed in this tool. If a file is not missing'
+ ' from a course, it will not be listed here at all. Also, only'
+ ' courses that are selected for download are displayed.'
)
Log.critical(
'For more complicated operations on the database a DB browser for SQLite'
+ ' is advantageous (https://sqlitebrowser.org/).'
)
if not courses:
print('No files are missing locally but stored in the local database. Nothing to do.')
return
print('Choose one of the courses:')
print('[Confirm your selection with the Enter key]')
print('')
selected_course_id = cutie.select(options=course_options)
selected_course = courses[selected_course_id]
section_options = []
sections = []
# Add the option to select all sections
section_options.append(COLOR_SEQ % MAGENTA + '[All sections]' + RESET_SEQ)
sections.append(None) # Add None at index 0 to avoid index shifting
for course_file in selected_course.files:
if not os.path.exists(course_file.saved_to) and (course_file.section_name not in sections):
section_options.append(COLOR_SEQ % MAGENTA + course_file.section_name + RESET_SEQ)
sections.append(course_file.section_name)
print('From which sections you want to select files?')
print('[You can select with the space bar and confirm your selection with the enter key]')
print('')
selected_sections_ids = cutie.select_multiple(options=section_options, minimal_count=1)
selected_sections = []
for selected_sections_id in selected_sections_ids:
if selected_sections_id == 0:
selected_sections = sections[1:]
break
elif (selected_sections_id) < len(sections):
selected_sections.append(sections[selected_sections_id])
file_options = []
files = []
# Add the option to select all files
file_options.append(COLOR_SEQ % CYAN + '[All files]' + RESET_SEQ)
files.append(None) # Add None at index 0 to avoid index shifting
for course_file in selected_course.files:
if not os.path.exists(course_file.saved_to) and (course_file.section_name in selected_sections):
file_options.append(COLOR_SEQ % CYAN + course_file.content_filename + RESET_SEQ)
files.append(course_file)
print('Which of the files should be removed form the database, so that they will be redownloaded?')
print('[You can select with the space bar and confirm your selection with the enter key]')
print('')
selected_files = cutie.select_multiple(options=file_options)
files_to_delete = []
for file_index in selected_files:
if file_index == 0: # If all files is selected
for file_to_delete in files[1:]: # Ignore the first element of the array set as None
if isinstance(file_to_delete, File):
files_to_delete.append(file_to_delete)
break
elif file_index < len(files) and isinstance(files[file_index], File):
files_to_delete.append(files[file_index])
self.state_recorder.batch_delete_files_from_db(files_to_delete)
def delete_old_files(self):
RESET_SEQ = '\033[0m'
COLOR_SEQ = '\033[1;%dm'
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
stored_files = self.state_recorder.get_old_files()
if len(stored_files) <= 0:
print('No old copies of files found. Nothing to do.')
return
print(
'This management tool will navigate you through a menu to'
+ ' selectively remove old copies of files from the database '
+ ' and form the file system'
)
course_options = []
for course in stored_files:
course_options.append(COLOR_SEQ % BLUE + course.fullname + RESET_SEQ)
print('Choose one of the courses:')
print('[Confirm your selection with the Enter key]')
print('')
selected_course_id = cutie.select(options=course_options)
selected_course = stored_files[selected_course_id]
section_options = []
sections = []
# Add the option to select all sections
section_options.append(COLOR_SEQ % MAGENTA + '[All sections]' + RESET_SEQ)
sections.append(None) # Add None at index 0 to avoid index shifting
for course_file in selected_course.files:
if course_file.section_name not in sections:
section_options.append(COLOR_SEQ % MAGENTA + course_file.section_name + RESET_SEQ)
sections.append(course_file.section_name)
print('From which sections you want to delete old files?')
print('[You can select with the space bar and confirm your selection with the enter key]')
print('')
selected_sections_ids = cutie.select_multiple(options=section_options, minimal_count=1)
selected_sections = []
for selected_sections_id in selected_sections_ids:
if selected_sections_id == 0:
selected_sections = sections[1:]
break
elif (selected_sections_id) < len(sections):
selected_sections.append(sections[selected_sections_id])
file_options = []
files = []
# Add the option to select all files
file_options.append(COLOR_SEQ % CYAN + '[All files]' + RESET_SEQ)
files.append(None) # Add None at index 0 to avoid index shifting
for course_file in selected_course.files:
if course_file.section_name in selected_sections:
file_options.append(COLOR_SEQ % CYAN + course_file.content_filename + RESET_SEQ)
files.append(course_file)
print('Which of the files should be deleted?')
print('[You can select with the space bar and confirm your selection with the enter key]')
print('')
selected_files = cutie.select_multiple(options=file_options)
files_to_delete = []
for file_index in selected_files:
if file_index == 0: # If all files is selected
for file_to_delete in files[1:]: # Ignore the first element of the array set as None
if isinstance(file_to_delete, File):
files_to_delete.append(file_to_delete)
if os.path.exists(file_to_delete.saved_to):
os.remove(file_to_delete.saved_to)
break
elif file_index < len(files) and isinstance(files[file_index], File):
files_to_delete.append(files[file_index])
if os.path.exists(files[file_index].saved_to):
os.remove(files[file_index].saved_to)
self.state_recorder.batch_delete_files_from_db(files_to_delete)
class MoodleService:
def __init__(
self,
config_helper: ConfigHelper,
storage_path: str,
skip_cert_verify: bool = False,
log_responses: bool = False,
):
self.config_helper = config_helper
self.storage_path = storage_path
self.recorder = StateRecorder(Path(storage_path) / 'moodle_state.db')
self.skip_cert_verify = skip_cert_verify
self.log_responses_to = None
if log_responses:
self.log_responses_to = str(Path(storage_path) / 'responses.log')
def interactively_acquire_token(self, use_stored_url: bool = False) -> str:
"""
Walks the user through executing a login into the Moodle-System to get
the Token and saves it.
@return: The Token for Moodle.
"""
print(
'[The following Credentials are not saved, it is only used temporarily to generate a login token.]'
)
moodle_token = None
while moodle_token is None:
if not use_stored_url:
moodle_url = input('URL of Moodle: ')
if not moodle_url.startswith('https://'):
Log.error(
'The url of your moodle must start with `https://`')
continue
moodle_uri = urlparse(moodle_url)
moodle_domain, moodle_path = self._split_moodle_uri(moodle_uri)
else:
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
moodle_username = input('Username for Moodle: ')
moodle_password = getpass('Password for Moodle [no output]: ')
try:
moodle_token, moodle_privatetoken = login_helper.obtain_login_token(
moodle_username, moodle_password, moodle_domain,
moodle_path, self.skip_cert_verify)
except RequestRejectedError as error:
print('Login Failed! (%s) Please try again.' % (error))
except (ValueError, RuntimeError) as error:
print(
'Error while communicating with the Moodle System! (%s) Please try again.'
% (error))
# Saves the created token and the successful Moodle parameters.
self.config_helper.set_property('token', moodle_token)
if moodle_privatetoken is not None:
self.config_helper.set_property('privatetoken',
moodle_privatetoken)
self.config_helper.set_property('moodle_domain', moodle_domain)
self.config_helper.set_property('moodle_path', moodle_path)
return moodle_token
def interactively_acquire_sso_token(self,
use_stored_url: bool = False) -> str:
"""
Walks the user through the receiving of a SSO token for the
Moodle-System and saves it.
@return: The Token for Moodle.
"""
if not use_stored_url:
moodle_url = input('URL of Moodle: ')
moodle_uri = urlparse(moodle_url)
moodle_domain, moodle_path = self._split_moodle_uri(moodle_uri)
else:
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
version = RequestHelper(
moodle_domain, moodle_path, '',
self.skip_cert_verify).get_simple_moodle_version()
if version > 3.8:
print('Between version 3.81 and 3.82 a change was added to' +
' Moodle so that automatic copying of the SSO token' +
' might not work.' +
'\nYou can still try it, your version is: ' + str(version))
print(
' If you want to copy the login-token manual, you will be guided through the manual copy process.'
)
do_automatic = cutie.prompt_yes_or_no(
'Do you want to try to receive the SSO token automatically?')
print(
'Please log into Moodle on this computer and then visit the following address in your web browser: '
)
if do_automatic:
print('https://' + moodle_domain + moodle_path +
'admin/tool/mobile/launch.php?service=' +
'moodle_mobile_app&passport=12345&' +
'urlscheme=http%3A%2F%2Flocalhost')
moodle_token = sso_token_receiver.receive_token()
else:
print(
'https://' + moodle_domain + moodle_path +
'admin/tool/mobile/launch.php?service=' +
'moodle_mobile_app&passport=12345&urlscheme=moodledownloader')
print('If you open the link in the browser, no web page should' +
' load, instead an error will occur. Open the' +
' developer console (press F12) and go to the Network Tab,' +
' if there is no error, reload the web page.')
print('Copy the link address of the website that could not be' +
' loaded (right click, then click on Copy, then click' +
' on copy link address).')
print('The script expects a URL that looks something like this:' +
'`moodledownloader://token=$apptoken`.' +
' Where $apptoken looks random. In reality it is a Base64' +
' encoded hash and the token we need to access moodle.')
token_address = input('Then insert the address here: ')
moodle_token, moodle_privatetoken = sso_token_receiver.extract_token(
token_address)
if moodle_token is None:
raise ValueError('Invalid URL!')
# Saves the created token and the successful Moodle parameters.
self.config_helper.set_property('token', moodle_token)
if moodle_privatetoken is not None:
self.config_helper.set_property('privatetoken',
moodle_privatetoken)
self.config_helper.set_property('moodle_domain', moodle_domain)
self.config_helper.set_property('moodle_path', moodle_path)
return moodle_token
def fetch_state(self) -> [Course]:
"""
Gets the current status of the configured Moodle account and compares
it with the last known status for changes. It does not change the
known state, nor does it download the files.
@return: List with detected changes
"""
logging.debug('Fetching current Moodle State...')
token = self.config_helper.get_token()
privatetoken = self.config_helper.get_privatetoken()
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
request_helper = RequestHelper(moodle_domain, moodle_path, token,
self.skip_cert_verify,
self.log_responses_to)
first_contact_handler = FirstContactHandler(request_helper)
results_handler = ResultsHandler(request_helper, moodle_domain,
moodle_path)
download_course_ids = self.config_helper.get_download_course_ids()
dont_download_course_ids = self.config_helper.get_dont_download_course_ids(
)
download_submissions = self.config_helper.get_download_submissions()
download_descriptions = self.config_helper.get_download_descriptions()
download_links_in_descriptions = self.config_helper.get_download_links_in_descriptions(
)
download_databases = self.config_helper.get_download_databases()
courses = []
filtered_courses = []
try:
print('\rDownloading account information\033[K', end='')
userid, version = first_contact_handler.fetch_userid_and_version()
assignments_handler = AssignmentsHandler(request_helper, version)
databases_handler = DatabasesHandler(request_helper, version)
results_handler.setVersion(version)
# generate a new cookie if necessary
cookie_handler = CookieHandler(request_helper, version,
self.storage_path, moodle_domain,
moodle_path)
cookie_handler.check_and_fetch_cookies(privatetoken, userid)
courses_list = first_contact_handler.fetch_courses(userid)
courses = []
# Filter unselected courses
for course in courses_list:
if ResultsHandler.should_download_course(
course.id, download_course_ids,
dont_download_course_ids):
courses.append(course)
assignments = assignments_handler.fetch_assignments(courses)
if download_submissions:
assignments = assignments_handler.fetch_submissions(
userid, assignments)
databases = databases_handler.fetch_databases(courses)
if download_databases:
databases = databases_handler.fetch_database_files(databases)
index = 0
for course in courses:
index += 1
# to limit the output to one line
limits = shutil.get_terminal_size()
shorted_course_name = course.fullname
if len(course.fullname) > 17:
shorted_course_name = course.fullname[:15] + '..'
into = '\rDownloading course information'
status_message = into + ' %3d/%3d [%17s|%6s]' % (
index, len(courses), shorted_course_name, course.id)
if len(status_message) > limits.columns:
status_message = status_message[0:limits.columns]
print(status_message + '\033[K', end='')
course_assignments = assignments.get(course.id, {})
course_databases = databases.get(course.id, {})
results_handler.set_fetch_addons(course_assignments,
course_databases)
course.files = results_handler.fetch_files(course.id)
filtered_courses.append(course)
print('')
except (RequestRejectedError, ValueError, RuntimeError) as error:
raise RuntimeError(
'Error while communicating with the Moodle System! (%s)' %
(error))
logging.debug('Checking for changes...')
changes = self.recorder.changes_of_new_version(filtered_courses)
# Filter changes
changes = self.filter_courses(
changes,
download_course_ids,
dont_download_course_ids,
download_submissions,
download_descriptions,
download_links_in_descriptions,
download_databases,
)
changes = self.add_options_to_courses(changes)
return changes
def add_options_to_courses(self, courses: [Course]):
"""
Updates a array of courses with its options
"""
options_of_courses = self.config_helper.get_options_of_courses()
for course in courses:
options = options_of_courses.get(str(course.id), None)
if options is not None:
course.overwrite_name_with = options.get(
'overwrite_name_with', None)
course.create_directory_structure = options.get(
'create_directory_structure', True)
return courses
@staticmethod
def filter_courses(
changes: [Course],
download_course_ids: [int],
dont_download_course_ids: [int],
download_submissions: bool,
download_descriptions: bool,
download_links_in_descriptions: bool,
download_databases: bool,
) -> [Course]:
"""
Filters the changes course list from courses that
should not get downloaded
@param download_course_ids: list of course ids
that should be downloaded
@param dont_download_course_ids: list of course ids
that should not be downloaded
@param download_submissions: boolean if submissions
should be downloaded
@param download_descriptions: boolean if descriptions
should be downloaded
@param download_links_in_descriptions: boolean if links in descriptions should be downloaded
@param download_databases: boolean if databases should be downloaded
@return: filtered changes course list
"""
filtered_changes = []
for course in changes:
if not download_submissions:
course_files = []
for file in course.files:
if file.content_type != 'submission_file':
course_files.append(file)
course.files = course_files
if not download_descriptions:
course_files = []
for file in course.files:
if file.content_type != 'description':
course_files.append(file)
course.files = course_files
if not download_links_in_descriptions:
course_files = []
for file in course.files:
if not file.module_modname.endswith('-description'):
course_files.append(file)
course.files = course_files
if not download_databases:
course_files = []
for file in course.files:
if file.content_type != 'database_file':
course_files.append(file)
course.files = course_files
if (ResultsHandler.should_download_course(
course.id, download_course_ids, dont_download_course_ids)
and len(course.files) > 0):
filtered_changes.append(course)
return filtered_changes
@staticmethod
def _split_moodle_uri(moodle_uri: str):
"""
Splits a given Moodle-Uri into the domain and the installation path
@return: moodle_domain, moodle_path as strings
"""
moodle_domain = moodle_uri.netloc
moodle_path = moodle_uri.path
if not moodle_path.endswith('/'):
moodle_path = moodle_path + '/'
if moodle_path == '':
moodle_path = '/'
return moodle_domain, moodle_path