def run_configure(storage_path, skip_cert_verify=False):
config = ConfigHelper(storage_path)
config.load() # because we do not want to override the other settings
ConfigService(config, storage_path, skip_cert_verify).interactively_acquire_config()
Log.success('Configuration successfully updated!')
def _select_sections_to_download(self, sections: [{}],
excluded: [int]) -> [int]:
"""
Asks the user for the sections that should be downloaded.
@param sections: All available sections
@param excluded sections currently excluded
"""
choices = []
defaults = []
for i, section in enumerate(sections):
section_id = section.get("id")
choices.append(('%5i\t%s' % (section_id, section.get("name"))))
if ResultsHandler.should_download_section(section_id, excluded):
defaults.append(i)
Log.special('Which of the sections should be downloaded?')
Log.info(
'[You can select with the space bar and confirm your selection with the enter key]'
)
print('')
selected_sections = cutie.select_multiple(options=choices,
ticked_indices=defaults)
dont_download_section_ids = []
for i, section in enumerate(sections):
if i not in selected_sections:
dont_download_section_ids.append(section.get("id"))
return dont_download_section_ids
def notify_about_failed_downloads(self, failed_downloads: [URLTarget]):
if len(failed_downloads) > 0:
print('')
Log.warning(
'Error while trying to download files, look at the log for more details. List of failed downloads:'
)
print('')
RESET_SEQ = '\033[0m'
COLOR_SEQ = '\033[1;%dm'
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
for url_target in failed_downloads:
print(
(COLOR_SEQ % CYAN)
+ url_target.file.content_filename
+ RESET_SEQ
+ (COLOR_SEQ % RED)
+ '\n\t'
+ str(url_target.error)
+ RESET_SEQ
)
print('')
def _select_should_download_descriptions(self):
"""
Asks the user if descriptions should be downloaded
"""
download_descriptions = self.config_helper.get_download_descriptions()
self.section_seperator()
Log.info(
'In Moodle courses, descriptions can be added to all kinds' +
' of resources, such as files, tasks, assignments or simply' +
' free text. These descriptions are usually unnecessary to' +
' download because you have already read the information or' +
' know it from context. However, there are situations where' +
' it might be interesting to download these descriptions. The' +
' descriptions are created as Markdown files and can be' +
' deleted as desired.')
Log.debug(
'Creating the description files does not take extra time, but they can be annoying'
+ ' if they only contain unnecessary information.')
print('')
download_descriptions = cutie.prompt_yes_or_no(
Log.special_str(
'Would you like to download descriptions of the courses you have selected?'
),
default_is_yes=download_descriptions,
)
self.config_helper.set_property('download_descriptions',
download_descriptions)
def run_change_notification_telegram(storage_path):
config = ConfigHelper(storage_path)
config.load()
TelegramService(config).interactively_configure()
Log.success('Telegram Configuration successfully updated!')
def run_change_notification_xmpp(storage_path):
config = ConfigHelper(storage_path)
config.load()
XmppService(config).interactively_configure()
Log.success('XMPP Configuration successfully updated!')
def _send_mail(self, subject, mail_content: (str, {str: str})):
"""
Sends an email
"""
if not self._is_configured():
return
mail_cfg = self.config_helper.get_property('mail')
try:
logging.debug('Sending Notification via Mail...')
Log.debug('Sending Notification via Mail... (Please wait)')
mail_shooter = MailShooter(
mail_cfg['sender'],
mail_cfg['server_host'],
int(mail_cfg['server_port']),
mail_cfg['username'],
mail_cfg['password'],
)
mail_shooter.send(mail_cfg['target'], subject, mail_content[0],
mail_content[1])
except BaseException as e:
error_formatted = traceback.format_exc()
logging.error('While sending notification:\n%s',
error_formatted,
extra={'exception': e})
raise e # to be properly notified via Sentry
def run_add_all_visible_courses(storage_path, skip_cert_verify):
config = ConfigHelper(storage_path)
config.load() # because we do not want to override the other settings
ConfigService(config, storage_path, skip_cert_verify).interactively_add_all_visible_courses()
Log.success('Configuration successfully updated!')
def run_delete_old_files(storage_path):
config = ConfigHelper(storage_path)
config.load() # Not really needed, we check all local courses
offline_service = OfflineService(config, storage_path)
offline_service.delete_old_files()
Log.success('All done.')
def run_manage_database(storage_path):
config = ConfigHelper(storage_path)
config.load() # because we want to only manage configured courses
offline_service = OfflineService(config, storage_path)
offline_service.interactively_manage_database()
Log.success('All done.')
def interactively_acquire_config(self):
"""
Guides the user through the process of configuring the downloader
for the courses to be downloaded and in what way
"""
token = self.config_helper.get_token()
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
use_http = self.config_helper.get_use_http()
request_helper = RequestHelper(moodle_domain,
moodle_path,
token,
self.skip_cert_verify,
use_http=use_http)
first_contact_handler = FirstContactHandler(request_helper)
courses = []
try:
userid, version = self.config_helper.get_userid_and_version()
if userid is None or version is None:
userid, version = first_contact_handler.fetch_userid_and_version(
)
self._select_should_userid_and_version_be_saved(
userid, version)
else:
first_contact_handler.version = version
courses = first_contact_handler.fetch_courses(userid)
except (RequestRejectedError, ValueError, RuntimeError,
ConnectionError) as error:
Log.error(
'Error while communicating with the Moodle System! (%s)' %
(error))
sys.exit(1)
course_ids = self._select_courses_to_download(courses)
sections = {}
for course_id in course_ids:
sections[course_id] = first_contact_handler.fetch_sections(
course_id)
self._set_options_of_courses(courses, sections)
self._select_should_download_submissions()
self._select_should_download_descriptions()
self._select_should_download_links_in_descriptions()
self._select_should_download_databases()
self._select_should_download_forums()
self._select_should_download_quizzes()
self._select_should_download_lessons()
self._select_should_download_workshops()
self._select_should_download_linked_files()
self._select_should_download_also_with_cookie()
def _max_path_length_workaround(path):
# Working around MAX_PATH limitation on Windows (see
# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
if os.name == 'nt':
absfilepath = os.path.abspath(path)
path = '\\\\?\\' + absfilepath
Log.debug("Using absolute paths")
else:
Log.info("You are not on Windows, you don't need to use this workaround")
return path
def _change_settings_of(self, course: Course, options_of_courses: {}):
"""
Ask for a new Name for the course.
Then asks if a file structure should be created.
"""
current_course_settings = options_of_courses.get(str(course.id), None)
# create default settings
if current_course_settings is None:
current_course_settings = {
'original_name': course.fullname,
'overwrite_name_with': None,
'create_directory_structure': True,
}
changed = False
# Ask for new name
overwrite_name_with = input(
Log.special_str(
'Enter a new name for this Course [leave blank for "%s"]: ' %
(course.fullname, )))
if overwrite_name_with == '':
overwrite_name_with = None
if (overwrite_name_with != course.fullname
and current_course_settings.get('overwrite_name_with',
None) != overwrite_name_with):
current_course_settings.update(
{'overwrite_name_with': overwrite_name_with})
changed = True
# Ask if a file structure should be created
create_directory_structure = current_course_settings.get(
'create_directory_structure', True)
create_directory_structure = cutie.prompt_yes_or_no(
Log.special_str(
'Should a directory structure be created for this course?'),
default_is_yes=create_directory_structure,
)
if create_directory_structure is not current_course_settings.get(
'create_directory_structure', True):
changed = True
current_course_settings.update(
{'create_directory_structure': create_directory_structure})
if changed:
options_of_courses.update(
{str(course.id): current_course_settings})
self.config_helper.set_property('options_of_courses',
options_of_courses)
def run_new_token(storage_path, use_sso=False, username: str = None, password: str = None, skip_cert_verify=False):
config = ConfigHelper(storage_path)
config.load() # because we do not want to override the other settings
moodle = MoodleService(config, storage_path, skip_cert_verify)
if use_sso:
moodle.interactively_acquire_sso_token(use_stored_url=True)
else:
moodle.interactively_acquire_token(use_stored_url=True, username=username, password=password)
Log.success('New Token successfully saved!')
def interactively_acquire_token(self, use_stored_url: bool = False) -> str:
"""
Walks the user through executing a login into the Moodle-System to get
the Token and saves it.
@return: The Token for Moodle.
"""
print(
'[The following Credentials are not saved, it is only used temporarily to generate a login token.]'
)
moodle_token = None
while moodle_token is None:
if not use_stored_url:
moodle_url = input('URL of Moodle: ')
if not moodle_url.startswith('https://'):
Log.error(
'The url of your moodle must start with `https://`')
continue
moodle_uri = urlparse(moodle_url)
moodle_domain, moodle_path = self._split_moodle_uri(moodle_uri)
else:
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
moodle_username = input('Username for Moodle: ')
moodle_password = getpass('Password for Moodle [no output]: ')
try:
moodle_token, moodle_privatetoken = login_helper.obtain_login_token(
moodle_username, moodle_password, moodle_domain,
moodle_path, self.skip_cert_verify)
except RequestRejectedError as error:
print('Login Failed! (%s) Please try again.' % (error))
except (ValueError, RuntimeError) as error:
print(
'Error while communicating with the Moodle System! (%s) Please try again.'
% (error))
# Saves the created token and the successful Moodle parameters.
self.config_helper.set_property('token', moodle_token)
if moodle_privatetoken is not None:
self.config_helper.set_property('privatetoken',
moodle_privatetoken)
self.config_helper.set_property('moodle_domain', moodle_domain)
self.config_helper.set_property('moodle_path', moodle_path)
return moodle_token
def _select_should_load_default_filename_character_map(self):
"""
Asks the user if the default filename character map should be loaded
"""
filename_character_map = self.config_helper.get_filename_character_map(
)
if os.name != 'nt':
self.section_seperator()
Log.info(
'On Windows many characters are forbidden in filenames and paths, if you want, these characters can be'
+ ' automatically removed from filenames.')
Log.warning(
'If you want to view the downloaded files on Windows this is important!'
)
print('Current filename character map: {}'.format(
filename_character_map))
Log.special(
'Do you want to load the default filename character map for Windows?'
)
choices = [
'No, leave it as it was.',
'No, load the default linux filename character map.',
'Yes, load the default windows filename character map.',
]
print('[Confirm your selection with the Enter key]')
print('')
selected_map = cutie.select(options=choices)
if selected_map == 0:
return
elif selected_map == 1:
self.config_helper.set_default_filename_character_map(False)
elif selected_map == 2:
self.config_helper.set_default_filename_character_map(True)
else:
if filename_character_map != ConfigHelper.windows_map:
self.section_seperator()
Log.warning(
'Warning: Your current filename character map does not match the standard Windows'
+ ' filename character map!')
print('Current filename character map: {}'.format(
filename_character_map))
load_default_map = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to load the default filename character map for Windows?'
),
default_is_yes=False,
)
if load_default_map:
self.config_helper.set_default_filename_character_map(True)
def notify_about_changes_in_moodle(self, changes: [Course]) -> None:
"""
Creates a terminal output about the downloaded changes.
@param changes: A list of changed courses with changed files.
"""
RESET_SEQ = '\033[0m'
COLOR_SEQ = '\033[1;%dm'
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
print('\n')
diff_count = 0
for course in changes:
diff_count += len(course.files)
if diff_count > 0:
msg_changes = '%s changes found for the configured Moodle-Account.'
logging.info(msg_changes, diff_count)
Log.success(msg_changes % (diff_count))
for course in changes:
if len(course.files) == 0:
continue
print(COLOR_SEQ % BLUE + course.fullname + RESET_SEQ)
for file in course.files:
if file.modified:
print(COLOR_SEQ % YELLOW + '≠\t' + file.saved_to + RESET_SEQ)
elif file.moved:
if file.new_file is not None:
print(
COLOR_SEQ % CYAN
+ '<->\t'
+ (
file.saved_to
+ RESET_SEQ
+ COLOR_SEQ % GREEN
+ ' ==> '
+ file.new_file.saved_to
+ RESET_SEQ
)
)
else:
print(COLOR_SEQ % CYAN + '<->\t' + file.saved_to + RESET_SEQ)
elif file.deleted:
print(COLOR_SEQ % MAGENTA + '-\t' + file.saved_to + RESET_SEQ)
else:
print(COLOR_SEQ % GREEN + '+\t' + file.saved_to + RESET_SEQ)
print('\n')
def _log_failures(self):
"""
Logs errors if any have occurred.
"""
print('')
if len(self.report['failure']) > 0:
Log.warning(
'Error while trying to download files, look at the log for more details.'
)
for url_target in self.report['failure']:
Log.error('%s\t%s' %
(url_target.file.content_filename, url_target.error))
def _select_should_download_forums(self):
"""
Asks the user if forums should be downloaded
"""
download_forums = self.config_helper.get_download_forums()
self.section_seperator()
Log.info(
'In forums, students and teachers can discuss and exchange information together.'
)
print('')
download_forums = cutie.prompt_yes_or_no(
Log.special_str('Do you want to download forums of your courses?'),
default_is_yes=download_forums)
self.config_helper.set_property('download_forums', download_forums)
def _select_should_userid_and_version_be_saved(self, userid, version):
"""
Asks the user if the userid and version should be saved in the configuration
"""
print('')
Log.info('The user id and version number of Moodle are downloaded' +
' at the beginning of each run of the downloader.' +
' Since this data rarely changes, it can be saved in the' +
' configuration.')
Log.critical(
f'Your user id is `{userid}` and the moodle version is `{version}`'
)
print('')
save_userid_and_version = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to store the user id and version number of Moodle in the configuration?'
),
default_is_yes=False,
)
if save_userid_and_version:
Log.warning('Remember to delete the version number from the' +
' configuration once Moodle has been updated' +
' and then run the configurator again!')
self.config_helper.set_property('userid', userid)
self.config_helper.set_property('version', version)
self.section_seperator()
def _select_should_download_quizzes(self):
"""
Asks the user if quizzes should be downloaded
"""
download_quizzes = self.config_helper.get_download_quizzes()
self.section_seperator()
Log.info(
'Quizzes are tests that a student must complete in a course and are graded on.'
+
' Only quizzes that are in progress or have been completed will be downloaded.'
)
print('')
download_quizzes = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to download quizzes of your courses?'),
default_is_yes=download_quizzes)
self.config_helper.set_property('download_quizzes', download_quizzes)
def _select_should_download_workshops(self):
"""
Asks the user if workshops should be downloaded
"""
download_workshops = self.config_helper.get_download_workshops()
self.section_seperator()
Log.info(
'Workshops function according to the peer review process.' +
' Students can make submissions and have to assess submissions of other students. '
)
print('')
download_workshops = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to download workshops of your courses?'),
default_is_yes=download_workshops)
self.config_helper.set_property('download_workshops',
download_workshops)
def check_and_fetch_cookies(self, privatetoken: str, userid: str) -> bool:
if os.path.exists(self.cookies_path):
# test if still logged in.
if self.test_cookies(self.moodle_test_url):
return True
warning_msg = 'Moodle cookie has expired, an attempt is made to generate a new cookie.'
logging.warning(warning_msg)
Log.warning('\r' + warning_msg + '\033[K')
if privatetoken is None:
error_msg = (
'Moodle Cookies are not retrieved because no private token is set.'
+
' To set a private token, use the `--new-token` option (if necessary also with `--sso`).'
)
logging.warning(error_msg)
Log.error('\r' + error_msg + '\033[K')
self.delete_cookie_file()
return False
autologin_key = self.fetch_autologin_key(privatetoken)
if autologin_key is None:
error_msg = 'Failed to download autologin key!'
logging.debug(error_msg)
print('')
Log.error(error_msg)
self.delete_cookie_file()
return False
print('\rDownloading cookies\033[K', end='')
post_data = {'key': autologin_key.get('key', ''), 'userid': userid}
url = autologin_key.get('autologinurl', '')
cookies_response, cookies_session = self.request_helper.post_URL(
url, post_data, self.cookies_path)
moodle_test_url = cookies_response.url
if self.test_cookies(moodle_test_url):
return True
else:
error_msg = 'Failed to generate cookies!'
logging.debug(error_msg)
print('')
Log.error(error_msg)
self.delete_cookie_file()
return False
def _send_messages(self, messages: [str]):
"""
Sends an message
"""
if not self._is_configured() or messages is None or len(messages) == 0:
return
xmpp_cfg = self.config_helper.get_property('xmpp')
logging.debug('Sending Notification via XMPP...')
Log.debug('Sending Notification via XMPP... (Please wait)')
try:
xmpp = XmppShooter(xmpp_cfg['sender'], xmpp_cfg['password'],
xmpp_cfg['target'])
xmpp.send_messages(messages)
except BaseException as e:
error_formatted = traceback.format_exc()
logging.error('While sending notification:\n%s' %
(error_formatted),
extra={'exception': e})
raise e # to be properly notified via Sentry
def _select_should_download_lessons(self):
"""
Asks the user if lessons should be downloaded
"""
download_lessons = self.config_helper.get_download_lessons()
self.section_seperator()
Log.info(
'Lessons are a kind of self-teaching with pages of information and other pages with questions to answer.'
+
' A student can be graded on their answers after completing a lesson. Currently, only lessons without'
+
' the answers are downloaded. The answers are potentially also available for download,'
+ ' but this has not been implemented.')
print('')
download_lessons = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to download lessons of your courses?'),
default_is_yes=download_lessons)
self.config_helper.set_property('download_lessons', download_lessons)
def _select_should_download_links_in_descriptions(self):
"""
Asks the user if links in descriptions should be downloaded
"""
download_links_in_descriptions = self.config_helper.get_download_links_in_descriptions(
)
self.section_seperator()
Log.info(
'In the descriptions of files, sections, assignments or courses the teacher can add links to webpages,'
+
' files or videos. That links can pont to a internal page on moodle or to an external webpage.'
)
print('')
download_links_in_descriptions = cutie.prompt_yes_or_no(
Log.special_str(
'Would you like to download links in descriptions?'),
default_is_yes=download_links_in_descriptions,
)
self.config_helper.set_property('download_links_in_descriptions',
download_links_in_descriptions)
def _select_should_download_databases(self):
"""
Asks the user if databases should be downloaded
"""
download_databases = self.config_helper.get_download_databases()
self.section_seperator()
Log.info('In the database module of Moodle data can be stored' +
' structured with information. Often it is also' +
' possible for students to upload data there. Because' +
' the implementation of the downloader has not yet been' +
' optimized at this point, it is optional to download the' +
' databases. Currently only files are downloaded, thumbails' +
' are ignored.')
print('')
download_databases = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to download databases of your courses?'),
default_is_yes=download_databases)
self.config_helper.set_property('download_databases',
download_databases)
def _select_should_download_submissions(self):
"""
Asks the user if submissions should be downloaded
"""
download_submissions = self.config_helper.get_download_submissions()
self.section_seperator()
Log.info('Submissions are files that you or a teacher have uploaded' +
' to your assignments. Moodle does not provide an' +
' interface for downloading information from all' +
' submissions to a course at once.')
Log.warning(
'Therefore, it may be slow to monitor changes to submissions.')
print('')
download_submissions = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to download submissions of your assignments?'),
default_is_yes=download_submissions,
)
self.config_helper.set_property('download_submissions',
download_submissions)
def receive_token() -> str:
"""
Starts an HTTP server to receive the SSO token from browser.
It waits till a token was received.
"""
server_address = ('localhost', 80)
try:
httpd = HTTPServer(server_address, TransferServer)
except PermissionError:
Log.error('Permission denied: Please start the' +
' downloader once with administrator rights, so that it' +
' can wait on port 80 for the token.')
sys.exit(1)
extracted_token = None
while extracted_token is None:
httpd.handle_request()
extracted_token = extract_token(TransferServer.received_token)
httpd.server_close()
return extracted_token
def _send_messages(self, messages: [str]):
"""
Sends an message
"""
if not self._is_configured() or messages is None or len(messages) == 0:
return
telegram_cfg = self.config_helper.get_property('telegram')
logging.debug('Sending Notification via Telegram...')
Log.debug('Sending Notification via Telegram... (Please wait)')
telegram_shooter = TelegramShooter(telegram_cfg['token'],
telegram_cfg['chat_id'])
for message_content in messages:
try:
telegram_shooter.send(message_content)
except BaseException as e:
error_formatted = traceback.format_exc()
logging.error('While sending notification:\n%s' %
(error_formatted),
extra={'exception': e})
raise e # to be properly notified via Sentry
