def _select_should_download_linked_files(self):
"""
Asks the user if linked files should be downloaded
"""
download_linked_files = self.config_helper.get_download_linked_files()
self.section_seperator()
Log.info('In Moodle courses the teacher can also link to external' +
' files. This can be audio, video, text or anything else.' +
' In particular, the teacher can link to Youtube videos.')
Log.debug('To download videos correctly you have to install ffmpeg. ')
Log.error('These files can increase the download volume considerably.')
Log.info('If you want to filter the external links by their domain,' +
' you can manually set a whitelist and a blacklist' +
' (https://github.com/C0D3D3V/Moodle-Downloader-2/' +
'wiki/Download-(external)-linked-files' +
' for more details).')
Log.warning(
'Please note that the size of the external files is determined during the download, so the total size'
+ ' changes during the download.')
print('')
download_linked_files = cutie.prompt_yes_or_no(
Log.special_str(
'Would you like to download linked files of the courses you have selected?'
),
default_is_yes=download_linked_files,
)
self.config_helper.set_property('download_linked_files',
download_linked_files)
def interactively_acquire_config(self):
"""
Guides the user through the process of configuring the downloader
for the courses to be downloaded and in what way
"""
token = self.config_helper.get_token()
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
use_http = self.config_helper.get_use_http()
request_helper = RequestHelper(moodle_domain,
moodle_path,
token,
self.skip_cert_verify,
use_http=use_http)
first_contact_handler = FirstContactHandler(request_helper)
courses = []
try:
userid, version = self.config_helper.get_userid_and_version()
if userid is None or version is None:
userid, version = first_contact_handler.fetch_userid_and_version(
)
self._select_should_userid_and_version_be_saved(
userid, version)
else:
first_contact_handler.version = version
courses = first_contact_handler.fetch_courses(userid)
except (RequestRejectedError, ValueError, RuntimeError,
ConnectionError) as error:
Log.error(
'Error while communicating with the Moodle System! (%s)' %
(error))
sys.exit(1)
course_ids = self._select_courses_to_download(courses)
sections = {}
for course_id in course_ids:
sections[course_id] = first_contact_handler.fetch_sections(
course_id)
self._set_options_of_courses(courses, sections)
self._select_should_download_submissions()
self._select_should_download_descriptions()
self._select_should_download_links_in_descriptions()
self._select_should_download_databases()
self._select_should_download_forums()
self._select_should_download_quizzes()
self._select_should_download_lessons()
self._select_should_download_workshops()
self._select_should_download_linked_files()
self._select_should_download_also_with_cookie()
def interactively_acquire_token(self, use_stored_url: bool = False) -> str:
"""
Walks the user through executing a login into the Moodle-System to get
the Token and saves it.
@return: The Token for Moodle.
"""
print(
'[The following Credentials are not saved, it is only used temporarily to generate a login token.]'
)
moodle_token = None
while moodle_token is None:
if not use_stored_url:
moodle_url = input('URL of Moodle: ')
if not moodle_url.startswith('https://'):
Log.error(
'The url of your moodle must start with `https://`')
continue
moodle_uri = urlparse(moodle_url)
moodle_domain, moodle_path = self._split_moodle_uri(moodle_uri)
else:
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
moodle_username = input('Username for Moodle: ')
moodle_password = getpass('Password for Moodle [no output]: ')
try:
moodle_token, moodle_privatetoken = login_helper.obtain_login_token(
moodle_username, moodle_password, moodle_domain,
moodle_path, self.skip_cert_verify)
except RequestRejectedError as error:
print('Login Failed! (%s) Please try again.' % (error))
except (ValueError, RuntimeError) as error:
print(
'Error while communicating with the Moodle System! (%s) Please try again.'
% (error))
# Saves the created token and the successful Moodle parameters.
self.config_helper.set_property('token', moodle_token)
if moodle_privatetoken is not None:
self.config_helper.set_property('privatetoken',
moodle_privatetoken)
self.config_helper.set_property('moodle_domain', moodle_domain)
self.config_helper.set_property('moodle_path', moodle_path)
return moodle_token
def _log_failures(self):
"""
Logs errors if any have occurred.
"""
print('')
if len(self.report['failure']) > 0:
Log.warning(
'Error while trying to download files, look at the log for more details.'
)
for url_target in self.report['failure']:
Log.error('%s\t%s' %
(url_target.file.content_filename, url_target.error))
def _select_should_download_also_with_cookie(self):
"""
Ask the user whether files for which a cookie is required should be downloaded.
"""
download_also_with_cookie = self.config_helper.get_download_also_with_cookie(
)
self.section_seperator()
Log.info(
'Descriptions may contain links to files that require a browser cookie so they can be downloaded.'
+
' There are also several Moodle plugins that cannot be displayed in the Moodle app,'
+ ' so you need a browser cookie to download these plugin files.')
Log.debug(
'The Moodle browser cookie is created using your private token and stored in the `Configs.txt` file.'
+
' As long as this option is activated, the file always contains a valid cookie.'
)
if self.config_helper.get_privatetoken() is None:
Log.error(
'Currently no private token is stored in the configuration.' +
' Create a private token with moodle-dl --new-token (if necessary with --sso)'
)
print('')
download_also_with_cookie = cutie.prompt_yes_or_no(
Log.special_str(
'Would you like to download files for which a cookie is required?'
),
default_is_yes=download_also_with_cookie,
)
self.config_helper.set_property('download_also_with_cookie',
download_also_with_cookie)
def receive_token() -> str:
"""
Starts an HTTP server to receive the SSO token from browser.
It waits till a token was received.
"""
server_address = ('localhost', 80)
try:
httpd = HTTPServer(server_address, TransferServer)
except PermissionError:
Log.error('Permission denied: Please start the' +
' downloader once with administrator rights, so that it' +
' can wait on port 80 for the token.')
sys.exit(1)
extracted_token = None
while extracted_token is None:
httpd.handle_request()
extracted_token = extract_token(TransferServer.received_token)
httpd.server_close()
return extracted_token
def check_and_fetch_cookies(self, privatetoken: str, userid: str) -> bool:
if os.path.exists(self.cookies_path):
# test if still logged in.
if self.test_cookies(self.moodle_test_url):
return True
warning_msg = 'Moodle cookie has expired, an attempt is made to generate a new cookie.'
logging.warning(warning_msg)
Log.warning('\r' + warning_msg + '\033[K')
if privatetoken is None:
error_msg = (
'Moodle Cookies are not retrieved because no private token is set.'
+
' To set a private token, use the `--new-token` option (if necessary also with `--sso`).'
)
logging.warning(error_msg)
Log.error('\r' + error_msg + '\033[K')
self.delete_cookie_file()
return False
autologin_key = self.fetch_autologin_key(privatetoken)
if autologin_key is None:
error_msg = 'Failed to download autologin key!'
logging.debug(error_msg)
print('')
Log.error(error_msg)
self.delete_cookie_file()
return False
print('\rDownloading cookies\033[K', end='')
post_data = {'key': autologin_key.get('key', ''), 'userid': userid}
url = autologin_key.get('autologinurl', '')
cookies_response, cookies_session = self.request_helper.post_URL(
url, post_data, self.cookies_path)
moodle_test_url = cookies_response.url
if self.test_cookies(moodle_test_url):
return True
else:
error_msg = 'Failed to generate cookies!'
logging.debug(error_msg)
print('')
Log.error(error_msg)
self.delete_cookie_file()
return False
def interactively_acquire_token(
self, use_stored_url: bool = False, username: str = None, password: str = None
) -> str:
"""
Walks the user through executing a login into the Moodle-System to get
the Token and saves it.
@return: The Token for Moodle.
"""
automated = False
stop_automatic_generation = False
if username is not None and password is not None:
automated = True
if not automated:
print('[The following Credentials are not saved, it is only used temporarily to generate a login token.]')
moodle_token = None
while moodle_token is None:
if stop_automatic_generation and automated:
break
if not use_stored_url:
moodle_url = input('URL of Moodle: ')
use_http = False
if moodle_url.startswith('http://'):
Log.error(
'Warning: You have entered an insecure URL! Are you sure that the Moodle is'
+ ' not accessible via `https://`? All your data will be transferred'
+ ' insecurely! If your Moodle is accessible via `https://`, then run'
+ ' the process again using `https://` to protect your data.'
)
use_http = True
elif not moodle_url.startswith('https://'):
Log.error('The url of your moodle must start with `https://`')
continue
moodle_uri = urlparse(moodle_url)
moodle_domain, moodle_path = self._split_moodle_uri(moodle_uri)
else:
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
use_http = self.config_helper.get_use_http()
if username is not None:
moodle_username = username
stop_automatic_generation = True
else:
moodle_username = input('Username for Moodle: ')
if password is not None:
moodle_password = password
else:
moodle_password = getpass('Password for Moodle [no output]: ')
try:
moodle_token, moodle_privatetoken = login_helper.obtain_login_token(
moodle_username,
moodle_password,
moodle_domain,
moodle_path,
self.skip_cert_verify,
use_http,
)
except RequestRejectedError as error:
Log.error('Login Failed! (%s) Please try again.' % (error))
except (ValueError, RuntimeError) as error:
Log.error('Error while communicating with the Moodle System! (%s) Please try again.' % (error))
except ConnectionError as error:
Log.error(str(error))
if automated is True and moodle_token is None:
sys.exit(1)
# Saves the created token and the successful Moodle parameters.
self.config_helper.set_property('token', moodle_token)
if moodle_privatetoken is not None:
self.config_helper.set_property('privatetoken', moodle_privatetoken)
self.config_helper.set_property('moodle_domain', moodle_domain)
self.config_helper.set_property('moodle_path', moodle_path)
if use_http is True:
self.config_helper.set_property('use_http', use_http)
return moodle_token
def run_main(
storage_path,
verbose=False,
skip_cert_verify=False,
ignore_ytdl_errors=False,
without_downloading_files=False,
log_responses=False,
):
log_formatter = logging.Formatter(
'%(asctime)s %(levelname)s {%(module)s} %(message)s',
'%Y-%m-%d %H:%M:%S')
log_file = os.path.join(storage_path, 'MoodleDownloader.log')
log_handler = RotatingFileHandler(log_file,
mode='a',
maxBytes=1 * 1024 * 1024,
backupCount=2,
encoding='utf-8',
delay=0)
log_handler.setFormatter(log_formatter)
if verbose:
log_handler.setLevel(logging.DEBUG)
else:
log_handler.setLevel(logging.INFO)
app_log = logging.getLogger()
if verbose:
app_log.setLevel(logging.DEBUG)
else:
app_log.setLevel(logging.INFO)
app_log.addHandler(log_handler)
logging.info('--- moodle-dl started ---------------------')
Log.info('Moodle Downloader starting...')
if verbose:
logging.debug('moodle-dl version: %s', __version__)
logging.debug('python version: %s',
".".join(map(str, sys.version_info[:3])))
ffmpeg_available = which('ffmpeg') is not None
logging.debug('Is ffmpeg available: %s', ffmpeg_available)
if IS_DEBUG:
logging.info('Debug-Mode detected. Errors will be re-risen.')
app_log.addHandler(ReRaiseOnError())
try:
msg_load_config = 'Loading config...'
logging.debug(msg_load_config)
Log.debug(msg_load_config)
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error(
'Error while trying to load the Configuration! %s Exiting...',
e,
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
tg_service = TelegramService(config)
xmpp_service = XmppService(config)
console_service = ConsoleService(config)
PathTools.restricted_filenames = config.get_restricted_filenames()
try:
if not IS_DEBUG:
process_lock.lock(storage_path)
moodle = MoodleService(config, storage_path, skip_cert_verify,
log_responses)
msg_checking_for_changes = 'Checking for changes for the configured Moodle-Account....'
logging.debug(msg_checking_for_changes)
Log.debug(msg_checking_for_changes)
changed_courses = moodle.fetch_state()
if log_responses:
msg_responses_logged = (
"All JSON-responses from Moodle have been written to the responses.log file. Exiting..."
)
logging.debug(msg_responses_logged)
Log.success(msg_responses_logged)
process_lock.unlock(storage_path)
return
msg_start_downloading = 'Start downloading changed files...'
logging.debug(msg_start_downloading)
Log.debug(msg_start_downloading)
if without_downloading_files:
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path,
skip_cert_verify, ignore_ytdl_errors)
downloader.run()
failed_downloads = downloader.get_failed_url_targets()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
if len(changed_courses_to_notify) > 0:
console_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
tg_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
xmpp_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
else:
msg_no_changes = 'No changes found for the configured Moodle-Account.'
logging.info(msg_no_changes)
Log.warning(msg_no_changes)
if len(failed_downloads) > 0:
console_service.notify_about_failed_downloads(failed_downloads)
mail_service.notify_about_failed_downloads(failed_downloads)
tg_service.notify_about_failed_downloads(failed_downloads)
xmpp_service.notify_about_failed_downloads(failed_downloads)
process_lock.unlock(storage_path)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
print('\n')
if not isinstance(e, process_lock.LockError):
process_lock.unlock(storage_path)
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
if verbose:
Log.critical('Exception:\n%s' % (error_formatted))
short_error = str(e)
if not short_error or short_error.isspace():
short_error = traceback.format_exc(limit=1)
console_service.notify_about_error(short_error)
mail_service.notify_about_error(short_error)
tg_service.notify_about_error(short_error)
xmpp_service.notify_about_error(short_error)
logging.debug('Exception-Handling completed. Exiting...')
sys.exit(1)
def notify_about_error(self, error_description: str):
Log.error('The following error occurred during execution:\n%s' %
(error_description))
def interactively_add_all_visible_courses(self):
"""
Guides the user through the process of adding all visible courses
to the list of courses to download in the configuration
"""
token = self.config_helper.get_token()
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
use_http = self.config_helper.get_use_http()
request_helper = RequestHelper(moodle_domain,
moodle_path,
token,
self.skip_cert_verify,
use_http=use_http)
first_contact_handler = FirstContactHandler(request_helper)
print('')
Log.info(
'It is possible to automatically complete the moodle-dl configuration'
+
' with all the courses you can see on your moodle. These are either'
+ ' courses to which you have the appropriate rights to see the' +
' course or the course is visible without enrollment.')
Log.critical(
'This process can take several minutes for large Moodels, as is common at'
+ ' large universities. Timeout is set to 20 minutes.')
print('')
add_all_visible_courses = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to add all visible courses of your Moodle to the configuration?'
),
default_is_yes=False,
)
if not add_all_visible_courses:
return
else:
Log.warning(
'Please wait for the result, this may take several minutes.' +
' In addition to adding the courses to the configuration,' +
' it will also create an `all_courses.json` file with all' +
' the courses available on your Moodle.')
courses = []
all_visible_courses = []
try:
userid, version = self.config_helper.get_userid_and_version()
if userid is None or version is None:
userid, version = first_contact_handler.fetch_userid_and_version(
)
else:
first_contact_handler.version = version
courses = first_contact_handler.fetch_courses(userid)
log_all_courses_to = str(
Path(self.storage_path) / 'all_courses.json')
all_visible_courses = first_contact_handler.fetch_all_visible_courses(
log_all_courses_to)
except (RequestRejectedError, ValueError, RuntimeError,
ConnectionError) as error:
Log.error(
'Error while communicating with the Moodle System! (%s)' %
(error))
sys.exit(1)
# Filter out courses the user is enroled in
filtered_all_courses = []
for visible_course in all_visible_courses:
add_to_final_list = True
for course in courses:
if visible_course.id == course.id:
add_to_final_list = False
break
if add_to_final_list:
filtered_all_courses.append(visible_course)
# Update Public Courses IDs
download_public_course_ids = self.config_helper.get_download_public_course_ids(
)
# Update Course settings List for all new Courses
options_of_courses = self.config_helper.get_options_of_courses()
for course in filtered_all_courses:
current_course_settings = options_of_courses.get(
str(course.id), None)
# create default settings
if current_course_settings is None:
current_course_settings = {
'original_name': course.fullname,
'overwrite_name_with': None,
'create_directory_structure': True,
}
options_of_courses.update(
{str(course.id): current_course_settings})
if course.id not in download_public_course_ids:
download_public_course_ids.append(course.id)
self.config_helper.set_property('options_of_courses',
options_of_courses)
self.config_helper.set_property('download_public_course_ids',
download_public_course_ids)
