def _select_should_userid_and_version_be_saved(self, userid, version):
"""
Asks the user if the userid and version should be saved in the configuration
"""
print('')
Log.info('The user id and version number of Moodle are downloaded' +
' at the beginning of each run of the downloader.' +
' Since this data rarely changes, it can be saved in the' +
' configuration.')
Log.critical(
f'Your user id is `{userid}` and the moodle version is `{version}`'
)
print('')
save_userid_and_version = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to store the user id and version number of Moodle in the configuration?'
),
default_is_yes=False,
)
if save_userid_and_version:
Log.warning('Remember to delete the version number from the' +
' configuration once Moodle has been updated' +
' and then run the configurator again!')
self.config_helper.set_property('userid', userid)
self.config_helper.set_property('version', version)
self.section_seperator()
def run_main(
storage_path,
verbose=False,
skip_cert_verify=False,
ignore_ytdl_errors=False,
without_downloading_files=False,
log_responses=False,
):
log_formatter = logging.Formatter(
'%(asctime)s %(levelname)s {%(module)s} %(message)s',
'%Y-%m-%d %H:%M:%S')
log_file = os.path.join(storage_path, 'MoodleDownloader.log')
log_handler = RotatingFileHandler(log_file,
mode='a',
maxBytes=1 * 1024 * 1024,
backupCount=2,
encoding='utf-8',
delay=0)
log_handler.setFormatter(log_formatter)
if verbose:
log_handler.setLevel(logging.DEBUG)
else:
log_handler.setLevel(logging.INFO)
app_log = logging.getLogger()
if verbose:
app_log.setLevel(logging.DEBUG)
else:
app_log.setLevel(logging.INFO)
app_log.addHandler(log_handler)
logging.info('--- moodle-dl started ---------------------')
Log.info('Moodle Downloader starting...')
if verbose:
logging.debug('moodle-dl version: %s', __version__)
logging.debug('python version: %s',
".".join(map(str, sys.version_info[:3])))
ffmpeg_available = which('ffmpeg') is not None
logging.debug('Is ffmpeg available: %s', ffmpeg_available)
if IS_DEBUG:
logging.info('Debug-Mode detected. Errors will be re-risen.')
app_log.addHandler(ReRaiseOnError())
try:
msg_load_config = 'Loading config...'
logging.debug(msg_load_config)
Log.debug(msg_load_config)
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error(
'Error while trying to load the Configuration! %s Exiting...',
e,
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
tg_service = TelegramService(config)
xmpp_service = XmppService(config)
console_service = ConsoleService(config)
PathTools.restricted_filenames = config.get_restricted_filenames()
try:
if not IS_DEBUG:
process_lock.lock(storage_path)
moodle = MoodleService(config, storage_path, skip_cert_verify,
log_responses)
msg_checking_for_changes = 'Checking for changes for the configured Moodle-Account....'
logging.debug(msg_checking_for_changes)
Log.debug(msg_checking_for_changes)
changed_courses = moodle.fetch_state()
if log_responses:
msg_responses_logged = (
"All JSON-responses from Moodle have been written to the responses.log file. Exiting..."
)
logging.debug(msg_responses_logged)
Log.success(msg_responses_logged)
process_lock.unlock(storage_path)
return
msg_start_downloading = 'Start downloading changed files...'
logging.debug(msg_start_downloading)
Log.debug(msg_start_downloading)
if without_downloading_files:
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path,
skip_cert_verify, ignore_ytdl_errors)
downloader.run()
failed_downloads = downloader.get_failed_url_targets()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
if len(changed_courses_to_notify) > 0:
console_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
tg_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
xmpp_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
else:
msg_no_changes = 'No changes found for the configured Moodle-Account.'
logging.info(msg_no_changes)
Log.warning(msg_no_changes)
if len(failed_downloads) > 0:
console_service.notify_about_failed_downloads(failed_downloads)
mail_service.notify_about_failed_downloads(failed_downloads)
tg_service.notify_about_failed_downloads(failed_downloads)
xmpp_service.notify_about_failed_downloads(failed_downloads)
process_lock.unlock(storage_path)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
print('\n')
if not isinstance(e, process_lock.LockError):
process_lock.unlock(storage_path)
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
if verbose:
Log.critical('Exception:\n%s' % (error_formatted))
short_error = str(e)
if not short_error or short_error.isspace():
short_error = traceback.format_exc(limit=1)
console_service.notify_about_error(short_error)
mail_service.notify_about_error(short_error)
tg_service.notify_about_error(short_error)
xmpp_service.notify_about_error(short_error)
logging.debug('Exception-Handling completed. Exiting...')
sys.exit(1)
def interactively_manage_database(self):
RESET_SEQ = '\033[0m'
COLOR_SEQ = '\033[1;%dm'
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
stored_files = self.state_recorder.get_stored_files()
stored_files = MoodleService.filter_courses(stored_files, self.config_helper)
if len(stored_files) <= 0:
return
course_options = []
courses = []
for course in stored_files:
for course_file in course.files:
if not os.path.exists(course_file.saved_to):
course_options.append(COLOR_SEQ % BLUE + course.fullname + RESET_SEQ)
courses.append(course)
break
print(
'This management tool will navigate you through a menu to'
+ ' selectively remove file entries from the database so'
+ ' that these files can be downloaded again.'
)
Log.warning(
'Only files that are missing locally but stored in the local'
+ ' database are displayed in this tool. If a file is not missing'
+ ' from a course, it will not be listed here at all. Also, only'
+ ' courses that are selected for download are displayed.'
)
Log.critical(
'For more complicated operations on the database a DB browser for SQLite'
+ ' is advantageous (https://sqlitebrowser.org/).'
)
if not courses:
print('No files are missing locally but stored in the local database. Nothing to do.')
return
print('Choose one of the courses:')
print('[Confirm your selection with the Enter key]')
print('')
selected_course_id = cutie.select(options=course_options)
selected_course = courses[selected_course_id]
section_options = []
sections = []
# Add the option to select all sections
section_options.append(COLOR_SEQ % MAGENTA + '[All sections]' + RESET_SEQ)
sections.append(None) # Add None at index 0 to avoid index shifting
for course_file in selected_course.files:
if not os.path.exists(course_file.saved_to) and (course_file.section_name not in sections):
section_options.append(COLOR_SEQ % MAGENTA + course_file.section_name + RESET_SEQ)
sections.append(course_file.section_name)
print('From which sections you want to select files?')
print('[You can select with the space bar and confirm your selection with the enter key]')
print('')
selected_sections_ids = cutie.select_multiple(options=section_options, minimal_count=1)
selected_sections = []
for selected_sections_id in selected_sections_ids:
if selected_sections_id == 0:
selected_sections = sections[1:]
break
elif (selected_sections_id) < len(sections):
selected_sections.append(sections[selected_sections_id])
file_options = []
files = []
# Add the option to select all files
file_options.append(COLOR_SEQ % CYAN + '[All files]' + RESET_SEQ)
files.append(None) # Add None at index 0 to avoid index shifting
for course_file in selected_course.files:
if not os.path.exists(course_file.saved_to) and (course_file.section_name in selected_sections):
file_options.append(COLOR_SEQ % CYAN + course_file.content_filename + RESET_SEQ)
files.append(course_file)
print('Which of the files should be removed form the database, so that they will be redownloaded?')
print('[You can select with the space bar and confirm your selection with the enter key]')
print('')
selected_files = cutie.select_multiple(options=file_options)
files_to_delete = []
for file_index in selected_files:
if file_index == 0: # If all files is selected
for file_to_delete in files[1:]: # Ignore the first element of the array set as None
if isinstance(file_to_delete, File):
files_to_delete.append(file_to_delete)
break
elif file_index < len(files) and isinstance(files[file_index], File):
files_to_delete.append(files[file_index])
self.state_recorder.batch_delete_files_from_db(files_to_delete)
def interactively_add_all_visible_courses(self):
"""
Guides the user through the process of adding all visible courses
to the list of courses to download in the configuration
"""
token = self.config_helper.get_token()
moodle_domain = self.config_helper.get_moodle_domain()
moodle_path = self.config_helper.get_moodle_path()
use_http = self.config_helper.get_use_http()
request_helper = RequestHelper(moodle_domain,
moodle_path,
token,
self.skip_cert_verify,
use_http=use_http)
first_contact_handler = FirstContactHandler(request_helper)
print('')
Log.info(
'It is possible to automatically complete the moodle-dl configuration'
+
' with all the courses you can see on your moodle. These are either'
+ ' courses to which you have the appropriate rights to see the' +
' course or the course is visible without enrollment.')
Log.critical(
'This process can take several minutes for large Moodels, as is common at'
+ ' large universities. Timeout is set to 20 minutes.')
print('')
add_all_visible_courses = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to add all visible courses of your Moodle to the configuration?'
),
default_is_yes=False,
)
if not add_all_visible_courses:
return
else:
Log.warning(
'Please wait for the result, this may take several minutes.' +
' In addition to adding the courses to the configuration,' +
' it will also create an `all_courses.json` file with all' +
' the courses available on your Moodle.')
courses = []
all_visible_courses = []
try:
userid, version = self.config_helper.get_userid_and_version()
if userid is None or version is None:
userid, version = first_contact_handler.fetch_userid_and_version(
)
else:
first_contact_handler.version = version
courses = first_contact_handler.fetch_courses(userid)
log_all_courses_to = str(
Path(self.storage_path) / 'all_courses.json')
all_visible_courses = first_contact_handler.fetch_all_visible_courses(
log_all_courses_to)
except (RequestRejectedError, ValueError, RuntimeError,
ConnectionError) as error:
Log.error(
'Error while communicating with the Moodle System! (%s)' %
(error))
sys.exit(1)
# Filter out courses the user is enroled in
filtered_all_courses = []
for visible_course in all_visible_courses:
add_to_final_list = True
for course in courses:
if visible_course.id == course.id:
add_to_final_list = False
break
if add_to_final_list:
filtered_all_courses.append(visible_course)
# Update Public Courses IDs
download_public_course_ids = self.config_helper.get_download_public_course_ids(
)
# Update Course settings List for all new Courses
options_of_courses = self.config_helper.get_options_of_courses()
for course in filtered_all_courses:
current_course_settings = options_of_courses.get(
str(course.id), None)
# create default settings
if current_course_settings is None:
current_course_settings = {
'original_name': course.fullname,
'overwrite_name_with': None,
'create_directory_structure': True,
}
options_of_courses.update(
{str(course.id): current_course_settings})
if course.id not in download_public_course_ids:
download_public_course_ids.append(course.id)
self.config_helper.set_property('options_of_courses',
options_of_courses)
self.config_helper.set_property('download_public_course_ids',
download_public_course_ids)
