def _select_should_userid_and_version_be_saved(self, userid, version): """ Asks the user if the userid and version should be saved in the configuration """ print('') Log.info('The user id and version number of Moodle are downloaded' + ' at the beginning of each run of the downloader.' + ' Since this data rarely changes, it can be saved in the' + ' configuration.') Log.critical( f'Your user id is `{userid}` and the moodle version is `{version}`' ) print('') save_userid_and_version = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to store the user id and version number of Moodle in the configuration?' ), default_is_yes=False, ) if save_userid_and_version: Log.warning('Remember to delete the version number from the' + ' configuration once Moodle has been updated' + ' and then run the configurator again!') self.config_helper.set_property('userid', userid) self.config_helper.set_property('version', version) self.section_seperator()
def run_main( storage_path, verbose=False, skip_cert_verify=False, ignore_ytdl_errors=False, without_downloading_files=False, log_responses=False, ): log_formatter = logging.Formatter( '%(asctime)s %(levelname)s {%(module)s} %(message)s', '%Y-%m-%d %H:%M:%S') log_file = os.path.join(storage_path, 'MoodleDownloader.log') log_handler = RotatingFileHandler(log_file, mode='a', maxBytes=1 * 1024 * 1024, backupCount=2, encoding='utf-8', delay=0) log_handler.setFormatter(log_formatter) if verbose: log_handler.setLevel(logging.DEBUG) else: log_handler.setLevel(logging.INFO) app_log = logging.getLogger() if verbose: app_log.setLevel(logging.DEBUG) else: app_log.setLevel(logging.INFO) app_log.addHandler(log_handler) logging.info('--- moodle-dl started ---------------------') Log.info('Moodle Downloader starting...') if verbose: logging.debug('moodle-dl version: %s', __version__) logging.debug('python version: %s', ".".join(map(str, sys.version_info[:3]))) ffmpeg_available = which('ffmpeg') is not None logging.debug('Is ffmpeg available: %s', ffmpeg_available) if IS_DEBUG: logging.info('Debug-Mode detected. Errors will be re-risen.') app_log.addHandler(ReRaiseOnError()) try: msg_load_config = 'Loading config...' logging.debug(msg_load_config) Log.debug(msg_load_config) config = ConfigHelper(storage_path) config.load() except BaseException as e: logging.error( 'Error while trying to load the Configuration! %s Exiting...', e, extra={'exception': e}) Log.error('Error while trying to load the Configuration!') sys.exit(1) r_client = False try: sentry_dsn = config.get_property('sentry_dsn') if sentry_dsn: sentry_sdk.init(sentry_dsn) except BaseException: pass mail_service = MailService(config) tg_service = TelegramService(config) xmpp_service = XmppService(config) console_service = ConsoleService(config) PathTools.restricted_filenames = config.get_restricted_filenames() try: if not IS_DEBUG: process_lock.lock(storage_path) moodle = MoodleService(config, storage_path, skip_cert_verify, log_responses) msg_checking_for_changes = 'Checking for changes for the configured Moodle-Account....' logging.debug(msg_checking_for_changes) Log.debug(msg_checking_for_changes) changed_courses = moodle.fetch_state() if log_responses: msg_responses_logged = ( "All JSON-responses from Moodle have been written to the responses.log file. Exiting..." ) logging.debug(msg_responses_logged) Log.success(msg_responses_logged) process_lock.unlock(storage_path) return msg_start_downloading = 'Start downloading changed files...' logging.debug(msg_start_downloading) Log.debug(msg_start_downloading) if without_downloading_files: downloader = FakeDownloadService(changed_courses, moodle, storage_path) else: downloader = DownloadService(changed_courses, moodle, storage_path, skip_cert_verify, ignore_ytdl_errors) downloader.run() failed_downloads = downloader.get_failed_url_targets() changed_courses_to_notify = moodle.recorder.changes_to_notify() if len(changed_courses_to_notify) > 0: console_service.notify_about_changes_in_moodle( changed_courses_to_notify) mail_service.notify_about_changes_in_moodle( changed_courses_to_notify) tg_service.notify_about_changes_in_moodle( changed_courses_to_notify) xmpp_service.notify_about_changes_in_moodle( changed_courses_to_notify) moodle.recorder.notified(changed_courses_to_notify) else: msg_no_changes = 'No changes found for the configured Moodle-Account.' logging.info(msg_no_changes) Log.warning(msg_no_changes) if len(failed_downloads) > 0: console_service.notify_about_failed_downloads(failed_downloads) mail_service.notify_about_failed_downloads(failed_downloads) tg_service.notify_about_failed_downloads(failed_downloads) xmpp_service.notify_about_failed_downloads(failed_downloads) process_lock.unlock(storage_path) logging.debug('All done. Exiting...') Log.success('All done. Exiting..') except BaseException as e: print('\n') if not isinstance(e, process_lock.LockError): process_lock.unlock(storage_path) error_formatted = traceback.format_exc() logging.error(error_formatted, extra={'exception': e}) if r_client: sentry_sdk.capture_exception(e) if verbose: Log.critical('Exception:\n%s' % (error_formatted)) short_error = str(e) if not short_error or short_error.isspace(): short_error = traceback.format_exc(limit=1) console_service.notify_about_error(short_error) mail_service.notify_about_error(short_error) tg_service.notify_about_error(short_error) xmpp_service.notify_about_error(short_error) logging.debug('Exception-Handling completed. Exiting...') sys.exit(1)
def interactively_manage_database(self): RESET_SEQ = '\033[0m' COLOR_SEQ = '\033[1;%dm' BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38) stored_files = self.state_recorder.get_stored_files() stored_files = MoodleService.filter_courses(stored_files, self.config_helper) if len(stored_files) <= 0: return course_options = [] courses = [] for course in stored_files: for course_file in course.files: if not os.path.exists(course_file.saved_to): course_options.append(COLOR_SEQ % BLUE + course.fullname + RESET_SEQ) courses.append(course) break print( 'This management tool will navigate you through a menu to' + ' selectively remove file entries from the database so' + ' that these files can be downloaded again.' ) Log.warning( 'Only files that are missing locally but stored in the local' + ' database are displayed in this tool. If a file is not missing' + ' from a course, it will not be listed here at all. Also, only' + ' courses that are selected for download are displayed.' ) Log.critical( 'For more complicated operations on the database a DB browser for SQLite' + ' is advantageous (https://sqlitebrowser.org/).' ) if not courses: print('No files are missing locally but stored in the local database. Nothing to do.') return print('Choose one of the courses:') print('[Confirm your selection with the Enter key]') print('') selected_course_id = cutie.select(options=course_options) selected_course = courses[selected_course_id] section_options = [] sections = [] # Add the option to select all sections section_options.append(COLOR_SEQ % MAGENTA + '[All sections]' + RESET_SEQ) sections.append(None) # Add None at index 0 to avoid index shifting for course_file in selected_course.files: if not os.path.exists(course_file.saved_to) and (course_file.section_name not in sections): section_options.append(COLOR_SEQ % MAGENTA + course_file.section_name + RESET_SEQ) sections.append(course_file.section_name) print('From which sections you want to select files?') print('[You can select with the space bar and confirm your selection with the enter key]') print('') selected_sections_ids = cutie.select_multiple(options=section_options, minimal_count=1) selected_sections = [] for selected_sections_id in selected_sections_ids: if selected_sections_id == 0: selected_sections = sections[1:] break elif (selected_sections_id) < len(sections): selected_sections.append(sections[selected_sections_id]) file_options = [] files = [] # Add the option to select all files file_options.append(COLOR_SEQ % CYAN + '[All files]' + RESET_SEQ) files.append(None) # Add None at index 0 to avoid index shifting for course_file in selected_course.files: if not os.path.exists(course_file.saved_to) and (course_file.section_name in selected_sections): file_options.append(COLOR_SEQ % CYAN + course_file.content_filename + RESET_SEQ) files.append(course_file) print('Which of the files should be removed form the database, so that they will be redownloaded?') print('[You can select with the space bar and confirm your selection with the enter key]') print('') selected_files = cutie.select_multiple(options=file_options) files_to_delete = [] for file_index in selected_files: if file_index == 0: # If all files is selected for file_to_delete in files[1:]: # Ignore the first element of the array set as None if isinstance(file_to_delete, File): files_to_delete.append(file_to_delete) break elif file_index < len(files) and isinstance(files[file_index], File): files_to_delete.append(files[file_index]) self.state_recorder.batch_delete_files_from_db(files_to_delete)
def interactively_add_all_visible_courses(self): """ Guides the user through the process of adding all visible courses to the list of courses to download in the configuration """ token = self.config_helper.get_token() moodle_domain = self.config_helper.get_moodle_domain() moodle_path = self.config_helper.get_moodle_path() use_http = self.config_helper.get_use_http() request_helper = RequestHelper(moodle_domain, moodle_path, token, self.skip_cert_verify, use_http=use_http) first_contact_handler = FirstContactHandler(request_helper) print('') Log.info( 'It is possible to automatically complete the moodle-dl configuration' + ' with all the courses you can see on your moodle. These are either' + ' courses to which you have the appropriate rights to see the' + ' course or the course is visible without enrollment.') Log.critical( 'This process can take several minutes for large Moodels, as is common at' + ' large universities. Timeout is set to 20 minutes.') print('') add_all_visible_courses = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to add all visible courses of your Moodle to the configuration?' ), default_is_yes=False, ) if not add_all_visible_courses: return else: Log.warning( 'Please wait for the result, this may take several minutes.' + ' In addition to adding the courses to the configuration,' + ' it will also create an `all_courses.json` file with all' + ' the courses available on your Moodle.') courses = [] all_visible_courses = [] try: userid, version = self.config_helper.get_userid_and_version() if userid is None or version is None: userid, version = first_contact_handler.fetch_userid_and_version( ) else: first_contact_handler.version = version courses = first_contact_handler.fetch_courses(userid) log_all_courses_to = str( Path(self.storage_path) / 'all_courses.json') all_visible_courses = first_contact_handler.fetch_all_visible_courses( log_all_courses_to) except (RequestRejectedError, ValueError, RuntimeError, ConnectionError) as error: Log.error( 'Error while communicating with the Moodle System! (%s)' % (error)) sys.exit(1) # Filter out courses the user is enroled in filtered_all_courses = [] for visible_course in all_visible_courses: add_to_final_list = True for course in courses: if visible_course.id == course.id: add_to_final_list = False break if add_to_final_list: filtered_all_courses.append(visible_course) # Update Public Courses IDs download_public_course_ids = self.config_helper.get_download_public_course_ids( ) # Update Course settings List for all new Courses options_of_courses = self.config_helper.get_options_of_courses() for course in filtered_all_courses: current_course_settings = options_of_courses.get( str(course.id), None) # create default settings if current_course_settings is None: current_course_settings = { 'original_name': course.fullname, 'overwrite_name_with': None, 'create_directory_structure': True, } options_of_courses.update( {str(course.id): current_course_settings}) if course.id not in download_public_course_ids: download_public_course_ids.append(course.id) self.config_helper.set_property('options_of_courses', options_of_courses) self.config_helper.set_property('download_public_course_ids', download_public_course_ids)