def run_configure(storage_path, skip_cert_verify=False): config = ConfigHelper(storage_path) config.load() # because we do not want to override the other settings ConfigService(config, storage_path, skip_cert_verify).interactively_acquire_config() Log.success('Configuration successfully updated!')
def _select_sections_to_download(self, sections: [{}], excluded: [int]) -> [int]: """ Asks the user for the sections that should be downloaded. @param sections: All available sections @param excluded sections currently excluded """ choices = [] defaults = [] for i, section in enumerate(sections): section_id = section.get("id") choices.append(('%5i\t%s' % (section_id, section.get("name")))) if ResultsHandler.should_download_section(section_id, excluded): defaults.append(i) Log.special('Which of the sections should be downloaded?') Log.info( '[You can select with the space bar and confirm your selection with the enter key]' ) print('') selected_sections = cutie.select_multiple(options=choices, ticked_indices=defaults) dont_download_section_ids = [] for i, section in enumerate(sections): if i not in selected_sections: dont_download_section_ids.append(section.get("id")) return dont_download_section_ids
def notify_about_failed_downloads(self, failed_downloads: [URLTarget]): if len(failed_downloads) > 0: print('') Log.warning( 'Error while trying to download files, look at the log for more details. List of failed downloads:' ) print('') RESET_SEQ = '\033[0m' COLOR_SEQ = '\033[1;%dm' BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38) for url_target in failed_downloads: print( (COLOR_SEQ % CYAN) + url_target.file.content_filename + RESET_SEQ + (COLOR_SEQ % RED) + '\n\t' + str(url_target.error) + RESET_SEQ ) print('')
def _select_should_download_descriptions(self): """ Asks the user if descriptions should be downloaded """ download_descriptions = self.config_helper.get_download_descriptions() self.section_seperator() Log.info( 'In Moodle courses, descriptions can be added to all kinds' + ' of resources, such as files, tasks, assignments or simply' + ' free text. These descriptions are usually unnecessary to' + ' download because you have already read the information or' + ' know it from context. However, there are situations where' + ' it might be interesting to download these descriptions. The' + ' descriptions are created as Markdown files and can be' + ' deleted as desired.') Log.debug( 'Creating the description files does not take extra time, but they can be annoying' + ' if they only contain unnecessary information.') print('') download_descriptions = cutie.prompt_yes_or_no( Log.special_str( 'Would you like to download descriptions of the courses you have selected?' ), default_is_yes=download_descriptions, ) self.config_helper.set_property('download_descriptions', download_descriptions)
def run_change_notification_telegram(storage_path): config = ConfigHelper(storage_path) config.load() TelegramService(config).interactively_configure() Log.success('Telegram Configuration successfully updated!')
def run_change_notification_xmpp(storage_path): config = ConfigHelper(storage_path) config.load() XmppService(config).interactively_configure() Log.success('XMPP Configuration successfully updated!')
def _send_mail(self, subject, mail_content: (str, {str: str})): """ Sends an email """ if not self._is_configured(): return mail_cfg = self.config_helper.get_property('mail') try: logging.debug('Sending Notification via Mail...') Log.debug('Sending Notification via Mail... (Please wait)') mail_shooter = MailShooter( mail_cfg['sender'], mail_cfg['server_host'], int(mail_cfg['server_port']), mail_cfg['username'], mail_cfg['password'], ) mail_shooter.send(mail_cfg['target'], subject, mail_content[0], mail_content[1]) except BaseException as e: error_formatted = traceback.format_exc() logging.error('While sending notification:\n%s', error_formatted, extra={'exception': e}) raise e # to be properly notified via Sentry
def run_add_all_visible_courses(storage_path, skip_cert_verify): config = ConfigHelper(storage_path) config.load() # because we do not want to override the other settings ConfigService(config, storage_path, skip_cert_verify).interactively_add_all_visible_courses() Log.success('Configuration successfully updated!')
def run_delete_old_files(storage_path): config = ConfigHelper(storage_path) config.load() # Not really needed, we check all local courses offline_service = OfflineService(config, storage_path) offline_service.delete_old_files() Log.success('All done.')
def run_manage_database(storage_path): config = ConfigHelper(storage_path) config.load() # because we want to only manage configured courses offline_service = OfflineService(config, storage_path) offline_service.interactively_manage_database() Log.success('All done.')
def interactively_acquire_config(self): """ Guides the user through the process of configuring the downloader for the courses to be downloaded and in what way """ token = self.config_helper.get_token() moodle_domain = self.config_helper.get_moodle_domain() moodle_path = self.config_helper.get_moodle_path() use_http = self.config_helper.get_use_http() request_helper = RequestHelper(moodle_domain, moodle_path, token, self.skip_cert_verify, use_http=use_http) first_contact_handler = FirstContactHandler(request_helper) courses = [] try: userid, version = self.config_helper.get_userid_and_version() if userid is None or version is None: userid, version = first_contact_handler.fetch_userid_and_version( ) self._select_should_userid_and_version_be_saved( userid, version) else: first_contact_handler.version = version courses = first_contact_handler.fetch_courses(userid) except (RequestRejectedError, ValueError, RuntimeError, ConnectionError) as error: Log.error( 'Error while communicating with the Moodle System! (%s)' % (error)) sys.exit(1) course_ids = self._select_courses_to_download(courses) sections = {} for course_id in course_ids: sections[course_id] = first_contact_handler.fetch_sections( course_id) self._set_options_of_courses(courses, sections) self._select_should_download_submissions() self._select_should_download_descriptions() self._select_should_download_links_in_descriptions() self._select_should_download_databases() self._select_should_download_forums() self._select_should_download_quizzes() self._select_should_download_lessons() self._select_should_download_workshops() self._select_should_download_linked_files() self._select_should_download_also_with_cookie()
def _max_path_length_workaround(path): # Working around MAX_PATH limitation on Windows (see # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx) if os.name == 'nt': absfilepath = os.path.abspath(path) path = '\\\\?\\' + absfilepath Log.debug("Using absolute paths") else: Log.info("You are not on Windows, you don't need to use this workaround") return path
def _change_settings_of(self, course: Course, options_of_courses: {}): """ Ask for a new Name for the course. Then asks if a file structure should be created. """ current_course_settings = options_of_courses.get(str(course.id), None) # create default settings if current_course_settings is None: current_course_settings = { 'original_name': course.fullname, 'overwrite_name_with': None, 'create_directory_structure': True, } changed = False # Ask for new name overwrite_name_with = input( Log.special_str( 'Enter a new name for this Course [leave blank for "%s"]: ' % (course.fullname, ))) if overwrite_name_with == '': overwrite_name_with = None if (overwrite_name_with != course.fullname and current_course_settings.get('overwrite_name_with', None) != overwrite_name_with): current_course_settings.update( {'overwrite_name_with': overwrite_name_with}) changed = True # Ask if a file structure should be created create_directory_structure = current_course_settings.get( 'create_directory_structure', True) create_directory_structure = cutie.prompt_yes_or_no( Log.special_str( 'Should a directory structure be created for this course?'), default_is_yes=create_directory_structure, ) if create_directory_structure is not current_course_settings.get( 'create_directory_structure', True): changed = True current_course_settings.update( {'create_directory_structure': create_directory_structure}) if changed: options_of_courses.update( {str(course.id): current_course_settings}) self.config_helper.set_property('options_of_courses', options_of_courses)
def run_new_token(storage_path, use_sso=False, username: str = None, password: str = None, skip_cert_verify=False): config = ConfigHelper(storage_path) config.load() # because we do not want to override the other settings moodle = MoodleService(config, storage_path, skip_cert_verify) if use_sso: moodle.interactively_acquire_sso_token(use_stored_url=True) else: moodle.interactively_acquire_token(use_stored_url=True, username=username, password=password) Log.success('New Token successfully saved!')
def interactively_acquire_token(self, use_stored_url: bool = False) -> str: """ Walks the user through executing a login into the Moodle-System to get the Token and saves it. @return: The Token for Moodle. """ print( '[The following Credentials are not saved, it is only used temporarily to generate a login token.]' ) moodle_token = None while moodle_token is None: if not use_stored_url: moodle_url = input('URL of Moodle: ') if not moodle_url.startswith('https://'): Log.error( 'The url of your moodle must start with `https://`') continue moodle_uri = urlparse(moodle_url) moodle_domain, moodle_path = self._split_moodle_uri(moodle_uri) else: moodle_domain = self.config_helper.get_moodle_domain() moodle_path = self.config_helper.get_moodle_path() moodle_username = input('Username for Moodle: ') moodle_password = getpass('Password for Moodle [no output]: ') try: moodle_token, moodle_privatetoken = login_helper.obtain_login_token( moodle_username, moodle_password, moodle_domain, moodle_path, self.skip_cert_verify) except RequestRejectedError as error: print('Login Failed! (%s) Please try again.' % (error)) except (ValueError, RuntimeError) as error: print( 'Error while communicating with the Moodle System! (%s) Please try again.' % (error)) # Saves the created token and the successful Moodle parameters. self.config_helper.set_property('token', moodle_token) if moodle_privatetoken is not None: self.config_helper.set_property('privatetoken', moodle_privatetoken) self.config_helper.set_property('moodle_domain', moodle_domain) self.config_helper.set_property('moodle_path', moodle_path) return moodle_token
def _select_should_load_default_filename_character_map(self): """ Asks the user if the default filename character map should be loaded """ filename_character_map = self.config_helper.get_filename_character_map( ) if os.name != 'nt': self.section_seperator() Log.info( 'On Windows many characters are forbidden in filenames and paths, if you want, these characters can be' + ' automatically removed from filenames.') Log.warning( 'If you want to view the downloaded files on Windows this is important!' ) print('Current filename character map: {}'.format( filename_character_map)) Log.special( 'Do you want to load the default filename character map for Windows?' ) choices = [ 'No, leave it as it was.', 'No, load the default linux filename character map.', 'Yes, load the default windows filename character map.', ] print('[Confirm your selection with the Enter key]') print('') selected_map = cutie.select(options=choices) if selected_map == 0: return elif selected_map == 1: self.config_helper.set_default_filename_character_map(False) elif selected_map == 2: self.config_helper.set_default_filename_character_map(True) else: if filename_character_map != ConfigHelper.windows_map: self.section_seperator() Log.warning( 'Warning: Your current filename character map does not match the standard Windows' + ' filename character map!') print('Current filename character map: {}'.format( filename_character_map)) load_default_map = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to load the default filename character map for Windows?' ), default_is_yes=False, ) if load_default_map: self.config_helper.set_default_filename_character_map(True)
def notify_about_changes_in_moodle(self, changes: [Course]) -> None: """ Creates a terminal output about the downloaded changes. @param changes: A list of changed courses with changed files. """ RESET_SEQ = '\033[0m' COLOR_SEQ = '\033[1;%dm' BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38) print('\n') diff_count = 0 for course in changes: diff_count += len(course.files) if diff_count > 0: msg_changes = '%s changes found for the configured Moodle-Account.' logging.info(msg_changes, diff_count) Log.success(msg_changes % (diff_count)) for course in changes: if len(course.files) == 0: continue print(COLOR_SEQ % BLUE + course.fullname + RESET_SEQ) for file in course.files: if file.modified: print(COLOR_SEQ % YELLOW + '≠\t' + file.saved_to + RESET_SEQ) elif file.moved: if file.new_file is not None: print( COLOR_SEQ % CYAN + '<->\t' + ( file.saved_to + RESET_SEQ + COLOR_SEQ % GREEN + ' ==> ' + file.new_file.saved_to + RESET_SEQ ) ) else: print(COLOR_SEQ % CYAN + '<->\t' + file.saved_to + RESET_SEQ) elif file.deleted: print(COLOR_SEQ % MAGENTA + '-\t' + file.saved_to + RESET_SEQ) else: print(COLOR_SEQ % GREEN + '+\t' + file.saved_to + RESET_SEQ) print('\n')
def _log_failures(self): """ Logs errors if any have occurred. """ print('') if len(self.report['failure']) > 0: Log.warning( 'Error while trying to download files, look at the log for more details.' ) for url_target in self.report['failure']: Log.error('%s\t%s' % (url_target.file.content_filename, url_target.error))
def _select_should_download_forums(self): """ Asks the user if forums should be downloaded """ download_forums = self.config_helper.get_download_forums() self.section_seperator() Log.info( 'In forums, students and teachers can discuss and exchange information together.' ) print('') download_forums = cutie.prompt_yes_or_no( Log.special_str('Do you want to download forums of your courses?'), default_is_yes=download_forums) self.config_helper.set_property('download_forums', download_forums)
def _select_should_userid_and_version_be_saved(self, userid, version): """ Asks the user if the userid and version should be saved in the configuration """ print('') Log.info('The user id and version number of Moodle are downloaded' + ' at the beginning of each run of the downloader.' + ' Since this data rarely changes, it can be saved in the' + ' configuration.') Log.critical( f'Your user id is `{userid}` and the moodle version is `{version}`' ) print('') save_userid_and_version = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to store the user id and version number of Moodle in the configuration?' ), default_is_yes=False, ) if save_userid_and_version: Log.warning('Remember to delete the version number from the' + ' configuration once Moodle has been updated' + ' and then run the configurator again!') self.config_helper.set_property('userid', userid) self.config_helper.set_property('version', version) self.section_seperator()
def _select_should_download_quizzes(self): """ Asks the user if quizzes should be downloaded """ download_quizzes = self.config_helper.get_download_quizzes() self.section_seperator() Log.info( 'Quizzes are tests that a student must complete in a course and are graded on.' + ' Only quizzes that are in progress or have been completed will be downloaded.' ) print('') download_quizzes = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to download quizzes of your courses?'), default_is_yes=download_quizzes) self.config_helper.set_property('download_quizzes', download_quizzes)
def _select_should_download_workshops(self): """ Asks the user if workshops should be downloaded """ download_workshops = self.config_helper.get_download_workshops() self.section_seperator() Log.info( 'Workshops function according to the peer review process.' + ' Students can make submissions and have to assess submissions of other students. ' ) print('') download_workshops = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to download workshops of your courses?'), default_is_yes=download_workshops) self.config_helper.set_property('download_workshops', download_workshops)
def check_and_fetch_cookies(self, privatetoken: str, userid: str) -> bool: if os.path.exists(self.cookies_path): # test if still logged in. if self.test_cookies(self.moodle_test_url): return True warning_msg = 'Moodle cookie has expired, an attempt is made to generate a new cookie.' logging.warning(warning_msg) Log.warning('\r' + warning_msg + '\033[K') if privatetoken is None: error_msg = ( 'Moodle Cookies are not retrieved because no private token is set.' + ' To set a private token, use the `--new-token` option (if necessary also with `--sso`).' ) logging.warning(error_msg) Log.error('\r' + error_msg + '\033[K') self.delete_cookie_file() return False autologin_key = self.fetch_autologin_key(privatetoken) if autologin_key is None: error_msg = 'Failed to download autologin key!' logging.debug(error_msg) print('') Log.error(error_msg) self.delete_cookie_file() return False print('\rDownloading cookies\033[K', end='') post_data = {'key': autologin_key.get('key', ''), 'userid': userid} url = autologin_key.get('autologinurl', '') cookies_response, cookies_session = self.request_helper.post_URL( url, post_data, self.cookies_path) moodle_test_url = cookies_response.url if self.test_cookies(moodle_test_url): return True else: error_msg = 'Failed to generate cookies!' logging.debug(error_msg) print('') Log.error(error_msg) self.delete_cookie_file() return False
def _send_messages(self, messages: [str]): """ Sends an message """ if not self._is_configured() or messages is None or len(messages) == 0: return xmpp_cfg = self.config_helper.get_property('xmpp') logging.debug('Sending Notification via XMPP...') Log.debug('Sending Notification via XMPP... (Please wait)') try: xmpp = XmppShooter(xmpp_cfg['sender'], xmpp_cfg['password'], xmpp_cfg['target']) xmpp.send_messages(messages) except BaseException as e: error_formatted = traceback.format_exc() logging.error('While sending notification:\n%s' % (error_formatted), extra={'exception': e}) raise e # to be properly notified via Sentry
def _select_should_download_lessons(self): """ Asks the user if lessons should be downloaded """ download_lessons = self.config_helper.get_download_lessons() self.section_seperator() Log.info( 'Lessons are a kind of self-teaching with pages of information and other pages with questions to answer.' + ' A student can be graded on their answers after completing a lesson. Currently, only lessons without' + ' the answers are downloaded. The answers are potentially also available for download,' + ' but this has not been implemented.') print('') download_lessons = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to download lessons of your courses?'), default_is_yes=download_lessons) self.config_helper.set_property('download_lessons', download_lessons)
def _select_should_download_links_in_descriptions(self): """ Asks the user if links in descriptions should be downloaded """ download_links_in_descriptions = self.config_helper.get_download_links_in_descriptions( ) self.section_seperator() Log.info( 'In the descriptions of files, sections, assignments or courses the teacher can add links to webpages,' + ' files or videos. That links can pont to a internal page on moodle or to an external webpage.' ) print('') download_links_in_descriptions = cutie.prompt_yes_or_no( Log.special_str( 'Would you like to download links in descriptions?'), default_is_yes=download_links_in_descriptions, ) self.config_helper.set_property('download_links_in_descriptions', download_links_in_descriptions)
def _select_should_download_databases(self): """ Asks the user if databases should be downloaded """ download_databases = self.config_helper.get_download_databases() self.section_seperator() Log.info('In the database module of Moodle data can be stored' + ' structured with information. Often it is also' + ' possible for students to upload data there. Because' + ' the implementation of the downloader has not yet been' + ' optimized at this point, it is optional to download the' + ' databases. Currently only files are downloaded, thumbails' + ' are ignored.') print('') download_databases = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to download databases of your courses?'), default_is_yes=download_databases) self.config_helper.set_property('download_databases', download_databases)
def _select_should_download_submissions(self): """ Asks the user if submissions should be downloaded """ download_submissions = self.config_helper.get_download_submissions() self.section_seperator() Log.info('Submissions are files that you or a teacher have uploaded' + ' to your assignments. Moodle does not provide an' + ' interface for downloading information from all' + ' submissions to a course at once.') Log.warning( 'Therefore, it may be slow to monitor changes to submissions.') print('') download_submissions = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to download submissions of your assignments?'), default_is_yes=download_submissions, ) self.config_helper.set_property('download_submissions', download_submissions)
def receive_token() -> str: """ Starts an HTTP server to receive the SSO token from browser. It waits till a token was received. """ server_address = ('localhost', 80) try: httpd = HTTPServer(server_address, TransferServer) except PermissionError: Log.error('Permission denied: Please start the' + ' downloader once with administrator rights, so that it' + ' can wait on port 80 for the token.') sys.exit(1) extracted_token = None while extracted_token is None: httpd.handle_request() extracted_token = extract_token(TransferServer.received_token) httpd.server_close() return extracted_token
def _send_messages(self, messages: [str]): """ Sends an message """ if not self._is_configured() or messages is None or len(messages) == 0: return telegram_cfg = self.config_helper.get_property('telegram') logging.debug('Sending Notification via Telegram...') Log.debug('Sending Notification via Telegram... (Please wait)') telegram_shooter = TelegramShooter(telegram_cfg['token'], telegram_cfg['chat_id']) for message_content in messages: try: telegram_shooter.send(message_content) except BaseException as e: error_formatted = traceback.format_exc() logging.error('While sending notification:\n%s' % (error_formatted), extra={'exception': e}) raise e # to be properly notified via Sentry