Example #1
0
def run_change_notification_telegram(storage_path):
    config = ConfigHelper(storage_path)
    config.load()

    TelegramService(config).interactively_configure()

    Log.success('Telegram Configuration successfully updated!')
Example #2
0
def run_change_notification_xmpp(storage_path):
    config = ConfigHelper(storage_path)
    config.load()

    XmppService(config).interactively_configure()

    Log.success('XMPP Configuration successfully updated!')
Example #3
0
def run_add_all_visible_courses(storage_path, skip_cert_verify):
    config = ConfigHelper(storage_path)
    config.load()  # because we do not want to override the other settings

    ConfigService(config, storage_path, skip_cert_verify).interactively_add_all_visible_courses()

    Log.success('Configuration successfully updated!')
Example #4
0
def run_configure(storage_path, skip_cert_verify=False):
    config = ConfigHelper(storage_path)
    config.load()  # because we do not want to override the other settings

    ConfigService(config, storage_path, skip_cert_verify).interactively_acquire_config()

    Log.success('Configuration successfully updated!')
Example #5
0
def run_manage_database(storage_path):
    config = ConfigHelper(storage_path)
    config.load()  # because we want to only manage configured courses

    offline_service = OfflineService(config, storage_path)
    offline_service.interactively_manage_database()

    Log.success('All done.')
Example #6
0
def run_delete_old_files(storage_path):
    config = ConfigHelper(storage_path)
    config.load()  # Not really needed, we check all local courses

    offline_service = OfflineService(config, storage_path)
    offline_service.delete_old_files()

    Log.success('All done.')
Example #7
0
def run_init(storage_path, use_sso=False, skip_cert_verify=False):
    config = ConfigHelper(storage_path)

    if config.is_present():
        do_override_input = cutie.prompt_yes_or_no(
            Log.error_str('Do you want to override the existing config?'))

        if not do_override_input:
            sys.exit(0)

    MailService(config).interactively_configure()
    TelegramService(config).interactively_configure()
    XmppService(config).interactively_configure()

    do_sentry = cutie.prompt_yes_or_no(
        'Do you want to configure Error Reporting via Sentry?')
    if do_sentry:
        sentry_dsn = input('Please enter your Sentry DSN:   ')
        config.set_property('sentry_dsn', sentry_dsn)

    moodle = MoodleService(config, storage_path, skip_cert_verify)

    if use_sso:
        moodle.interactively_acquire_sso_token()
    else:
        moodle.interactively_acquire_token()

    Log.success('Configuration finished and saved!')

    if os.name != 'nt':
        if storage_path == '.':
            Log.info(
                '  To set a cron-job for this program on your Unix-System:\n' +
                '    1. `crontab -e`\n' +
                '    2. Add `*/15 * * * * cd "{}" && moodle-dl`\n'.format(
                    os.getcwd()) + '    3. Save and you\'re done!')
        else:
            Log.info(
                '  To set a cron-job for this program on your Unix-System:\n' +
                '    1. `crontab -e`\n' +
                '    2. Add `*/15 * * * * cd "{}" && moodle-dl -p "{}"`\n'.
                format(os.getcwd(), storage_path) +
                '    3. Save and you\'re done!')

    print('')

    Log.info(
        'You can always do the additional configuration later with the --config option.'
    )

    do_config = cutie.prompt_yes_or_no(
        'Do you want to make additional configurations now?')

    if do_config:
        run_configure(storage_path, skip_cert_verify)

    print('')
    Log.success('All set and ready to go!')
Example #8
0
def run_init(storage_path, use_sso=False, skip_cert_verify=False):
    config = ConfigHelper(storage_path)

    if config.is_present():
        do_override_input = cutie.prompt_yes_or_no(Log.error_str('Do you want to override the existing config?'))

        if not do_override_input:
            sys.exit(0)

    MailService(config).interactively_configure()
    TelegramService(config).interactively_configure()
    XmppService(config).interactively_configure()

    do_sentry = cutie.prompt_yes_or_no('Do you want to configure Error Reporting via Sentry?')
    if do_sentry:
        sentry_dsn = input('Please enter your Sentry DSN:   ')
        config.set_property('sentry_dsn', sentry_dsn)

    moodle = MoodleService(config, storage_path, skip_cert_verify)

    if use_sso:
        moodle.interactively_acquire_sso_token()
    else:
        moodle.interactively_acquire_token()

    Log.success('Configuration finished and saved!')

    if os.name != 'nt':
        working_dir = os.path.abspath(storage_path)
        moodle_dl_path = os.path.abspath(sys.argv[0])
        Log.info(
            '  To set a cron-job for this program on your Unix-System:\n'
            + '    1. `crontab -e`\n'
            + '    2. Add `*/15 * * * * cd "{}" && "{}" >/dev/null 2>&1`\n'.format(working_dir, moodle_dl_path)
            + '    3. Save and you\'re done!'
        )

        Log.info(
            'For more ways to run `moodle-dl` periodically, take a look at the wiki (https://github.com/C0D3D3V/Moodle-Downloader-2/wiki/Start-Moodle-dl-periodically-or-via-Telegram)'
        )
    else:
        Log.info(
            'If you want to run moodle-dl periodically, you can take a look at the wiki (https://github.com/C0D3D3V/Moodle-Downloader-2/wiki/Start-Moodle-dl-periodically-or-via-Telegram)'
        )

    print('')

    Log.info('You can always do the additional configuration later with the --config option.')

    do_config = cutie.prompt_yes_or_no('Do you want to make additional configurations now?')

    if do_config:
        run_configure(storage_path, skip_cert_verify)

    print('')
    Log.success('All set and ready to go!')
Example #9
0
def run_new_token(storage_path, use_sso=False, username: str = None, password: str = None, skip_cert_verify=False):
    config = ConfigHelper(storage_path)
    config.load()  # because we do not want to override the other settings

    moodle = MoodleService(config, storage_path, skip_cert_verify)

    if use_sso:
        moodle.interactively_acquire_sso_token(use_stored_url=True)
    else:
        moodle.interactively_acquire_token(use_stored_url=True, username=username, password=password)

    Log.success('New Token successfully saved!')
Example #10
0
    def filter_courses(
        changes: [Course],
        config_helper: ConfigHelper,
        cookie_handler: CookieHandler = None,
        courses_list: [Course] = None,
    ) -> [Course]:
        """
        Filters the changes course list from courses that
        should not get downloaded
        @param config_helper: ConfigHelper to obtain all the diffrent filter configs
        @param cookie_handler: CookieHandler to check if the cookie is valid
        @param courses_list: A list of all courses that are available online
        @return: filtered changes course list
        """

        download_course_ids = config_helper.get_download_course_ids()
        download_public_course_ids = config_helper.get_download_public_course_ids()
        dont_download_course_ids = config_helper.get_dont_download_course_ids()
        download_submissions = config_helper.get_download_submissions()
        download_descriptions = config_helper.get_download_descriptions()
        download_links_in_descriptions = config_helper.get_download_links_in_descriptions()
        download_databases = config_helper.get_download_databases()
        download_quizzes = config_helper.get_download_quizzes()
        download_lessons = config_helper.get_download_lessons()
        download_workshops = config_helper.get_download_workshops()
        exclude_file_extensions = config_helper.get_exclude_file_extensions()
        download_also_with_cookie = config_helper.get_download_also_with_cookie()
        if cookie_handler is not None:
            download_also_with_cookie = cookie_handler.test_cookies()

        filtered_changes = []

        for course in changes:
            if not ResultsHandler.should_download_course(
                course.id, download_course_ids + download_public_course_ids, dont_download_course_ids
            ):
                # Filter courses that should not be downloaded
                continue

            if courses_list is not None:
                not_online = True
                # Filter courses that are not available online
                for online_course in courses_list:
                    if online_course.id == course.id:
                        not_online = False
                        break
                if not_online:
                    Log.warning(f'The Moodle course with id {course.id} is no longer available online.')
                    logging.warning('The Moodle course with id %d is no longer available online.', course.id)
                    continue

            course_files = []
            for file in course.files:
                # Filter Files based on options
                if (
                    # Filter Assignment Submission Files
                    (download_submissions or (not (file.module_modname.endswith('assign') and file.deleted)))
                    # Filter Description Files (except the forum posts)
                    and (
                        download_descriptions
                        or file.content_type != 'description'
                        or (file.module_modname == 'forum' and file.content_type == 'description' and file.content_filename != 'Forum intro')
                    )
                    # Filter Database Files
                    and (download_databases or file.content_type != 'database_file')
                    # Filter Quiz Files
                    and (download_quizzes or (not (file.module_modname.endswith('quiz') and file.deleted)))
                    # Filter Lesson Files
                    and (download_lessons or (not (file.module_modname.endswith('lesson') and file.deleted)))
                    # Filter Workshops Files
                    and (download_workshops or (not (file.module_modname.endswith('workshop') and file.deleted)))
                    # Filter Files that requiere a Cookie
                    and (download_also_with_cookie or (not file.module_modname.startswith('cookie_mod-')))
                    # Exclude files whose file extension is blacklisted
                    and (not (determine_ext(file.content_filename) in exclude_file_extensions))
                    # Exclude files that are in excluded sections
                    and (ResultsHandler.should_download_section(file.section_id, course.excluded_sections))
                ):
                    course_files.append(file)
            course.files = course_files

            # Filter Description URLs
            course_files = []
            for file in course.files:
                if not file.content_type == 'description-url':
                    course_files.append(file)

                elif download_links_in_descriptions:
                    add_description_url = True
                    for test_file in course.files:
                        if file.content_fileurl == test_file.content_fileurl:
                            if test_file.content_type != 'description-url':
                                # If a URL in a description also exists as a real link in the course,
                                # then ignore this URL
                                add_description_url = False
                                break
                            elif file.module_id > test_file.module_id:
                                # Always use the link from the older description.
                                add_description_url = False
                                break

                    if add_description_url:
                        course_files.append(file)
            course.files = course_files

            if len(course.files) > 0:
                filtered_changes.append(course)

        return filtered_changes
Example #11
0
    def filter_courses(changes: [Course],
                       config_helper: ConfigHelper,
                       cookie_handler: CookieHandler = None) -> [Course]:
        """
        Filters the changes course list from courses that
        should not get downloaded
        @param config_helper: ConfigHelper to obtain all the diffrent filter configs
        @param cookie_handler: CookieHandler to check if the cookie is valid
        @return: filtered changes course list
        """

        download_course_ids = config_helper.get_download_course_ids()
        download_public_course_ids = config_helper.get_download_public_course_ids(
        )
        dont_download_course_ids = config_helper.get_dont_download_course_ids()
        download_submissions = config_helper.get_download_submissions()
        download_descriptions = config_helper.get_download_descriptions()
        download_links_in_descriptions = config_helper.get_download_links_in_descriptions(
        )
        download_databases = config_helper.get_download_databases()
        download_also_with_cookie = config_helper.get_download_also_with_cookie(
        )
        if cookie_handler is not None:
            download_also_with_cookie = cookie_handler.test_cookies()

        filtered_changes = []

        for course in changes:
            if not download_submissions:
                course_files = []
                for file in course.files:
                    if not (file.module_modname.endswith('assign')
                            and file.deleted):
                        course_files.append(file)
                course.files = course_files

            if not download_descriptions:
                course_files = []
                for file in course.files:
                    if file.content_type != 'description':
                        course_files.append(file)
                course.files = course_files

            course_files = []
            for file in course.files:
                if not file.content_type == 'description-url':
                    course_files.append(file)

                elif download_links_in_descriptions:
                    add_description_url = True
                    for test_file in course.files:
                        if file.content_fileurl == test_file.content_fileurl:
                            if test_file.content_type != 'description-url':
                                # If a URL in a description also exists as a real link in the course,
                                # then ignore this URL
                                add_description_url = False
                                break
                            elif file.module_id > test_file.module_id:
                                # Always use the link from the older description.
                                add_description_url = False
                                break

                    if add_description_url:
                        course_files.append(file)

            course.files = course_files

            if not download_databases:
                course_files = []
                for file in course.files:
                    if file.content_type != 'database_file':
                        course_files.append(file)
                course.files = course_files

            if not download_also_with_cookie:
                course_files = []
                for file in course.files:
                    if not file.module_modname.startswith('cookie_mod-'):
                        course_files.append(file)
                course.files = course_files

            if (ResultsHandler.should_download_course(
                    course.id, download_course_ids +
                    download_public_course_ids, dont_download_course_ids)
                    and len(course.files) > 0):
                filtered_changes.append(course)

        return filtered_changes
Example #12
0
def run_main(
    storage_path,
    verbose=False,
    skip_cert_verify=False,
    ignore_ytdl_errors=False,
    without_downloading_files=False,
    log_responses=False,
):

    log_formatter = logging.Formatter(
        '%(asctime)s  %(levelname)s  {%(module)s}  %(message)s',
        '%Y-%m-%d %H:%M:%S')
    log_file = os.path.join(storage_path, 'MoodleDownloader.log')
    log_handler = RotatingFileHandler(log_file,
                                      mode='a',
                                      maxBytes=1 * 1024 * 1024,
                                      backupCount=2,
                                      encoding='utf-8',
                                      delay=0)

    log_handler.setFormatter(log_formatter)
    if verbose:
        log_handler.setLevel(logging.DEBUG)
    else:
        log_handler.setLevel(logging.INFO)

    app_log = logging.getLogger()
    if verbose:
        app_log.setLevel(logging.DEBUG)
    else:
        app_log.setLevel(logging.INFO)
    app_log.addHandler(log_handler)

    logging.info('--- moodle-dl started ---------------------')
    Log.info('Moodle Downloader starting...')
    if verbose:
        logging.debug('moodle-dl version: %s', __version__)
        logging.debug('python version: %s',
                      ".".join(map(str, sys.version_info[:3])))
        ffmpeg_available = which('ffmpeg') is not None
        logging.debug('Is ffmpeg available: %s', ffmpeg_available)

    if IS_DEBUG:
        logging.info('Debug-Mode detected. Errors will be re-risen.')
        app_log.addHandler(ReRaiseOnError())

    try:
        msg_load_config = 'Loading config...'
        logging.debug(msg_load_config)
        Log.debug(msg_load_config)

        config = ConfigHelper(storage_path)
        config.load()
    except BaseException as e:
        logging.error(
            'Error while trying to load the Configuration! %s Exiting...',
            e,
            extra={'exception': e})
        Log.error('Error while trying to load the Configuration!')
        sys.exit(1)

    r_client = False
    try:
        sentry_dsn = config.get_property('sentry_dsn')
        if sentry_dsn:
            sentry_sdk.init(sentry_dsn)
    except BaseException:
        pass

    mail_service = MailService(config)
    tg_service = TelegramService(config)
    xmpp_service = XmppService(config)
    console_service = ConsoleService(config)

    PathTools.restricted_filenames = config.get_restricted_filenames()

    try:
        if not IS_DEBUG:
            process_lock.lock(storage_path)

        moodle = MoodleService(config, storage_path, skip_cert_verify,
                               log_responses)

        msg_checking_for_changes = 'Checking for changes for the configured Moodle-Account....'
        logging.debug(msg_checking_for_changes)
        Log.debug(msg_checking_for_changes)
        changed_courses = moodle.fetch_state()

        if log_responses:
            msg_responses_logged = (
                "All JSON-responses from Moodle have been written to the responses.log file. Exiting..."
            )
            logging.debug(msg_responses_logged)
            Log.success(msg_responses_logged)
            process_lock.unlock(storage_path)
            return

        msg_start_downloading = 'Start downloading changed files...'
        logging.debug(msg_start_downloading)
        Log.debug(msg_start_downloading)

        if without_downloading_files:
            downloader = FakeDownloadService(changed_courses, moodle,
                                             storage_path)
        else:
            downloader = DownloadService(changed_courses, moodle, storage_path,
                                         skip_cert_verify, ignore_ytdl_errors)
        downloader.run()
        failed_downloads = downloader.get_failed_url_targets()

        changed_courses_to_notify = moodle.recorder.changes_to_notify()

        if len(changed_courses_to_notify) > 0:
            console_service.notify_about_changes_in_moodle(
                changed_courses_to_notify)
            mail_service.notify_about_changes_in_moodle(
                changed_courses_to_notify)
            tg_service.notify_about_changes_in_moodle(
                changed_courses_to_notify)
            xmpp_service.notify_about_changes_in_moodle(
                changed_courses_to_notify)

            moodle.recorder.notified(changed_courses_to_notify)

        else:
            msg_no_changes = 'No changes found for the configured Moodle-Account.'
            logging.info(msg_no_changes)
            Log.warning(msg_no_changes)

        if len(failed_downloads) > 0:
            console_service.notify_about_failed_downloads(failed_downloads)
            mail_service.notify_about_failed_downloads(failed_downloads)
            tg_service.notify_about_failed_downloads(failed_downloads)
            xmpp_service.notify_about_failed_downloads(failed_downloads)

        process_lock.unlock(storage_path)

        logging.debug('All done. Exiting...')
        Log.success('All done. Exiting..')
    except BaseException as e:
        print('\n')
        if not isinstance(e, process_lock.LockError):
            process_lock.unlock(storage_path)

        error_formatted = traceback.format_exc()
        logging.error(error_formatted, extra={'exception': e})

        if r_client:
            sentry_sdk.capture_exception(e)

        if verbose:
            Log.critical('Exception:\n%s' % (error_formatted))

        short_error = str(e)
        if not short_error or short_error.isspace():
            short_error = traceback.format_exc(limit=1)

        console_service.notify_about_error(short_error)
        mail_service.notify_about_error(short_error)
        tg_service.notify_about_error(short_error)
        xmpp_service.notify_about_error(short_error)

        logging.debug('Exception-Handling completed. Exiting...')

        sys.exit(1)
Example #13
0
def run_init(storage_path, use_sso=False, skip_cert_verify=False):
    config = ConfigHelper(storage_path)

    if config.is_present():
        do_override_input = cutie.prompt_yes_or_no(Log.error_str('Do you want to override the existing config?'))

        if not do_override_input:
            sys.exit(0)

    MailService(config).interactively_configure()
    TelegramService(config).interactively_configure()

    do_sentry = cutie.prompt_yes_or_no('Do you want to configure Error Reporting via Sentry?')
    if do_sentry:
        sentry_dsn = input('Please enter your Sentry DSN:   ')
        config.set_property('sentry_dsn', sentry_dsn)

    moodle = MoodleService(config, storage_path, skip_cert_verify)

    if use_sso:
        moodle.interactively_acquire_sso_token()
    else:
        moodle.interactively_acquire_token()

    if os.name != 'nt':
        Log.info(
            'On Windows many characters are forbidden in filenames and paths, if you want, these characters can be'
            + ' automatically removed from filenames.'
        )

        Log.warning('If you want to view the downloaded files on Windows this is important!')

        default_windows_map = cutie.prompt_yes_or_no(
            'Do you want to load the default filename character map for windows?'
        )
        if default_windows_map:
            config.set_default_filename_character_map(True)
        else:
            config.set_default_filename_character_map(False)
    else:
        config.set_default_filename_character_map(True)

    Log.success('Configuration finished and saved!')

    if os.name != 'nt':
        if storage_path == '.':
            Log.info(
                '  To set a cron-job for this program on your Unix-System:\n'
                + '    1. `crontab -e`\n'
                + '    2. Add `*/15 * * * * cd %s && python3 %smain.py`\n'
                % (os.getcwd(), os.path.join(os.path.dirname(os.path.realpath(__file__)), ''))
                + '    3. Save and you\'re done!'
            )
        else:
            Log.info(
                '  To set a cron-job for this program on your Unix-System:\n'
                + '    1. `crontab -e`\n'
                + '    2. Add `*/15 * * * *'
                + ' cd %s && python3 %smain.py --path %s`\n'
                % (os.getcwd(), os.path.join(os.path.dirname(os.path.realpath(__file__)), ''), storage_path)
                + '    3. Save and you\'re done!'
            )

    print('')

    Log.info('You can always do the additional configuration later with the --config option.')

    do_config = cutie.prompt_yes_or_no('Do you want to make additional configurations now?')

    if do_config:
        run_configure(storage_path, skip_cert_verify)

    print('')
    Log.success('All set and ready to go!')