def run_change_notification_telegram(storage_path):
config = ConfigHelper(storage_path)
config.load()
TelegramService(config).interactively_configure()
Log.success('Telegram Configuration successfully updated!')
def run_change_notification_xmpp(storage_path):
config = ConfigHelper(storage_path)
config.load()
XmppService(config).interactively_configure()
Log.success('XMPP Configuration successfully updated!')
def run_add_all_visible_courses(storage_path, skip_cert_verify):
config = ConfigHelper(storage_path)
config.load() # because we do not want to override the other settings
ConfigService(config, storage_path, skip_cert_verify).interactively_add_all_visible_courses()
Log.success('Configuration successfully updated!')
def run_configure(storage_path, skip_cert_verify=False):
config = ConfigHelper(storage_path)
config.load() # because we do not want to override the other settings
ConfigService(config, storage_path, skip_cert_verify).interactively_acquire_config()
Log.success('Configuration successfully updated!')
def run_manage_database(storage_path):
config = ConfigHelper(storage_path)
config.load() # because we want to only manage configured courses
offline_service = OfflineService(config, storage_path)
offline_service.interactively_manage_database()
Log.success('All done.')
def run_delete_old_files(storage_path):
config = ConfigHelper(storage_path)
config.load() # Not really needed, we check all local courses
offline_service = OfflineService(config, storage_path)
offline_service.delete_old_files()
Log.success('All done.')
def run_init(storage_path, use_sso=False, skip_cert_verify=False):
config = ConfigHelper(storage_path)
if config.is_present():
do_override_input = cutie.prompt_yes_or_no(
Log.error_str('Do you want to override the existing config?'))
if not do_override_input:
sys.exit(0)
MailService(config).interactively_configure()
TelegramService(config).interactively_configure()
XmppService(config).interactively_configure()
do_sentry = cutie.prompt_yes_or_no(
'Do you want to configure Error Reporting via Sentry?')
if do_sentry:
sentry_dsn = input('Please enter your Sentry DSN: ')
config.set_property('sentry_dsn', sentry_dsn)
moodle = MoodleService(config, storage_path, skip_cert_verify)
if use_sso:
moodle.interactively_acquire_sso_token()
else:
moodle.interactively_acquire_token()
Log.success('Configuration finished and saved!')
if os.name != 'nt':
if storage_path == '.':
Log.info(
' To set a cron-job for this program on your Unix-System:\n' +
' 1. `crontab -e`\n' +
' 2. Add `*/15 * * * * cd "{}" && moodle-dl`\n'.format(
os.getcwd()) + ' 3. Save and you\'re done!')
else:
Log.info(
' To set a cron-job for this program on your Unix-System:\n' +
' 1. `crontab -e`\n' +
' 2. Add `*/15 * * * * cd "{}" && moodle-dl -p "{}"`\n'.
format(os.getcwd(), storage_path) +
' 3. Save and you\'re done!')
print('')
Log.info(
'You can always do the additional configuration later with the --config option.'
)
do_config = cutie.prompt_yes_or_no(
'Do you want to make additional configurations now?')
if do_config:
run_configure(storage_path, skip_cert_verify)
print('')
Log.success('All set and ready to go!')
def run_init(storage_path, use_sso=False, skip_cert_verify=False):
config = ConfigHelper(storage_path)
if config.is_present():
do_override_input = cutie.prompt_yes_or_no(Log.error_str('Do you want to override the existing config?'))
if not do_override_input:
sys.exit(0)
MailService(config).interactively_configure()
TelegramService(config).interactively_configure()
XmppService(config).interactively_configure()
do_sentry = cutie.prompt_yes_or_no('Do you want to configure Error Reporting via Sentry?')
if do_sentry:
sentry_dsn = input('Please enter your Sentry DSN: ')
config.set_property('sentry_dsn', sentry_dsn)
moodle = MoodleService(config, storage_path, skip_cert_verify)
if use_sso:
moodle.interactively_acquire_sso_token()
else:
moodle.interactively_acquire_token()
Log.success('Configuration finished and saved!')
if os.name != 'nt':
working_dir = os.path.abspath(storage_path)
moodle_dl_path = os.path.abspath(sys.argv[0])
Log.info(
' To set a cron-job for this program on your Unix-System:\n'
+ ' 1. `crontab -e`\n'
+ ' 2. Add `*/15 * * * * cd "{}" && "{}" >/dev/null 2>&1`\n'.format(working_dir, moodle_dl_path)
+ ' 3. Save and you\'re done!'
)
Log.info(
'For more ways to run `moodle-dl` periodically, take a look at the wiki (https://github.com/C0D3D3V/Moodle-Downloader-2/wiki/Start-Moodle-dl-periodically-or-via-Telegram)'
)
else:
Log.info(
'If you want to run moodle-dl periodically, you can take a look at the wiki (https://github.com/C0D3D3V/Moodle-Downloader-2/wiki/Start-Moodle-dl-periodically-or-via-Telegram)'
)
print('')
Log.info('You can always do the additional configuration later with the --config option.')
do_config = cutie.prompt_yes_or_no('Do you want to make additional configurations now?')
if do_config:
run_configure(storage_path, skip_cert_verify)
print('')
Log.success('All set and ready to go!')
def run_new_token(storage_path, use_sso=False, username: str = None, password: str = None, skip_cert_verify=False):
config = ConfigHelper(storage_path)
config.load() # because we do not want to override the other settings
moodle = MoodleService(config, storage_path, skip_cert_verify)
if use_sso:
moodle.interactively_acquire_sso_token(use_stored_url=True)
else:
moodle.interactively_acquire_token(use_stored_url=True, username=username, password=password)
Log.success('New Token successfully saved!')
def filter_courses(
changes: [Course],
config_helper: ConfigHelper,
cookie_handler: CookieHandler = None,
courses_list: [Course] = None,
) -> [Course]:
"""
Filters the changes course list from courses that
should not get downloaded
@param config_helper: ConfigHelper to obtain all the diffrent filter configs
@param cookie_handler: CookieHandler to check if the cookie is valid
@param courses_list: A list of all courses that are available online
@return: filtered changes course list
"""
download_course_ids = config_helper.get_download_course_ids()
download_public_course_ids = config_helper.get_download_public_course_ids()
dont_download_course_ids = config_helper.get_dont_download_course_ids()
download_submissions = config_helper.get_download_submissions()
download_descriptions = config_helper.get_download_descriptions()
download_links_in_descriptions = config_helper.get_download_links_in_descriptions()
download_databases = config_helper.get_download_databases()
download_quizzes = config_helper.get_download_quizzes()
download_lessons = config_helper.get_download_lessons()
download_workshops = config_helper.get_download_workshops()
exclude_file_extensions = config_helper.get_exclude_file_extensions()
download_also_with_cookie = config_helper.get_download_also_with_cookie()
if cookie_handler is not None:
download_also_with_cookie = cookie_handler.test_cookies()
filtered_changes = []
for course in changes:
if not ResultsHandler.should_download_course(
course.id, download_course_ids + download_public_course_ids, dont_download_course_ids
):
# Filter courses that should not be downloaded
continue
if courses_list is not None:
not_online = True
# Filter courses that are not available online
for online_course in courses_list:
if online_course.id == course.id:
not_online = False
break
if not_online:
Log.warning(f'The Moodle course with id {course.id} is no longer available online.')
logging.warning('The Moodle course with id %d is no longer available online.', course.id)
continue
course_files = []
for file in course.files:
# Filter Files based on options
if (
# Filter Assignment Submission Files
(download_submissions or (not (file.module_modname.endswith('assign') and file.deleted)))
# Filter Description Files (except the forum posts)
and (
download_descriptions
or file.content_type != 'description'
or (file.module_modname == 'forum' and file.content_type == 'description' and file.content_filename != 'Forum intro')
)
# Filter Database Files
and (download_databases or file.content_type != 'database_file')
# Filter Quiz Files
and (download_quizzes or (not (file.module_modname.endswith('quiz') and file.deleted)))
# Filter Lesson Files
and (download_lessons or (not (file.module_modname.endswith('lesson') and file.deleted)))
# Filter Workshops Files
and (download_workshops or (not (file.module_modname.endswith('workshop') and file.deleted)))
# Filter Files that requiere a Cookie
and (download_also_with_cookie or (not file.module_modname.startswith('cookie_mod-')))
# Exclude files whose file extension is blacklisted
and (not (determine_ext(file.content_filename) in exclude_file_extensions))
# Exclude files that are in excluded sections
and (ResultsHandler.should_download_section(file.section_id, course.excluded_sections))
):
course_files.append(file)
course.files = course_files
# Filter Description URLs
course_files = []
for file in course.files:
if not file.content_type == 'description-url':
course_files.append(file)
elif download_links_in_descriptions:
add_description_url = True
for test_file in course.files:
if file.content_fileurl == test_file.content_fileurl:
if test_file.content_type != 'description-url':
# If a URL in a description also exists as a real link in the course,
# then ignore this URL
add_description_url = False
break
elif file.module_id > test_file.module_id:
# Always use the link from the older description.
add_description_url = False
break
if add_description_url:
course_files.append(file)
course.files = course_files
if len(course.files) > 0:
filtered_changes.append(course)
return filtered_changes
def filter_courses(changes: [Course],
config_helper: ConfigHelper,
cookie_handler: CookieHandler = None) -> [Course]:
"""
Filters the changes course list from courses that
should not get downloaded
@param config_helper: ConfigHelper to obtain all the diffrent filter configs
@param cookie_handler: CookieHandler to check if the cookie is valid
@return: filtered changes course list
"""
download_course_ids = config_helper.get_download_course_ids()
download_public_course_ids = config_helper.get_download_public_course_ids(
)
dont_download_course_ids = config_helper.get_dont_download_course_ids()
download_submissions = config_helper.get_download_submissions()
download_descriptions = config_helper.get_download_descriptions()
download_links_in_descriptions = config_helper.get_download_links_in_descriptions(
)
download_databases = config_helper.get_download_databases()
download_also_with_cookie = config_helper.get_download_also_with_cookie(
)
if cookie_handler is not None:
download_also_with_cookie = cookie_handler.test_cookies()
filtered_changes = []
for course in changes:
if not download_submissions:
course_files = []
for file in course.files:
if not (file.module_modname.endswith('assign')
and file.deleted):
course_files.append(file)
course.files = course_files
if not download_descriptions:
course_files = []
for file in course.files:
if file.content_type != 'description':
course_files.append(file)
course.files = course_files
course_files = []
for file in course.files:
if not file.content_type == 'description-url':
course_files.append(file)
elif download_links_in_descriptions:
add_description_url = True
for test_file in course.files:
if file.content_fileurl == test_file.content_fileurl:
if test_file.content_type != 'description-url':
# If a URL in a description also exists as a real link in the course,
# then ignore this URL
add_description_url = False
break
elif file.module_id > test_file.module_id:
# Always use the link from the older description.
add_description_url = False
break
if add_description_url:
course_files.append(file)
course.files = course_files
if not download_databases:
course_files = []
for file in course.files:
if file.content_type != 'database_file':
course_files.append(file)
course.files = course_files
if not download_also_with_cookie:
course_files = []
for file in course.files:
if not file.module_modname.startswith('cookie_mod-'):
course_files.append(file)
course.files = course_files
if (ResultsHandler.should_download_course(
course.id, download_course_ids +
download_public_course_ids, dont_download_course_ids)
and len(course.files) > 0):
filtered_changes.append(course)
return filtered_changes
def run_main(
storage_path,
verbose=False,
skip_cert_verify=False,
ignore_ytdl_errors=False,
without_downloading_files=False,
log_responses=False,
):
log_formatter = logging.Formatter(
'%(asctime)s %(levelname)s {%(module)s} %(message)s',
'%Y-%m-%d %H:%M:%S')
log_file = os.path.join(storage_path, 'MoodleDownloader.log')
log_handler = RotatingFileHandler(log_file,
mode='a',
maxBytes=1 * 1024 * 1024,
backupCount=2,
encoding='utf-8',
delay=0)
log_handler.setFormatter(log_formatter)
if verbose:
log_handler.setLevel(logging.DEBUG)
else:
log_handler.setLevel(logging.INFO)
app_log = logging.getLogger()
if verbose:
app_log.setLevel(logging.DEBUG)
else:
app_log.setLevel(logging.INFO)
app_log.addHandler(log_handler)
logging.info('--- moodle-dl started ---------------------')
Log.info('Moodle Downloader starting...')
if verbose:
logging.debug('moodle-dl version: %s', __version__)
logging.debug('python version: %s',
".".join(map(str, sys.version_info[:3])))
ffmpeg_available = which('ffmpeg') is not None
logging.debug('Is ffmpeg available: %s', ffmpeg_available)
if IS_DEBUG:
logging.info('Debug-Mode detected. Errors will be re-risen.')
app_log.addHandler(ReRaiseOnError())
try:
msg_load_config = 'Loading config...'
logging.debug(msg_load_config)
Log.debug(msg_load_config)
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error(
'Error while trying to load the Configuration! %s Exiting...',
e,
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
tg_service = TelegramService(config)
xmpp_service = XmppService(config)
console_service = ConsoleService(config)
PathTools.restricted_filenames = config.get_restricted_filenames()
try:
if not IS_DEBUG:
process_lock.lock(storage_path)
moodle = MoodleService(config, storage_path, skip_cert_verify,
log_responses)
msg_checking_for_changes = 'Checking for changes for the configured Moodle-Account....'
logging.debug(msg_checking_for_changes)
Log.debug(msg_checking_for_changes)
changed_courses = moodle.fetch_state()
if log_responses:
msg_responses_logged = (
"All JSON-responses from Moodle have been written to the responses.log file. Exiting..."
)
logging.debug(msg_responses_logged)
Log.success(msg_responses_logged)
process_lock.unlock(storage_path)
return
msg_start_downloading = 'Start downloading changed files...'
logging.debug(msg_start_downloading)
Log.debug(msg_start_downloading)
if without_downloading_files:
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path,
skip_cert_verify, ignore_ytdl_errors)
downloader.run()
failed_downloads = downloader.get_failed_url_targets()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
if len(changed_courses_to_notify) > 0:
console_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
tg_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
xmpp_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
else:
msg_no_changes = 'No changes found for the configured Moodle-Account.'
logging.info(msg_no_changes)
Log.warning(msg_no_changes)
if len(failed_downloads) > 0:
console_service.notify_about_failed_downloads(failed_downloads)
mail_service.notify_about_failed_downloads(failed_downloads)
tg_service.notify_about_failed_downloads(failed_downloads)
xmpp_service.notify_about_failed_downloads(failed_downloads)
process_lock.unlock(storage_path)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
print('\n')
if not isinstance(e, process_lock.LockError):
process_lock.unlock(storage_path)
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
if verbose:
Log.critical('Exception:\n%s' % (error_formatted))
short_error = str(e)
if not short_error or short_error.isspace():
short_error = traceback.format_exc(limit=1)
console_service.notify_about_error(short_error)
mail_service.notify_about_error(short_error)
tg_service.notify_about_error(short_error)
xmpp_service.notify_about_error(short_error)
logging.debug('Exception-Handling completed. Exiting...')
sys.exit(1)
def run_init(storage_path, use_sso=False, skip_cert_verify=False):
config = ConfigHelper(storage_path)
if config.is_present():
do_override_input = cutie.prompt_yes_or_no(Log.error_str('Do you want to override the existing config?'))
if not do_override_input:
sys.exit(0)
MailService(config).interactively_configure()
TelegramService(config).interactively_configure()
do_sentry = cutie.prompt_yes_or_no('Do you want to configure Error Reporting via Sentry?')
if do_sentry:
sentry_dsn = input('Please enter your Sentry DSN: ')
config.set_property('sentry_dsn', sentry_dsn)
moodle = MoodleService(config, storage_path, skip_cert_verify)
if use_sso:
moodle.interactively_acquire_sso_token()
else:
moodle.interactively_acquire_token()
if os.name != 'nt':
Log.info(
'On Windows many characters are forbidden in filenames and paths, if you want, these characters can be'
+ ' automatically removed from filenames.'
)
Log.warning('If you want to view the downloaded files on Windows this is important!')
default_windows_map = cutie.prompt_yes_or_no(
'Do you want to load the default filename character map for windows?'
)
if default_windows_map:
config.set_default_filename_character_map(True)
else:
config.set_default_filename_character_map(False)
else:
config.set_default_filename_character_map(True)
Log.success('Configuration finished and saved!')
if os.name != 'nt':
if storage_path == '.':
Log.info(
' To set a cron-job for this program on your Unix-System:\n'
+ ' 1. `crontab -e`\n'
+ ' 2. Add `*/15 * * * * cd %s && python3 %smain.py`\n'
% (os.getcwd(), os.path.join(os.path.dirname(os.path.realpath(__file__)), ''))
+ ' 3. Save and you\'re done!'
)
else:
Log.info(
' To set a cron-job for this program on your Unix-System:\n'
+ ' 1. `crontab -e`\n'
+ ' 2. Add `*/15 * * * *'
+ ' cd %s && python3 %smain.py --path %s`\n'
% (os.getcwd(), os.path.join(os.path.dirname(os.path.realpath(__file__)), ''), storage_path)
+ ' 3. Save and you\'re done!'
)
print('')
Log.info('You can always do the additional configuration later with the --config option.')
do_config = cutie.prompt_yes_or_no('Do you want to make additional configurations now?')
if do_config:
run_configure(storage_path, skip_cert_verify)
print('')
Log.success('All set and ready to go!')
