def _select_should_download_linked_files(self):
"""
Asks the user if linked files should be downloaded
"""
download_linked_files = self.config_helper.get_download_linked_files()
self.section_seperator()
Log.info('In Moodle courses the teacher can also link to external' +
' files. This can be audio, video, text or anything else.' +
' In particular, the teacher can link to Youtube videos.')
Log.debug('To download videos correctly you have to install ffmpeg. ')
Log.error('These files can increase the download volume considerably.')
Log.info('If you want to filter the external links by their domain,' +
' you can manually set a whitelist and a blacklist' +
' (https://github.com/C0D3D3V/Moodle-Downloader-2/' +
'wiki/Download-(external)-linked-files' +
' for more details).')
Log.warning(
'Please note that the size of the external files is determined during the download, so the total size'
+ ' changes during the download.')
print('')
download_linked_files = cutie.prompt_yes_or_no(
Log.special_str(
'Would you like to download linked files of the courses you have selected?'
),
default_is_yes=download_linked_files,
)
self.config_helper.set_property('download_linked_files',
download_linked_files)
def _send_mail(self, subject, mail_content: (str, {str: str})):
"""
Sends an email
"""
if not self._is_configured():
return
mail_cfg = self.config_helper.get_property('mail')
try:
logging.debug('Sending Notification via Mail...')
Log.debug('Sending Notification via Mail... (Please wait)')
mail_shooter = MailShooter(
mail_cfg['sender'],
mail_cfg['server_host'],
int(mail_cfg['server_port']),
mail_cfg['username'],
mail_cfg['password'],
)
mail_shooter.send(mail_cfg['target'], subject, mail_content[0],
mail_content[1])
except BaseException as e:
error_formatted = traceback.format_exc()
logging.error('While sending notification:\n%s',
error_formatted,
extra={'exception': e})
raise e # to be properly notified via Sentry
def _select_should_download_descriptions(self):
"""
Asks the user if descriptions should be downloaded
"""
download_descriptions = self.config_helper.get_download_descriptions()
self.section_seperator()
Log.info(
'In Moodle courses, descriptions can be added to all kinds' +
' of resources, such as files, tasks, assignments or simply' +
' free text. These descriptions are usually unnecessary to' +
' download because you have already read the information or' +
' know it from context. However, there are situations where' +
' it might be interesting to download these descriptions. The' +
' descriptions are created as Markdown files and can be' +
' deleted as desired.')
Log.debug(
'Creating the description files does not take extra time, but they can be annoying'
+ ' if they only contain unnecessary information.')
print('')
download_descriptions = cutie.prompt_yes_or_no(
Log.special_str(
'Would you like to download descriptions of the courses you have selected?'
),
default_is_yes=download_descriptions,
)
self.config_helper.set_property('download_descriptions',
download_descriptions)
def _max_path_length_workaround(path):
# Working around MAX_PATH limitation on Windows (see
# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
if os.name == 'nt':
absfilepath = os.path.abspath(path)
path = '\\\\?\\' + absfilepath
Log.debug("Using absolute paths")
else:
Log.info("You are not on Windows, you don't need to use this workaround")
return path
def _send_messages(self, messages: [str]):
"""
Sends an message
"""
if not self._is_configured() or messages is None or len(messages) == 0:
return
xmpp_cfg = self.config_helper.get_property('xmpp')
logging.debug('Sending Notification via XMPP...')
Log.debug('Sending Notification via XMPP... (Please wait)')
try:
xmpp = XmppShooter(xmpp_cfg['sender'], xmpp_cfg['password'],
xmpp_cfg['target'])
xmpp.send_messages(messages)
except BaseException as e:
error_formatted = traceback.format_exc()
logging.error('While sending notification:\n%s' %
(error_formatted),
extra={'exception': e})
raise e # to be properly notified via Sentry
def _select_should_download_also_with_cookie(self):
"""
Ask the user whether files for which a cookie is required should be downloaded.
"""
download_also_with_cookie = self.config_helper.get_download_also_with_cookie(
)
self.section_seperator()
Log.info(
'Descriptions may contain links to files that require a browser cookie so they can be downloaded.'
+
' There are also several Moodle plugins that cannot be displayed in the Moodle app,'
+ ' so you need a browser cookie to download these plugin files.')
Log.debug(
'The Moodle browser cookie is created using your private token and stored in the `Configs.txt` file.'
+
' As long as this option is activated, the file always contains a valid cookie.'
)
if self.config_helper.get_privatetoken() is None:
Log.error(
'Currently no private token is stored in the configuration.' +
' Create a private token with moodle-dl --new-token (if necessary with --sso)'
)
print('')
download_also_with_cookie = cutie.prompt_yes_or_no(
Log.special_str(
'Would you like to download files for which a cookie is required?'
),
default_is_yes=download_also_with_cookie,
)
self.config_helper.set_property('download_also_with_cookie',
download_also_with_cookie)
def _send_messages(self, messages: [str]):
"""
Sends an message
"""
if not self._is_configured() or messages is None or len(messages) == 0:
return
telegram_cfg = self.config_helper.get_property('telegram')
logging.debug('Sending Notification via Telegram...')
Log.debug('Sending Notification via Telegram... (Please wait)')
telegram_shooter = TelegramShooter(telegram_cfg['token'],
telegram_cfg['chat_id'])
for message_content in messages:
try:
telegram_shooter.send(message_content)
except BaseException as e:
error_formatted = traceback.format_exc()
logging.error('While sending notification:\n%s' %
(error_formatted),
extra={'exception': e})
raise e # to be properly notified via Sentry
def check_debug():
global IS_DEBUG
if 'pydevd' in sys.modules:
IS_DEBUG = True
Log.debug('[RUNNING IN DEBUG-MODE!]')
def run_main(
storage_path,
verbose=False,
skip_cert_verify=False,
ignore_ytdl_errors=False,
without_downloading_files=False,
log_responses=False,
):
log_formatter = logging.Formatter(
'%(asctime)s %(levelname)s {%(module)s} %(message)s',
'%Y-%m-%d %H:%M:%S')
log_file = os.path.join(storage_path, 'MoodleDownloader.log')
log_handler = RotatingFileHandler(log_file,
mode='a',
maxBytes=1 * 1024 * 1024,
backupCount=2,
encoding='utf-8',
delay=0)
log_handler.setFormatter(log_formatter)
if verbose:
log_handler.setLevel(logging.DEBUG)
else:
log_handler.setLevel(logging.INFO)
app_log = logging.getLogger()
if verbose:
app_log.setLevel(logging.DEBUG)
else:
app_log.setLevel(logging.INFO)
app_log.addHandler(log_handler)
logging.info('--- moodle-dl started ---------------------')
Log.info('Moodle Downloader starting...')
if verbose:
logging.debug('moodle-dl version: %s', __version__)
logging.debug('python version: %s',
".".join(map(str, sys.version_info[:3])))
ffmpeg_available = which('ffmpeg') is not None
logging.debug('Is ffmpeg available: %s', ffmpeg_available)
if IS_DEBUG:
logging.info('Debug-Mode detected. Errors will be re-risen.')
app_log.addHandler(ReRaiseOnError())
try:
msg_load_config = 'Loading config...'
logging.debug(msg_load_config)
Log.debug(msg_load_config)
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error(
'Error while trying to load the Configuration! %s Exiting...',
e,
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
tg_service = TelegramService(config)
xmpp_service = XmppService(config)
console_service = ConsoleService(config)
PathTools.restricted_filenames = config.get_restricted_filenames()
try:
if not IS_DEBUG:
process_lock.lock(storage_path)
moodle = MoodleService(config, storage_path, skip_cert_verify,
log_responses)
msg_checking_for_changes = 'Checking for changes for the configured Moodle-Account....'
logging.debug(msg_checking_for_changes)
Log.debug(msg_checking_for_changes)
changed_courses = moodle.fetch_state()
if log_responses:
msg_responses_logged = (
"All JSON-responses from Moodle have been written to the responses.log file. Exiting..."
)
logging.debug(msg_responses_logged)
Log.success(msg_responses_logged)
process_lock.unlock(storage_path)
return
msg_start_downloading = 'Start downloading changed files...'
logging.debug(msg_start_downloading)
Log.debug(msg_start_downloading)
if without_downloading_files:
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path,
skip_cert_verify, ignore_ytdl_errors)
downloader.run()
failed_downloads = downloader.get_failed_url_targets()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
if len(changed_courses_to_notify) > 0:
console_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
tg_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
xmpp_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
else:
msg_no_changes = 'No changes found for the configured Moodle-Account.'
logging.info(msg_no_changes)
Log.warning(msg_no_changes)
if len(failed_downloads) > 0:
console_service.notify_about_failed_downloads(failed_downloads)
mail_service.notify_about_failed_downloads(failed_downloads)
tg_service.notify_about_failed_downloads(failed_downloads)
xmpp_service.notify_about_failed_downloads(failed_downloads)
process_lock.unlock(storage_path)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
print('\n')
if not isinstance(e, process_lock.LockError):
process_lock.unlock(storage_path)
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
if verbose:
Log.critical('Exception:\n%s' % (error_formatted))
short_error = str(e)
if not short_error or short_error.isspace():
short_error = traceback.format_exc(limit=1)
console_service.notify_about_error(short_error)
mail_service.notify_about_error(short_error)
tg_service.notify_about_error(short_error)
xmpp_service.notify_about_error(short_error)
logging.debug('Exception-Handling completed. Exiting...')
sys.exit(1)
