def run_init(storage_path):
config = ConfigHelper(storage_path)
if config.is_present():
do_override_input = input('Do you want to override the existing' +
' config [y/n]? ').lower()
while do_override_input not in ['y', 'n']:
do_override_input = input('Unrecognized input.' +
' Try again: ').lower()
if do_override_input == 'n':
sys.exit(0)
MailService(config).interactively_configure()
raw_do_sentry = ''
while raw_do_sentry not in ['y', 'n']:
raw_do_sentry = input('Do you want to configure Error Reporting via' +
' Sentry? [y/n] ').lower()
if raw_do_sentry == 'y':
sentry_dsn = input('Please enter your Sentry DSN: ')
config.set_property('sentry_dsn', sentry_dsn)
moodle = MoodleService(config, storage_path)
moodle.interactively_acquire_token()
print('Configuration finished and saved!')
if (storage_path == '.'):
print(' To set a cron-job for this program on your Unix-System:\n' +
' 1. `crontab -e`\n' +
' 2. Add `*/15 * * * * cd %s && python3 %smain.py`\n' %
(os.getcwd(),
os.path.join(os.path.dirname(os.path.realpath(__file__)), '')) +
' 3. Save and you\'re done!')
else:
print(' To set a cron-job for this program on your Unix-System:\n' +
' 1. `crontab -e`\n' + ' 2. Add `*/15 * * * *' +
' cd %s && python3 %smain.py --path %s`\n' %
(os.getcwd(),
os.path.join(os.path.dirname(os.path.realpath(__file__)), ''),
storage_path) + ' 3. Save and you\'re done!')
print('')
raw_do_config = ''
while raw_do_config not in ['y', 'n']:
raw_do_config = input(
'Do you want to make additional configurations now?' +
' You can always do the additional configuration later' +
' with the --config option. [y/n] ').lower()
if raw_do_config == 'y':
run_configure(storage_path)
print('')
print('All set and ready to go!')
def run_new_token(storage_path):
config = ConfigHelper(storage_path)
config.load() # because we do not want to override the other settings
MoodleService(config, storage_path).interactively_acquire_token()
print('New Token successfully saved!')
def run_init(storage_path, use_sso=False, skip_cert_verify=False):
config = ConfigHelper(storage_path)
if config.is_present():
do_override_input = cutie.prompt_yes_or_no(
'Do you want to override the existing config?')
if not do_override_input:
sys.exit(0)
MailService(config).interactively_configure()
do_sentry = cutie.prompt_yes_or_no(
'Do you want to configure Error Reporting via Sentry?')
if do_sentry:
sentry_dsn = input('Please enter your Sentry DSN: ')
config.set_property('sentry_dsn', sentry_dsn)
moodle = MoodleService(config, storage_path, skip_cert_verify)
if (use_sso):
moodle.interactively_acquire_sso_token()
else:
moodle.interactively_acquire_token()
print('Configuration finished and saved!')
if (storage_path == '.'):
print(
' To set a cron-job for this program on your Unix-System:\n' +
' 1. `crontab -e`\n' +
' 2. Add `*/15 * * * * cd %s && python3 %smain.py`\n' % (
os.getcwd(), os.path.join(os.path.dirname(
os.path.realpath(__file__)), '')) +
' 3. Save and you\'re done!'
)
else:
print(
' To set a cron-job for this program on your Unix-System:\n' +
' 1. `crontab -e`\n' +
' 2. Add `*/15 * * * *' +
' cd %s && python3 %smain.py --path %s`\n' % (
os.getcwd(), os.path.join(os.path.dirname(
os.path.realpath(__file__)), ''), storage_path) +
' 3. Save and you\'re done!'
)
print('')
print('You can always do the additional configuration later' +
' with the --config option.')
do_config = cutie.prompt_yes_or_no(
'Do you want to make additional configurations now?')
if do_config:
run_configure(storage_path, skip_cert_verify)
print('')
print('All set and ready to go!')
def run_new_token(storage_path, use_sso=False, skip_cert_verify=False):
config = ConfigHelper(storage_path)
config.load() # because we do not want to override the other settings
moodle = MoodleService(config, storage_path, skip_cert_verify)
if (use_sso):
moodle.interactively_acquire_sso_token()
else:
moodle.interactively_acquire_token()
print('New Token successfully saved!')
def run_main(storage_path,
skip_cert_verify=False,
without_downloading_files=False):
logging.basicConfig(
filename=os.path.join(storage_path, 'MoodleDownloader.log'),
level=logging.DEBUG,
format='%(asctime)s %(levelname)s {%(module)s} %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logging.info('--- main started ---------------------')
Log.info('Moodle Downloader starting...')
if IS_DEBUG:
logging.info(
'Debug-Mode detected. Errors will not be logged but instead' +
' re-risen.')
debug_logger = logging.getLogger()
debug_logger.setLevel(logging.ERROR)
debug_logger.addHandler(ReRaiseOnError())
try:
logging.debug('Loading config...')
Log.debug('Loading config...')
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error('Error while trying to load the Configuration! ' +
'Exiting...',
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(-1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
console_service = ConsoleService(config)
try:
moodle = MoodleService(config, storage_path, skip_cert_verify)
logging.debug(
'Checking for changes for the configured Moodle-Account....')
Log.debug('Checking for changes for the configured Moodle-Account...')
changed_courses = moodle.fetch_state()
diff_count = 0
logging.debug('Start downloading changed files...')
Log.debug('Start downloading changed files...')
if (without_downloading_files):
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path)
downloader.run()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
for course in changed_courses:
diff_count += len(course.files)
if diff_count > 0:
logging.info(
'%s changes found for the configured Moodle-Account.' %
(diff_count))
Log.success('%s changes found for the configured Moodle-Account.' %
(diff_count))
console_service.notify_about_changes_in_moodle(changed_courses)
else:
logging.info('No changes found for the configured Moodle-Account.')
Log.warning('No changes found for the configured Moodle-Account.')
if (len(changed_courses_to_notify) > 0):
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
mail_service.notify_about_error(str(e))
logging.debug('Exception-Handling completed. Exiting...',
extra={'exception': e})
Log.critical('Exception:\n%s' % (error_formatted))
Log.error('The following error occurred during execution: %s' %
(str(e)))
sys.exit(-1)
def interactively_manage_database(self):
RESET_SEQ = '\033[0m'
COLOR_SEQ = '\033[1;%dm'
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
stored_files = self.state_recorder.get_stored_files()
download_course_ids = self.config_helper.get_download_course_ids()
dont_download_course_ids = self.config_helper.get_dont_download_course_ids(
)
download_submissions = self.config_helper.get_download_submissions()
download_descriptions = self.config_helper.get_download_descriptions()
download_databases = self.config_helper.get_download_databases()
stored_files = MoodleService._filter_courses(
stored_files, download_course_ids, dont_download_course_ids,
download_submissions, download_descriptions, download_databases)
if len(stored_files) <= 0:
return
print('This management tool will navigate you through a menu to' +
' selectively remove file entries from the database so' +
' that these files can be downloaded again.')
course_options = []
courses = []
for course in stored_files:
for course_file in course.files:
if not os.path.exists(course_file.saved_to):
course_options.append(COLOR_SEQ % BLUE + course.fullname +
RESET_SEQ)
courses.append(course)
break
print('Choose one of the courses:')
print('[Confirm your selection with the Enter key]')
print('')
selected_course_id = cutie.select(options=course_options)
selected_course = courses[selected_course_id]
section_options = []
sections = []
for course_file in selected_course.files:
if not os.path.exists(course_file.saved_to) and (
course_file.section_name not in sections):
section_options.append(COLOR_SEQ % MAGENTA +
course_file.section_name + RESET_SEQ)
sections.append(course_file.section_name)
print('From which sections you want to select files.')
print(
'[You can select with the space bar and confirm your selection with the enter key]'
)
print('')
selected_sections_ids = cutie.select_multiple(options=section_options,
minimal_count=1)
selected_sections = []
for selected_sections_id in selected_sections_ids:
if selected_sections_id < len(sections):
selected_sections.append(sections[selected_sections_id])
file_options = []
files = []
for course_file in selected_course.files:
if not os.path.exists(course_file.saved_to) and (
course_file.section_name in selected_sections):
file_options.append(COLOR_SEQ % CYAN +
course_file.content_filename + RESET_SEQ)
files.append(course_file)
print(
'Which of the files should be removed form the database, so that they will be redownloaded?'
)
print(
'[You can select with the space bar and confirm your selection with the enter key]'
)
print('')
selected_files = cutie.select_multiple(options=file_options)
files_to_delete = []
for file_index in selected_files:
if file_index < len(files) and isinstance(files[file_index], File):
files_to_delete.append(files[file_index])
self.state_recorder.batch_delete_files_from_db(files_to_delete)
def __init__(self,
courses: [Course],
moodle_service: MoodleService,
storage_path: str,
skip_cert_verify: bool = False):
"""
Initiates the DownloadService with all files that
need to be downloaded. A URLTarget is created for each file.
@param courses: A list of courses that contains all modified files.
@param moodle_service: A reference to the moodle_service, currently
only to get to the state_recorder and the token.
@param storage_path: The location where the files will be saved.
"""
# How much threads should be created
DownloadService.thread_count = 5
# How often should the downloader try to download
# a file again if an error occurs.
DownloadService.url_tries = 3
self.courses = courses
self.state_recorder = moodle_service.recorder
self.token = moodle_service.get_token()
self.storage_path = storage_path
# The wait queue for all URL targets to be downloaded.
self.queue = Queue(0)
# A list of the created threads
self.threads = []
# A lock to stabilize thread insecure resources.
# writing in DB
self.lock = threading.Lock()
# reading file system
self.lock2 = threading.Lock()
# report is used to collect successful and failed downloads
self.report = {'success': [], 'failure': []}
# thread_report is used to get live reports from the threads
self.thread_report = [{
'total': 0,
'percentage': 0
} for i in range(self.thread_count)]
# Collects the total size of the files that needs to be downloaded.
self.total_to_download = 0
# delete files, that should be deleted
self.state_recorder.batch_delete_files(self.courses)
if skip_cert_verify:
self.ssl_context = ssl._create_unverified_context()
else:
self.ssl_context = ssl._create_default_https_context()
# Prepopulate queue with any files that were given
for course in self.courses:
for file in course.files:
if (file.deleted is False):
self.total_to_download += file.content_filesize
save_destination = self.genPath(self.storage_path, course,
file)
self.queue.put(
URLTarget(file, course, save_destination, self.token,
self.thread_report, self.lock2,
self.ssl_context))
def run_init(storage_path, use_sso=False, skip_cert_verify=False):
config = ConfigHelper(storage_path)
if config.is_present():
do_override_input = cutie.prompt_yes_or_no(
Log.error_str('Do you want to override the existing config?'))
if not do_override_input:
sys.exit(0)
MailService(config).interactively_configure()
TelegramService(config).interactively_configure()
do_sentry = cutie.prompt_yes_or_no(
'Do you want to configure Error Reporting via Sentry?')
if do_sentry:
sentry_dsn = input('Please enter your Sentry DSN: ')
config.set_property('sentry_dsn', sentry_dsn)
moodle = MoodleService(config, storage_path, skip_cert_verify)
if use_sso:
moodle.interactively_acquire_sso_token()
else:
moodle.interactively_acquire_token()
if os.name != 'nt':
Log.info(
'On Windows many characters are forbidden in filenames and paths, if you want, these characters can be'
+ ' automatically removed from filenames.')
Log.warning(
'If you want to view the downloaded files on Windows this is important!'
)
default_windows_map = cutie.prompt_yes_or_no(
'Do you want to load the default filename character map for windows?'
)
if default_windows_map:
config.set_default_filename_character_map(True)
else:
config.set_default_filename_character_map(False)
else:
config.set_default_filename_character_map(True)
Log.success('Configuration finished and saved!')
if os.name != 'nt':
if storage_path == '.':
Log.info(
' To set a cron-job for this program on your Unix-System:\n' +
' 1. `crontab -e`\n' +
' 2. Add `*/15 * * * * cd %s && python3 %smain.py`\n' %
(os.getcwd(),
os.path.join(os.path.dirname(os.path.realpath(__file__)), ''))
+ ' 3. Save and you\'re done!')
else:
Log.info(
' To set a cron-job for this program on your Unix-System:\n' +
' 1. `crontab -e`\n' + ' 2. Add `*/15 * * * *' +
' cd %s && python3 %smain.py --path %s`\n' %
(os.getcwd(),
os.path.join(os.path.dirname(os.path.realpath(__file__)), ''),
storage_path) + ' 3. Save and you\'re done!')
print('')
Log.info(
'You can always do the additional configuration later with the --config option.'
)
do_config = cutie.prompt_yes_or_no(
'Do you want to make additional configurations now?')
if do_config:
run_configure(storage_path, skip_cert_verify)
print('')
Log.success('All set and ready to go!')
def run_main(storage_path,
skip_cert_verify=False,
without_downloading_files=False):
log_formatter = logging.Formatter(
'%(asctime)s %(levelname)s {%(module)s} %(message)s',
'%Y-%m-%d %H:%M:%S')
log_file = os.path.join(storage_path, 'MoodleDownloader.log')
log_handler = RotatingFileHandler(log_file,
mode='a',
maxBytes=1 * 1024 * 1024,
backupCount=2,
encoding=None,
delay=0)
log_handler.setFormatter(log_formatter)
log_handler.setLevel(logging.DEBUG)
app_log = logging.getLogger()
app_log.setLevel(logging.DEBUG)
app_log.addHandler(log_handler)
logging.info('--- main started ---------------------')
Log.info('Moodle Downloader starting...')
if IS_DEBUG:
logging.info(
'Debug-Mode detected. Errors will not be logged but instead re-risen.'
)
debug_logger = logging.getLogger()
debug_logger.setLevel(logging.ERROR)
debug_logger.addHandler(ReRaiseOnError())
try:
logging.debug('Loading config...')
Log.debug('Loading config...')
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error(
'Error while trying to load the Configuration! Exiting...',
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(-1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
tg_service = TelegramService(config)
console_service = ConsoleService(config)
PathTools.filename_character_map = config.get_filename_character_map()
try:
if not IS_DEBUG:
process_lock.lock(storage_path)
moodle = MoodleService(config, storage_path, skip_cert_verify)
logging.debug(
'Checking for changes for the configured Moodle-Account....')
Log.debug('Checking for changes for the configured Moodle-Account...')
changed_courses = moodle.fetch_state()
logging.debug('Start downloading changed files...')
Log.debug('Start downloading changed files...')
if without_downloading_files:
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path,
skip_cert_verify)
downloader.run()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
if len(changed_courses_to_notify) > 0:
console_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
tg_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
else:
logging.info('No changes found for the configured Moodle-Account.')
Log.warning('No changes found for the configured Moodle-Account.')
process_lock.unlock(storage_path)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
if not isinstance(e, process_lock.LockError):
process_lock.unlock(storage_path)
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
short_error = '%s\r\n%s' % (str(e), traceback.format_exc(limit=1))
mail_service.notify_about_error(short_error)
tg_service.notify_about_error(short_error)
logging.debug('Exception-Handling completed. Exiting...',
extra={'exception': e})
Log.critical('Exception:\n%s' % (error_formatted))
Log.error('The following error occurred during execution: %s' %
(str(e)))
sys.exit(-1)
def __init__(self, courses: [Course], moodle_service: MoodleService,
storage_path: str):
"""
Initiates the DownloadService with all files that
need to be downloaded. A URLTarget is created for each file.
@param courses: A list of courses that contains all modified files.
@param moodle_service: A reference to the moodle_service, currently
only to get to the state_recorder and the token.
@param storage_path: The location where the files will be saved.
"""
# How much threads should be created
DownloadService.thread_count = 5
# How often should the downloader try to download
# a file again if an error occurs.
DownloadService.url_tries = 3
self.courses = courses
self.state_recorder = moodle_service.recorder
self.token = moodle_service.get_token()
self.storage_path = storage_path
# The wait queue for all URL targets to be downloaded.
self.queue = Queue(0)
# A list of the created threads
self.threads = []
# A lock to stabilize thread insecure resources.
# writing in DB
self.lock = threading.Lock()
# reading file system
self.lock2 = threading.Lock()
# report is used to collect successful and failed downloads
self.report = {'success': [], 'failure': []}
# thread_report is used to get live reports from the threads
self.thread_report = [{'total': 0, 'percentage': 0}
for i in range(self.thread_count)]
# Collects the total size of the files that needs to be downloaded.
self.total_to_download = 0
# delete files, that should be deleted
self.state_recorder.batch_delete_files(self.courses)
# Prepopulate queue with any files that were given
for course in self.courses:
for file in course.files:
if(file.deleted is False):
self.total_to_download += file.content_filesize
save_destination = StringTools.path_of_file(
self.storage_path, course.fullname,
file.section_name,
file.content_filepath
)
# If the file is located in a folder or in an assignment,
# it should be saved in a subfolder
# (with the name of the module).
if (file.module_modname == "assign" or
file.module_modname == "folder"):
file_path = file.content_filepath
if (file.content_type == "submission_file"):
file_path = os.path.join('/submissions/',
file_path.strip('/'))
save_destination = StringTools.path_of_file_in_module(
self.storage_path, course.fullname,
file.section_name, file.module_name,
file_path
)
self.queue.put(URLTarget(
file, course, save_destination, self.token,
self.thread_report, self.lock2))