def gen_path(storage_path: str, course: Course, file: File):
"""
Generates the directory path where a file should be stored
"""
course_name = course.fullname
if course.overwrite_name_with is not None:
course_name = course.overwrite_name_with
# if a flat path is requested
if not course.create_directory_structure:
return PathTools.flat_path_of_file(storage_path, course_name, file.content_filepath)
# If the file is located in a folder or in an assignment,
# it should be saved in a sub-folder
# (with the name of the module).
if file.module_modname.endswith(('assign', 'folder', 'data', 'forum')):
file_path = file.content_filepath
if file.content_type == 'submission_file':
file_path = os.path.join('/submissions/', file_path.strip('/'))
return PathTools.path_of_file_in_module(
storage_path, course_name, file.section_name, file.module_name, file_path
)
else:
return PathTools.path_of_file(storage_path, course_name, file.section_name, file.content_filepath)
def __str__(self):
message = 'File ('
message += 'module_id: %s' % (self.module_id)
message += ', section_name: "%s"' % (PathTools.to_valid_name(
self.section_name))
message += ', section_id: "%s"' % (self.section_id)
message += ', module_name: "%s"' % (PathTools.to_valid_name(
self.module_name))
message += ', content_filepath: %s' % (self.content_filepath)
message += ', content_filename: "%s"' % (PathTools.to_valid_name(
self.content_filename))
message += ', content_fileurl: "%s"' % (self.content_fileurl)
message += ', content_filesize: %s' % (self.content_filesize)
message += ', content_timemodified: %s' % (self.content_timemodified)
message += ', module_modname: %s' % (self.module_modname)
message += ', content_type: %s' % (self.content_type)
message += ', content_isexternalfile: %s' % (
self.content_isexternalfile)
message += ', saved_to: "%s"' % (self.saved_to)
message += ', time_stamp: %s' % (self.time_stamp)
message += ', modified: %s' % (self.modified)
message += ', moved: %s' % (self.moved)
message += ', deleted: %s' % (self.deleted)
message += ', notified: %s' % (self.notified)
message += ', hash: %s' % (self.hash)
message += ', file_id: %s' % (self.file_id)
message += ', old_file_id: %s' % (self.old_file_id)
message += ')'
return message
def __str__(self):
message = 'Course ('
message += 'id: %s' % (self.id)
message += ', fullname: "%s"' % (PathTools.to_valid_name(self.fullname))
message += ', overwrite_name_with: "%s"' % (PathTools.to_valid_name(self.overwrite_name_with))
message += ', create_directory_structure: %s' % (self.create_directory_structure)
message += ', files: %s' % (len(self.files))
# for i, file in enumerate(self.files):
# message += ', file[%i]: %s' % (i, file)
message += ')'
return message
def __init__(self, _id: int, fullname: str, files: [File] = None):
self.id = _id
self.fullname = PathTools.to_valid_name(fullname)
if files is not None:
self.files = files
else:
self.files = []
self.overwrite_name_with = None
self.create_directory_structure = True
self.excluded_sections = []
def __init__(self, courses: [Course], moodle_service: MoodleService,
storage_path: str):
"""
Initiates the FakeDownloadService with all files that
need to be downloaded (saved in the database).
@param courses: A list of courses that contains all modified files.
@param moodle_service: A reference to the moodle_service, currently
only to get to the state_recorder.
@param storage_path: The location where the files would be saved.
"""
self.courses = courses
self.state_recorder = moodle_service.recorder
self.storage_path = storage_path
# delete files, that should be deleted
self.state_recorder.batch_delete_files(self.courses)
# Prepopulate queue with any files that were given
for course in self.courses:
for file in course.files:
if file.deleted is False:
save_destination = DownloadService.gen_path(
self.storage_path, course, file)
filename = PathTools.to_valid_name(file.content_filename)
file.saved_to = str(Path(save_destination) / filename)
if file.content_type == 'description':
file.saved_to = str(
Path(save_destination) / (filename + '.md'))
elif file.content_type == 'html':
file.saved_to = str(
Path(save_destination) / (filename + '.html'))
elif file.module_modname.startswith('url'):
file.saved_to = str(
Path(save_destination) / (filename + '.desktop'))
if os.name == 'nt' or platform.system() == "Darwin":
file.saved_to = str(
Path(save_destination) / (filename + '.URL'))
self.state_recorder.save_file(file, course.id,
course.fullname)
def __init__(
self,
file: File,
course: Course,
destination: str,
token: str,
thread_report: [],
fs_lock: threading.Lock,
ssl_context: ssl.SSLContext,
skip_cert_verify: bool,
options: {},
):
"""
Initiating an URL target.
"""
self.file = file
self.course = course
self.destination = destination
self.token = token
self.fs_lock = fs_lock
self.ssl_context = ssl_context
self.skip_cert_verify = skip_cert_verify
self.verify_cert = not skip_cert_verify
self.options = options
# get valid filename
self.filename = PathTools.to_valid_name(self.file.content_filename)
# To return errors
self.success = False
self.error = None
# To create live reports.
self.thread_id = 0
self.thread_report = thread_report
# Total downloaded.
self.downloaded = 0
# For Youtube-dl errors
self.youtube_dl_failed_with_error = False
def _get_files_of_attempt(self, attempt_result: {}, lesson_name: str) -> []:
result = []
answerpages = attempt_result.get('answerpages', [])
# The review page should actually be generated here.
# https://github.com/moodle/moodle/blob/511a87f5fc357f18a4c53911f6e6c7f7b526246e/mod/lesson/report.php#L278-L366
# Grade is in: attempt_result.userstats.gradeinfo.earned (max points: attempt_result.userstats.gradeinfo.total)
# Take care, grade can be None
grade = attempt_result.get('userstats', {}).get('gradeinfo', {}).get('earned', None)
grade_total = attempt_result.get('userstats', {}).get('gradeinfo', {}).get('total', None)
if grade is not None and grade_total is not None:
grade_file = {
'filename': 'grade',
'filepath': '/',
'timemodified': 0,
'description': str(grade) + ' / ' + str(grade_total),
'type': 'description',
}
result.append(grade_file)
# build lesson HTML
lesson_html = moodle_html_header
attempt_filename = PathTools.to_valid_name(lesson_name)
lesson_is_empty = True
for i, answerpage in enumerate(answerpages):
page_id = answerpage.get('page', {}).get('id', 0)
lesson_id = answerpage.get('page', {}).get('lessonid', 0)
shorted_lesson_name = lesson_name
if len(shorted_lesson_name) > 17:
shorted_lesson_name = shorted_lesson_name[:15] + '..'
# print(
# '\rDownloading page of lesson [%-17s] %3d/%3d\033[K'
# % (shorted_lesson_name, i, len(answerpages) - 1),
# end='',
# )
data = {'lessonid': lesson_id, 'pageid': page_id, 'returncontents': 1}
try:
page_result = self.request_helper.post_REST('mod_lesson_get_page_data', data)
except RequestRejectedError:
continue
pagecontent = page_result.get('pagecontent', '').split('<script>')[0]
if pagecontent != '':
lesson_is_empty = False
lesson_html += pagecontent + '\n'
page_files = page_result.get('contentfiles', [])
for page_file in page_files:
file_type = page_file.get('type', '')
if file_type is None or file_type == '':
page_file.update({'type': 'lesson_file'})
result.append(page_file)
if not lesson_is_empty:
lesson_html += moodle_html_footer
attempt_file = {
'filename': attempt_filename,
'filepath': '/',
'timemodified': 0,
'html': lesson_html,
'type': 'html',
'no_search_for_urls': True,
}
result.append(attempt_file)
return result
def _get_files_of_discussions(self, latest_discussions: []) -> []:
result = []
for i, discussion in enumerate(latest_discussions):
valid_subject = PathTools.to_valid_name(
discussion.get('subject', ''))
shorted_discussion_name = valid_subject
if len(shorted_discussion_name) > 17:
shorted_discussion_name = shorted_discussion_name[:15] + '..'
discussion_id = discussion.get('discussion_id', 0)
discussion_created = discussion.get('created', 0)
print(
'\rDownloading posts of discussion [%-17s|%6s] %3d/%3d\033[K' %
(shorted_discussion_name, discussion_id, i,
len(latest_discussions) - 1),
end='',
)
data = {
'discussionid': discussion_id,
'sortby': 'modified',
'sortdirection': 'ASC',
}
posts_result = self.request_helper.post_REST(
'mod_forum_get_forum_discussion_posts', data)
posts = posts_result.get('posts', [])
for post in posts:
post_message = post.get('message', '')
post_modified = post.get('modified', 0)
post_id = post.get('id', 0)
post_parent = post.get('parent', 0)
post_userfullname = post.get('userfullname', '')
post_filename = PathTools.to_valid_name('[' + str(post_id) +
'] ' +
post_userfullname)
if post_parent != 0:
post_filename = PathTools.to_valid_name(post_filename +
' response to [' +
str(post_parent) +
']')
post_path = PathTools.to_valid_name(
datetime.utcfromtimestamp(discussion_created).strftime(
'%y-%m-%d') + ' ' + valid_subject)
post_files = post.get('messageinlinefiles', [])
post_files += post.get('attachments', [])
post_file = {
'filename': post_filename,
'filepath': post_path,
'timemodified': post_modified,
'description': post_message,
'type': 'description',
}
result.append(post_file)
for post_file in post_files:
file_type = post_file.get('type', '')
if file_type is None or file_type == '':
post_file.update({'type': 'forum_file'})
post_file.update({'filepath': post_path})
result.append(post_file)
return result
def _get_files_of_attempts(self, attempts: [], quiz_name: str) -> []:
result = []
for i, attempt in enumerate(attempts):
attempt_id = attempt.get('id', 0)
attempt_state = attempt.get('state', 'unknown')
attempt_filename = PathTools.to_valid_name(quiz_name +
' (attempt ' +
str(attempt_id) + ' ' +
attempt_state + ')')
shorted_quiz_name = quiz_name
if len(shorted_quiz_name) > 17:
shorted_quiz_name = shorted_quiz_name[:15] + '..'
# print(
# '\rDownloading attempt of quiz [%-17s|%6s] %3d/%3d\033[K'
# % (shorted_quiz_name, attempt_id, i, len(attempts) - 1),
# end='',
# )
data = {'attemptid': attempt_id}
try:
if attempt_state == 'finished':
attempt_result = self.request_helper.post_REST(
'mod_quiz_get_attempt_review', data)
elif attempt_state == 'inprogress':
attempt_result = self.request_helper.post_REST(
'mod_quiz_get_attempt_summary', data)
else:
continue
except RequestRejectedError:
continue
questions = attempt_result.get('questions', [])
# build quiz HTML
quiz_html = moodle_html_header
for question in questions:
question_html = question.get('html', '').split('<script>')[0]
if question_html is None:
question_html = ''
quiz_html += question_html + '\n'
question_files = question.get('responsefileareas', [])
for question_file in question_files:
file_type = question_file.get('type', '')
if file_type is None or file_type == '':
question_file.update({'type': 'quiz_file'})
result.append(question_file)
quiz_html += moodle_html_footer
attempt_file = {
'filename': attempt_filename,
'filepath': '/',
'timemodified': 0,
'html': quiz_html,
'type': 'html',
'no_search_for_urls': True,
}
result.append(attempt_file)
return result