def genPath(storage_path: str, course: Course, file: File):
"""
Generates the directory path where a file should be stored
"""
course_name = course.fullname
if (course.overwrite_name_with is not None):
course_name = course.overwrite_name_with
# if a flat path is requested
if (not course.create_directory_structure):
return StringTools.flat_path_of_file(storage_path, course_name,
file.content_filepath)
# If the file is located in a folder or in an assignment,
# it should be saved in a sub-folder
# (with the name of the module).
if (file.module_modname == "assign"
or file.module_modname == "folder"):
file_path = file.content_filepath
if (file.content_type == "submission_file"):
file_path = os.path.join('/submissions/', file_path.strip('/'))
return StringTools.path_of_file_in_module(storage_path,
course_name,
file.section_name,
file.module_name,
file_path)
else:
return StringTools.path_of_file(storage_path, course_name,
file.section_name,
file.content_filepath)
def __init__(self, file: File, course: Course, destination: str,
token: str, thread_report: [], lock: threading.Lock,
ssl_context: ssl.SSLContext):
"""
Initiating an URL target.
"""
self.file = file
self.course = course
self.destination = destination
self.token = token
self.lock = lock
self.ssl_context = ssl_context
# get valid filename
self.filename = StringTools.to_valid_name(self.file.content_filename)
# Counts the download attempts
self.url_tried = 0
# To return errors
self.success = False
self.error = None
# To create live reports.
self.thread_id = 0
self.thread_report = thread_report
# Total downloaded.
self.downloaded = 0
def __init__(self, courses: [Course], moodle_service: MoodleService,
storage_path: str):
"""
Initiates the FakeDownloadService with all files that
need to be downloaded (saved in the database).
@param courses: A list of courses that contains all modified files.
@param moodle_service: A reference to the moodle_service, currently
only to get to the state_recorder.
@param storage_path: The location where the files would be saved.
"""
self.courses = courses
self.state_recorder = moodle_service.recorder
self.storage_path = storage_path
# delete files, that should be deleted
self.state_recorder.batch_delete_files(self.courses)
# Prepopulate queue with any files that were given
for course in self.courses:
for file in course.files:
if (file.deleted is False):
save_destination = DownloadService.genPath(
self.storage_path, course, file)
filename = StringTools.to_valid_name(file.content_filename)
file.saved_to = str(Path(save_destination) / filename)
if (file.module_modname == 'url'):
file.saved_to = str(
Path(save_destination) / (filename + ".desktop"))
if os.name == "nt":
file.saved_to = str(
Path(save_destination) / (filename + ".URL"))
self.state_recorder.save_file(file, course.id,
course.fullname)
def __init__(self, courses: [Course], moodle_service: MoodleService,
storage_path: str):
"""
Initiates the DownloadService with all files that
need to be downloaded. A URLTarget is created for each file.
@param courses: A list of courses that contains all modified files.
@param moodle_service: A reference to the moodle_service, currently
only to get to the state_recorder and the token.
@param storage_path: The location where the files will be saved.
"""
# How much threads should be created
DownloadService.thread_count = 5
# How often should the downloader try to download
# a file again if an error occurs.
DownloadService.url_tries = 3
self.courses = courses
self.state_recorder = moodle_service.recorder
self.token = moodle_service.get_token()
self.storage_path = storage_path
# The wait queue for all URL targets to be downloaded.
self.queue = Queue(0)
# A list of the created threads
self.threads = []
# A lock to stabilize thread insecure resources.
# writing in DB
self.lock = threading.Lock()
# reading file system
self.lock2 = threading.Lock()
# report is used to collect successful and failed downloads
self.report = {'success': [], 'failure': []}
# thread_report is used to get live reports from the threads
self.thread_report = [{'total': 0, 'percentage': 0}
for i in range(self.thread_count)]
# Collects the total size of the files that needs to be downloaded.
self.total_to_download = 0
# delete files, that should be deleted
self.state_recorder.batch_delete_files(self.courses)
# Prepopulate queue with any files that were given
for course in self.courses:
for file in course.files:
if(file.deleted is False):
self.total_to_download += file.content_filesize
save_destination = StringTools.path_of_file(
self.storage_path, course.fullname,
file.section_name,
file.content_filepath
)
# If the file is located in a folder or in an assignment,
# it should be saved in a subfolder
# (with the name of the module).
if (file.module_modname == "assign" or
file.module_modname == "folder"):
file_path = file.content_filepath
if (file.content_type == "submission_file"):
file_path = os.path.join('/submissions/',
file_path.strip('/'))
save_destination = StringTools.path_of_file_in_module(
self.storage_path, course.fullname,
file.section_name, file.module_name,
file_path
)
self.queue.put(URLTarget(
file, course, save_destination, self.token,
self.thread_report, self.lock2))