def _select_should_download_linked_files(self):
"""
Asks the user if linked files should be downloaded
"""
download_linked_files = self.config_helper.get_download_linked_files()
print('')
Log.info('In Moodle courses the teacher can also link to external' +
' files. This can be audio, video, text or anything else.' +
' In particular, the teacher can link to Youtube videos.')
Log.debug('To download videos correctly you have to install ffmpeg. ')
Log.error('These files can increase the download volume considerably.')
Log.info('If you want to filter the external links by their domain,' +
' you can manually set a whitelist and a blacklist' +
' (https://github.com/C0D3D3V/Moodle-Downloader-2/' +
'wiki/Download-(external)-linked-files' +
' for more details).')
Log.warning(
'Please note that the size of the external files is determined during the download, so the total size'
+ ' changes during the download.')
print('')
download_linked_files = cutie.prompt_yes_or_no(
'Would you like to download linked files of the courses you have selected?',
default_is_yes=download_linked_files,
)
self.config_helper.set_property('download_linked_files',
download_linked_files)
def _select_should_download_descriptions(self):
"""
Asks the user if descriptions should be downloaded
"""
download_descriptions = self.config_helper.get_download_descriptions()
print('')
Log.info(
'In Moodle courses, descriptions can be added to all kinds' +
' of resources, such as files, tasks, assignments or simply' +
' free text. These descriptions are usually unnecessary to' +
' download because you have already read the information or' +
' know it from context. However, there are situations where' +
' it might be interesting to download these descriptions. The' +
' descriptions are created as Markdown files and can be' +
' deleted as desired.')
Log.debug(
'Creating the description files does not take extra time, but they can be annoying'
+ ' if they only contain unnecessary information.')
print('')
download_descriptions = cutie.prompt_yes_or_no(
Log.special_str(
'Would you like to download descriptions of the courses you have selected?'
),
default_is_yes=download_descriptions,
)
self.config_helper.set_property('download_descriptions',
download_descriptions)
def run_main(storage_path,
skip_cert_verify=False,
without_downloading_files=False):
logging.basicConfig(
filename=os.path.join(storage_path, 'MoodleDownloader.log'),
level=logging.DEBUG,
format='%(asctime)s %(levelname)s {%(module)s} %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logging.info('--- main started ---------------------')
Log.info('Moodle Downloader starting...')
if IS_DEBUG:
logging.info(
'Debug-Mode detected. Errors will not be logged but instead' +
' re-risen.')
debug_logger = logging.getLogger()
debug_logger.setLevel(logging.ERROR)
debug_logger.addHandler(ReRaiseOnError())
try:
logging.debug('Loading config...')
Log.debug('Loading config...')
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error('Error while trying to load the Configuration! ' +
'Exiting...',
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(-1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
console_service = ConsoleService(config)
try:
moodle = MoodleService(config, storage_path, skip_cert_verify)
logging.debug(
'Checking for changes for the configured Moodle-Account....')
Log.debug('Checking for changes for the configured Moodle-Account...')
changed_courses = moodle.fetch_state()
diff_count = 0
logging.debug('Start downloading changed files...')
Log.debug('Start downloading changed files...')
if (without_downloading_files):
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path)
downloader.run()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
for course in changed_courses:
diff_count += len(course.files)
if diff_count > 0:
logging.info(
'%s changes found for the configured Moodle-Account.' %
(diff_count))
Log.success('%s changes found for the configured Moodle-Account.' %
(diff_count))
console_service.notify_about_changes_in_moodle(changed_courses)
else:
logging.info('No changes found for the configured Moodle-Account.')
Log.warning('No changes found for the configured Moodle-Account.')
if (len(changed_courses_to_notify) > 0):
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
mail_service.notify_about_error(str(e))
logging.debug('Exception-Handling completed. Exiting...',
extra={'exception': e})
Log.critical('Exception:\n%s' % (error_formatted))
Log.error('The following error occurred during execution: %s' %
(str(e)))
sys.exit(-1)
class MisticaClient(object):
def __init__(self, key, args, verbose):
self.name = type(self).__name__
self.wrapper = None
self.overlay = None
self.qsotp = Queue()
self.qdata = Queue()
self.sem = Semaphore(1)
self.sotp_sem = Semaphore(1)
self.released = False
# Overlay and Wrapper args
self.overlayname = args["overlay"]
self.wrappername = args["wrapper"]
self.overlayargs = args["overlay_args"]
self.wrapperargs = args["wrapper_args"]
# Mistica Client args
self.key = key
# Arguments depended of overlay used
self.tag = None
# Arguments depended of wrapper used
self.max_size = None
self.poll_delay = None
self.response_timeout = None
self.max_retries = None
# Logger parameters
self.logger = Log('_client', verbose) if verbose > 0 else None
self._LOGGING_ = False if self.logger is None else True
def doWrapper(self):
self.sem.acquire()
self._LOGGING_ and self.logger.info(f"[Wrapper] Initializing...")
self.wrapper = [
x for x in ClientWrapper.__subclasses__()
if x.NAME == self.wrappername
][0](self.qsotp, self.wrapperargs, self.logger)
self.wrapper.start()
# setting sotp arguments depending on the wrapper to be used
self.max_size = self.wrapper.max_size
self.response_timeout = self.wrapper.response_timeout
self.poll_delay = self.wrapper.poll_delay
self.max_retries = self.wrapper.max_retries
self.sem.release()
def doSotp(self):
try:
self.sem.acquire()
self._LOGGING_ and self.logger.info(
f"[{self.name}] Initializing...")
self.sem.release()
self.sotp_sem.acquire()
i = 1
s = ClientWorker(self.key, self.max_retries, self.max_size,
self.tag, self.overlayname, self.wrappername,
self.qdata, self.logger)
dataThread = Thread(target=s.dataEntry, args=(self.qsotp, ))
dataThread.start()
while not s.exit:
answers = []
self._LOGGING_ and self.logger.debug(
f"[{self.name}] Iteration nº{i} Status: {s.st} Seq: {s.seqnumber}/65535"
)
if s.wait_reply:
timeout = self.response_timeout
else:
timeout = self.poll_delay
try:
dataEntry = self.qsotp.get(True, timeout)
answers = s.Entrypoint(dataEntry)
except Empty:
if s.wait_reply:
answers = s.lookForRetries()
else:
answers = s.getPollRequest()
for answer in answers:
self._LOGGING_ and self.logger.debug(
f"[{self.name}] Header Sent: {answer.printHeader()}")
if answer.receiver == self.wrapper.name:
self._LOGGING_ and self.logger.debug_all(
f"[{self.name}] Retries {s.retries}/{self.max_retries}"
)
self.wrapper.inbox.put(answer)
elif answer.receiver == self.overlay.name:
self.overlay.inbox.put(answer)
else:
raise Exception(
f"Invalid answer to {answer.receiver} in client loop"
)
if self.released == False and s.sotp_first_push:
self._LOGGING_ and self.logger.debug(
f"[{self.name}] Initialized! Unblocked sem")
self.sotp_sem.release()
self.released = True
i += 1
pass
dataThread.join()
self._LOGGING_ and self.logger.info(f"[DataThread] Terminated")
self._LOGGING_ and self.logger.info(f"[{s.name}] Terminated")
except Exception as e:
print(e)
self._LOGGING_ and self.logger.exception(
f"[ClientWorker] Exception in doSotp: {e}")
self.overlay.inbox.put(
Message("clientworker", 0, self.overlayname, 0,
MessageType.SIGNAL, SignalType.TERMINATE))
self.wrapper.inbox.put(
Message("clientworker", 0, self.wrappername, 0,
MessageType.SIGNAL, SignalType.TERMINATE))
def doOverlay(self):
self.sem.acquire()
self._LOGGING_ and self.logger.info(f"[Overlay] Initializing...")
self.overlay = [
x for x in ClientOverlay.__subclasses__()
if x.NAME == self.overlayname
][0](self.qsotp, self.qdata, self.overlayargs, self.logger)
self.overlay.start()
# setting sotp arguments depending on the overlay to be used
self.tag = self.overlay.tag
self.sem.release()
def captureInput(self):
self.sem.acquire()
self._LOGGING_ and self.logger.info(f"[Input] Initializing...")
self.sem.release()
if system() != 'Windows':
polling = poll()
while True:
if self.overlay.exit:
break
try:
if system() == 'Windows':
# Ugly loop for windows
rawdata = stdin.buffer.raw.read(300000)
sleep(0.1)
else:
# Nice loop for unix
polling.register(stdin.buffer.raw.fileno(), POLLIN)
polling.poll()
rawdata = stdin.buffer.raw.read(300000)
if rawdata == b'': # assume EOF
self.sotp_sem.acquire()
self._LOGGING_ and self.logger.debug(
f"[Input] SOTP initialized, sending terminate because input recv EOF"
)
self.overlay.inbox.put(
Message("input", 0, self.overlayname, 0,
MessageType.SIGNAL, SignalType.TERMINATE))
self.sotp_sem.release()
break
if rawdata and len(rawdata) > 0 and self.overlay.hasInput:
self.overlay.inbox.put(
Message("input", 0, self.overlayname, 0,
MessageType.STREAM, rawdata))
except KeyboardInterrupt:
self._LOGGING_ and self.logger.debug(
"[Input] CTRL+C detected. Passing to wrapper")
self.overlay.inbox.put(
Message("input", 0, self.overlayname, 0,
MessageType.SIGNAL, SignalType.TERMINATE))
break
self._LOGGING_ and self.logger.info(f"[Input] Terminated")
return
def captureExit(self, signal_received, frame):
self._LOGGING_ and self.logger.debug(
"[Input] CTRL+C detected. Passing to overlay")
self.overlay.inbox.put(
Message("input", 0, self.overlayname, 0, MessageType.SIGNAL,
SignalType.TERMINATE))
def run(self):
try:
self.doWrapper()
self.doOverlay()
sotpThread = Thread(target=self.doSotp)
sotpThread.start()
if self.overlay.hasInput:
self.captureInput()
else:
signal(SIGINT, self.captureExit)
# Crappy loop for windows
if system() == 'Windows':
while not self.overlay.exit:
sleep(0.5)
# Nice sync primitive for unix
else:
sotpThread.join()
except Exception as e:
self._LOGGING_ and self.logger.exception(
f"Exception at run(): {e}")
finally:
self._LOGGING_ and self.logger.debug(f"[{self.name}] Terminated")
class MisticaServer():
def __init__(self, mode, key, verbose, moduleargs):
# Get args and set attributes
self.args = moduleargs
# Logger params
self.logger = Log('_server', verbose) if verbose > 0 else None
self._LOGGING_ = False if self.logger is None else True
# init Router
self.key = key
self.Router = Router(self.key, self.logger)
self.Router.start()
self.mode = mode
self.procid = 0
if self.mode == MisticaMode.SINGLE:
self.overlayname = self.args["overlay"]
self.wrappername = self.args["wrapper"]
# Checks if the wrap_server of a certain wrap_module is up and running.
def dependencyLaunched(self, name):
dname = self.getDependencyName(name)
for elem in self.Router.wrapServers:
if dname == elem.name:
return True
return False
# Gets the name of the wrap_server associated to a wrap_module
def getDependencyName(self, name):
modulename = f'wrapper.server.wrap_module.{name}.module'
try:
module = import_module(modulename)
except Exception:
print(f"Failed to import module {name}")
return ""
return module.getServerDependency()
# Returns a wrap_module, wrap_server or overlay module
def getModuleInstance(self, type, name, args):
self.procid += 1
if (type == ModuleType.WRAP_MODULE):
mpath = f"wrapper.server.wrap_module.{name}.module"
return import_module(mpath).getInstance(self.procid,
self.Router.inbox, args,
self.logger)
elif (type == ModuleType.WRAP_SERVER):
mpath = f"wrapper.server.wrap_server.{name}.module"
return import_module(mpath).getInstance(self.procid, args,
self.logger)
else:
mpath = f"overlay.server.{name}.module"
return import_module(mpath).getInstance(self.procid,
self.Router.inbox,
self.mode, args,
self.logger)
def sigintDetect(self, signum, frame):
self._LOGGING_ and self.logger.info("[Sotp] SIGINT detected")
if (self.mode == MisticaMode.SINGLE):
targetoverlay = self.Router.overlayModules[0]
targetoverlay.inbox.put(
Message('input', 0, self.Router.overlayModules[0].name,
self.Router.overlayModules[0].id, MessageType.SIGNAL,
SignalType.TERMINATE))
else:
# TODO: Depends on who's on the foreground
pass
def captureInput(self, overlay):
while True:
if overlay.exit:
break
if system() != 'Windows':
polling = poll()
try:
if system() == 'Windows':
# Ugly loop for windows
rawdata = stdin.buffer.raw.read(50000)
sleep(0.1)
else:
# Nice loop for unix
polling.register(stdin.buffer.raw.fileno(), POLLIN)
polling.poll()
rawdata = stdin.buffer.raw.read(50000)
if rawdata == b'': # assume EOF
self._LOGGING_ and self.logger.info(
"[MísticaServer] Input is dead")
self.Router.join()
if rawdata and len(rawdata) > 0 and overlay.hasInput:
overlay.inbox.put(
Message('input', 0, 'overlay', overlay.id,
MessageType.STREAM, rawdata))
except KeyboardInterrupt:
self._LOGGING_ and self.logger.info(
"[MísticaServer] CTRL+C detected. Passing to overlay")
overlay.inbox.put(
Message('input', 0, self.Router.overlayModules[0].name,
self.Router.overlayModules[0].id,
MessageType.SIGNAL, SignalType.TERMINATE))
break
return
def run(self):
# If the mode is single-handler only a wrapper module and overlay module is used
if self.mode == MisticaMode.SINGLE:
# Launch wrap_module
wmitem = self.getModuleInstance(ModuleType.WRAP_MODULE,
self.wrappername,
self.args["wrapper_args"])
wmitem.start()
self.Router.wrapModules.append(wmitem)
# Check wrap_server dependency of wrap_module and launch it
wsname = self.getDependencyName(self.wrappername)
if (not self.dependencyLaunched(self.wrappername)):
wsitem = self.getModuleInstance(ModuleType.WRAP_SERVER, wsname,
self.args["wrap_server_args"])
wsitem.start()
self.Router.wrapServers.append(wsitem)
else:
for elem in self.Router.wrapServers:
if wsname == elem.name:
wsitem = elem
break
# add wrap_module to wrap_server list
wsitem.addWrapModule(wmitem)
# Launch overlay module
omitem = self.getModuleInstance(ModuleType.OVERLAY,
self.overlayname,
self.args["overlay_args"])
omitem.start()
self.Router.overlayModules.append(omitem)
targetoverlay = self.Router.overlayModules[0]
if targetoverlay.hasInput:
self.captureInput(targetoverlay)
self.Router.join()
self._LOGGING_ and self.logger.debug("[MísticaServer] Terminated")
elif self.mode == MisticaMode.MULTI:
# Launch prompt etc.
# Before registering a wrapper or an overlay, we must make sure that there is no other
# module with incompatible parameters (e.g 2 DNS base64-based wrap_modules)
self.Router.inbox.put(
Message("Mistica", 0, "sotp", 0, MessageType.SIGNAL,
SignalType.TERMINATE))
print("Multi-handler mode is not implemented yet! use -h")
exit(0)
def run_main(storage_path,
skip_cert_verify=False,
without_downloading_files=False):
log_formatter = logging.Formatter(
'%(asctime)s %(levelname)s {%(module)s} %(message)s',
'%Y-%m-%d %H:%M:%S')
log_file = os.path.join(storage_path, 'MoodleDownloader.log')
log_handler = RotatingFileHandler(log_file,
mode='a',
maxBytes=1 * 1024 * 1024,
backupCount=2,
encoding=None,
delay=0)
log_handler.setFormatter(log_formatter)
log_handler.setLevel(logging.DEBUG)
app_log = logging.getLogger()
app_log.setLevel(logging.DEBUG)
app_log.addHandler(log_handler)
logging.info('--- main started ---------------------')
Log.info('Moodle Downloader starting...')
if IS_DEBUG:
logging.info(
'Debug-Mode detected. Errors will not be logged but instead re-risen.'
)
debug_logger = logging.getLogger()
debug_logger.setLevel(logging.ERROR)
debug_logger.addHandler(ReRaiseOnError())
try:
logging.debug('Loading config...')
Log.debug('Loading config...')
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error(
'Error while trying to load the Configuration! Exiting...',
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(-1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
tg_service = TelegramService(config)
console_service = ConsoleService(config)
PathTools.filename_character_map = config.get_filename_character_map()
try:
if not IS_DEBUG:
process_lock.lock(storage_path)
moodle = MoodleService(config, storage_path, skip_cert_verify)
logging.debug(
'Checking for changes for the configured Moodle-Account....')
Log.debug('Checking for changes for the configured Moodle-Account...')
changed_courses = moodle.fetch_state()
logging.debug('Start downloading changed files...')
Log.debug('Start downloading changed files...')
if without_downloading_files:
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path,
skip_cert_verify)
downloader.run()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
if len(changed_courses_to_notify) > 0:
console_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
tg_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
else:
logging.info('No changes found for the configured Moodle-Account.')
Log.warning('No changes found for the configured Moodle-Account.')
process_lock.unlock(storage_path)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
if not isinstance(e, process_lock.LockError):
process_lock.unlock(storage_path)
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
short_error = '%s\r\n%s' % (str(e), traceback.format_exc(limit=1))
mail_service.notify_about_error(short_error)
tg_service.notify_about_error(short_error)
logging.debug('Exception-Handling completed. Exiting...',
extra={'exception': e})
Log.critical('Exception:\n%s' % (error_formatted))
Log.error('The following error occurred during execution: %s' %
(str(e)))
sys.exit(-1)
def _dir_path(path):
if os.path.isdir(path):
return path
else:
raise argparse.ArgumentTypeError(
'"%s" is not a valid path. Make sure the directory exists.' %
(str(path)))
# --- called at the program invocation: -------------------------------------
IS_DEBUG = False
if 'pydevd' in sys.modules:
IS_DEBUG = True
Log.debug('[RUNNING IN DEBUG-MODE!]')
parser = argparse.ArgumentParser(description=(
'Moodle Downloader 2 helps you download all the course files of your Moodle account.'
))
group = parser.add_mutually_exclusive_group()
group.add_argument(
'--init',
action='store_true',
help=('Guides you trough the configuration of the' +
' software, including the activation of' +
' mail-notifications and obtainment of a' +
' login-token for your Moodle-Account. It' +
' does not fetch the current state of your' + ' Moodle-Account.'),
)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
For building an managing filters in gmail
"""
from utils.gmail import GMailLabelAPI, GMailFilterAPI
from utils.filter_builder import GMailFilter
from utils.yaml_organizer import YamlWrapper
from utils.logger import Log
log = Log('main-script')
log.debug('Logging initiated')
# Read in the YAML file
# Debug points to the example yaml file in this repo
# When True: points to the example yaml file in this repo.
# When False (default): takes in 1st argument in script run (i.e., sys.argv[1])
gmail_filters = YamlWrapper().gmail_filters
# Load tools & API services
filter_tools = GMailFilter()
log.debug('Initializing APIs')
label_svc = GMailLabelAPI()
filter_svc = GMailFilterAPI()
# Get already-existing labels
all_labels = label_svc.get_all_labels(label_type='user')
label_name_list = [x['name'] for x in all_labels]
label_id_list = [x['id'] for x in all_labels]
# Get already-existing filters
all_filters = filter_svc.list_filters()
# Remove all the old filters
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
For cleaning & sorting entries in a YAML file
"""
from utils.yaml_organizer import YamlWrapper
from utils.logger import Log
log = Log('filter-cleaner')
log.debug('Initializing script')
# Read in the YAML file
# Debug
# When True: points to the example yaml file in this repo.
# When False (default): takes in 1st argument in script run (i.e., sys.argv[1])
YamlWrapper(debug=False).sort_and_save()
log.debug('Filter cleaning complete. Ending script.')
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
For building an managing filters in gmail
"""
from utils.filter_builder import GMailFilter
from utils.yaml_organizer import YamlWrapper
from utils.xml_builder import XMLBuilder
from utils.logger import Log
# Read in the YAML file
# Debug
# When True: points to the example yaml file in this repo.
# When False (default): takes in 1st argument in script run (i.e., sys.argv[1])
log = Log('main-script')
log.debug('Logging initiated')
gmail_filters = YamlWrapper(debug=False).gmail_filters
# Load tools & API services
filter_tools = GMailFilter()
xml_tools = XMLBuilder(gmail_filters)
# Generate the xml & save to path
# (defaults to ~/Documents/gmail_filters.xml)
xml_tools.generate_xml()
log.debug('XML file generated. Ending script.')
