def _select_should_download_linked_files(self): """ Asks the user if linked files should be downloaded """ download_linked_files = self.config_helper.get_download_linked_files() print('') Log.info('In Moodle courses the teacher can also link to external' + ' files. This can be audio, video, text or anything else.' + ' In particular, the teacher can link to Youtube videos.') Log.debug('To download videos correctly you have to install ffmpeg. ') Log.error('These files can increase the download volume considerably.') Log.info('If you want to filter the external links by their domain,' + ' you can manually set a whitelist and a blacklist' + ' (https://github.com/C0D3D3V/Moodle-Downloader-2/' + 'wiki/Download-(external)-linked-files' + ' for more details).') Log.warning( 'Please note that the size of the external files is determined during the download, so the total size' + ' changes during the download.') print('') download_linked_files = cutie.prompt_yes_or_no( 'Would you like to download linked files of the courses you have selected?', default_is_yes=download_linked_files, ) self.config_helper.set_property('download_linked_files', download_linked_files)
def _select_should_download_descriptions(self): """ Asks the user if descriptions should be downloaded """ download_descriptions = self.config_helper.get_download_descriptions() print('') Log.info( 'In Moodle courses, descriptions can be added to all kinds' + ' of resources, such as files, tasks, assignments or simply' + ' free text. These descriptions are usually unnecessary to' + ' download because you have already read the information or' + ' know it from context. However, there are situations where' + ' it might be interesting to download these descriptions. The' + ' descriptions are created as Markdown files and can be' + ' deleted as desired.') Log.debug( 'Creating the description files does not take extra time, but they can be annoying' + ' if they only contain unnecessary information.') print('') download_descriptions = cutie.prompt_yes_or_no( Log.special_str( 'Would you like to download descriptions of the courses you have selected?' ), default_is_yes=download_descriptions, ) self.config_helper.set_property('download_descriptions', download_descriptions)
def run_main(storage_path, skip_cert_verify=False, without_downloading_files=False): logging.basicConfig( filename=os.path.join(storage_path, 'MoodleDownloader.log'), level=logging.DEBUG, format='%(asctime)s %(levelname)s {%(module)s} %(message)s', datefmt='%Y-%m-%d %H:%M:%S') logging.info('--- main started ---------------------') Log.info('Moodle Downloader starting...') if IS_DEBUG: logging.info( 'Debug-Mode detected. Errors will not be logged but instead' + ' re-risen.') debug_logger = logging.getLogger() debug_logger.setLevel(logging.ERROR) debug_logger.addHandler(ReRaiseOnError()) try: logging.debug('Loading config...') Log.debug('Loading config...') config = ConfigHelper(storage_path) config.load() except BaseException as e: logging.error('Error while trying to load the Configuration! ' + 'Exiting...', extra={'exception': e}) Log.error('Error while trying to load the Configuration!') sys.exit(-1) r_client = False try: sentry_dsn = config.get_property('sentry_dsn') if sentry_dsn: sentry_sdk.init(sentry_dsn) except BaseException: pass mail_service = MailService(config) console_service = ConsoleService(config) try: moodle = MoodleService(config, storage_path, skip_cert_verify) logging.debug( 'Checking for changes for the configured Moodle-Account....') Log.debug('Checking for changes for the configured Moodle-Account...') changed_courses = moodle.fetch_state() diff_count = 0 logging.debug('Start downloading changed files...') Log.debug('Start downloading changed files...') if (without_downloading_files): downloader = FakeDownloadService(changed_courses, moodle, storage_path) else: downloader = DownloadService(changed_courses, moodle, storage_path) downloader.run() changed_courses_to_notify = moodle.recorder.changes_to_notify() for course in changed_courses: diff_count += len(course.files) if diff_count > 0: logging.info( '%s changes found for the configured Moodle-Account.' % (diff_count)) Log.success('%s changes found for the configured Moodle-Account.' % (diff_count)) console_service.notify_about_changes_in_moodle(changed_courses) else: logging.info('No changes found for the configured Moodle-Account.') Log.warning('No changes found for the configured Moodle-Account.') if (len(changed_courses_to_notify) > 0): mail_service.notify_about_changes_in_moodle( changed_courses_to_notify) moodle.recorder.notified(changed_courses_to_notify) logging.debug('All done. Exiting...') Log.success('All done. Exiting..') except BaseException as e: error_formatted = traceback.format_exc() logging.error(error_formatted, extra={'exception': e}) if r_client: sentry_sdk.capture_exception(e) mail_service.notify_about_error(str(e)) logging.debug('Exception-Handling completed. Exiting...', extra={'exception': e}) Log.critical('Exception:\n%s' % (error_formatted)) Log.error('The following error occurred during execution: %s' % (str(e))) sys.exit(-1)
class MisticaClient(object): def __init__(self, key, args, verbose): self.name = type(self).__name__ self.wrapper = None self.overlay = None self.qsotp = Queue() self.qdata = Queue() self.sem = Semaphore(1) self.sotp_sem = Semaphore(1) self.released = False # Overlay and Wrapper args self.overlayname = args["overlay"] self.wrappername = args["wrapper"] self.overlayargs = args["overlay_args"] self.wrapperargs = args["wrapper_args"] # Mistica Client args self.key = key # Arguments depended of overlay used self.tag = None # Arguments depended of wrapper used self.max_size = None self.poll_delay = None self.response_timeout = None self.max_retries = None # Logger parameters self.logger = Log('_client', verbose) if verbose > 0 else None self._LOGGING_ = False if self.logger is None else True def doWrapper(self): self.sem.acquire() self._LOGGING_ and self.logger.info(f"[Wrapper] Initializing...") self.wrapper = [ x for x in ClientWrapper.__subclasses__() if x.NAME == self.wrappername ][0](self.qsotp, self.wrapperargs, self.logger) self.wrapper.start() # setting sotp arguments depending on the wrapper to be used self.max_size = self.wrapper.max_size self.response_timeout = self.wrapper.response_timeout self.poll_delay = self.wrapper.poll_delay self.max_retries = self.wrapper.max_retries self.sem.release() def doSotp(self): try: self.sem.acquire() self._LOGGING_ and self.logger.info( f"[{self.name}] Initializing...") self.sem.release() self.sotp_sem.acquire() i = 1 s = ClientWorker(self.key, self.max_retries, self.max_size, self.tag, self.overlayname, self.wrappername, self.qdata, self.logger) dataThread = Thread(target=s.dataEntry, args=(self.qsotp, )) dataThread.start() while not s.exit: answers = [] self._LOGGING_ and self.logger.debug( f"[{self.name}] Iteration nº{i} Status: {s.st} Seq: {s.seqnumber}/65535" ) if s.wait_reply: timeout = self.response_timeout else: timeout = self.poll_delay try: dataEntry = self.qsotp.get(True, timeout) answers = s.Entrypoint(dataEntry) except Empty: if s.wait_reply: answers = s.lookForRetries() else: answers = s.getPollRequest() for answer in answers: self._LOGGING_ and self.logger.debug( f"[{self.name}] Header Sent: {answer.printHeader()}") if answer.receiver == self.wrapper.name: self._LOGGING_ and self.logger.debug_all( f"[{self.name}] Retries {s.retries}/{self.max_retries}" ) self.wrapper.inbox.put(answer) elif answer.receiver == self.overlay.name: self.overlay.inbox.put(answer) else: raise Exception( f"Invalid answer to {answer.receiver} in client loop" ) if self.released == False and s.sotp_first_push: self._LOGGING_ and self.logger.debug( f"[{self.name}] Initialized! Unblocked sem") self.sotp_sem.release() self.released = True i += 1 pass dataThread.join() self._LOGGING_ and self.logger.info(f"[DataThread] Terminated") self._LOGGING_ and self.logger.info(f"[{s.name}] Terminated") except Exception as e: print(e) self._LOGGING_ and self.logger.exception( f"[ClientWorker] Exception in doSotp: {e}") self.overlay.inbox.put( Message("clientworker", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) self.wrapper.inbox.put( Message("clientworker", 0, self.wrappername, 0, MessageType.SIGNAL, SignalType.TERMINATE)) def doOverlay(self): self.sem.acquire() self._LOGGING_ and self.logger.info(f"[Overlay] Initializing...") self.overlay = [ x for x in ClientOverlay.__subclasses__() if x.NAME == self.overlayname ][0](self.qsotp, self.qdata, self.overlayargs, self.logger) self.overlay.start() # setting sotp arguments depending on the overlay to be used self.tag = self.overlay.tag self.sem.release() def captureInput(self): self.sem.acquire() self._LOGGING_ and self.logger.info(f"[Input] Initializing...") self.sem.release() if system() != 'Windows': polling = poll() while True: if self.overlay.exit: break try: if system() == 'Windows': # Ugly loop for windows rawdata = stdin.buffer.raw.read(300000) sleep(0.1) else: # Nice loop for unix polling.register(stdin.buffer.raw.fileno(), POLLIN) polling.poll() rawdata = stdin.buffer.raw.read(300000) if rawdata == b'': # assume EOF self.sotp_sem.acquire() self._LOGGING_ and self.logger.debug( f"[Input] SOTP initialized, sending terminate because input recv EOF" ) self.overlay.inbox.put( Message("input", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) self.sotp_sem.release() break if rawdata and len(rawdata) > 0 and self.overlay.hasInput: self.overlay.inbox.put( Message("input", 0, self.overlayname, 0, MessageType.STREAM, rawdata)) except KeyboardInterrupt: self._LOGGING_ and self.logger.debug( "[Input] CTRL+C detected. Passing to wrapper") self.overlay.inbox.put( Message("input", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) break self._LOGGING_ and self.logger.info(f"[Input] Terminated") return def captureExit(self, signal_received, frame): self._LOGGING_ and self.logger.debug( "[Input] CTRL+C detected. Passing to overlay") self.overlay.inbox.put( Message("input", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) def run(self): try: self.doWrapper() self.doOverlay() sotpThread = Thread(target=self.doSotp) sotpThread.start() if self.overlay.hasInput: self.captureInput() else: signal(SIGINT, self.captureExit) # Crappy loop for windows if system() == 'Windows': while not self.overlay.exit: sleep(0.5) # Nice sync primitive for unix else: sotpThread.join() except Exception as e: self._LOGGING_ and self.logger.exception( f"Exception at run(): {e}") finally: self._LOGGING_ and self.logger.debug(f"[{self.name}] Terminated")
class MisticaServer(): def __init__(self, mode, key, verbose, moduleargs): # Get args and set attributes self.args = moduleargs # Logger params self.logger = Log('_server', verbose) if verbose > 0 else None self._LOGGING_ = False if self.logger is None else True # init Router self.key = key self.Router = Router(self.key, self.logger) self.Router.start() self.mode = mode self.procid = 0 if self.mode == MisticaMode.SINGLE: self.overlayname = self.args["overlay"] self.wrappername = self.args["wrapper"] # Checks if the wrap_server of a certain wrap_module is up and running. def dependencyLaunched(self, name): dname = self.getDependencyName(name) for elem in self.Router.wrapServers: if dname == elem.name: return True return False # Gets the name of the wrap_server associated to a wrap_module def getDependencyName(self, name): modulename = f'wrapper.server.wrap_module.{name}.module' try: module = import_module(modulename) except Exception: print(f"Failed to import module {name}") return "" return module.getServerDependency() # Returns a wrap_module, wrap_server or overlay module def getModuleInstance(self, type, name, args): self.procid += 1 if (type == ModuleType.WRAP_MODULE): mpath = f"wrapper.server.wrap_module.{name}.module" return import_module(mpath).getInstance(self.procid, self.Router.inbox, args, self.logger) elif (type == ModuleType.WRAP_SERVER): mpath = f"wrapper.server.wrap_server.{name}.module" return import_module(mpath).getInstance(self.procid, args, self.logger) else: mpath = f"overlay.server.{name}.module" return import_module(mpath).getInstance(self.procid, self.Router.inbox, self.mode, args, self.logger) def sigintDetect(self, signum, frame): self._LOGGING_ and self.logger.info("[Sotp] SIGINT detected") if (self.mode == MisticaMode.SINGLE): targetoverlay = self.Router.overlayModules[0] targetoverlay.inbox.put( Message('input', 0, self.Router.overlayModules[0].name, self.Router.overlayModules[0].id, MessageType.SIGNAL, SignalType.TERMINATE)) else: # TODO: Depends on who's on the foreground pass def captureInput(self, overlay): while True: if overlay.exit: break if system() != 'Windows': polling = poll() try: if system() == 'Windows': # Ugly loop for windows rawdata = stdin.buffer.raw.read(50000) sleep(0.1) else: # Nice loop for unix polling.register(stdin.buffer.raw.fileno(), POLLIN) polling.poll() rawdata = stdin.buffer.raw.read(50000) if rawdata == b'': # assume EOF self._LOGGING_ and self.logger.info( "[MísticaServer] Input is dead") self.Router.join() if rawdata and len(rawdata) > 0 and overlay.hasInput: overlay.inbox.put( Message('input', 0, 'overlay', overlay.id, MessageType.STREAM, rawdata)) except KeyboardInterrupt: self._LOGGING_ and self.logger.info( "[MísticaServer] CTRL+C detected. Passing to overlay") overlay.inbox.put( Message('input', 0, self.Router.overlayModules[0].name, self.Router.overlayModules[0].id, MessageType.SIGNAL, SignalType.TERMINATE)) break return def run(self): # If the mode is single-handler only a wrapper module and overlay module is used if self.mode == MisticaMode.SINGLE: # Launch wrap_module wmitem = self.getModuleInstance(ModuleType.WRAP_MODULE, self.wrappername, self.args["wrapper_args"]) wmitem.start() self.Router.wrapModules.append(wmitem) # Check wrap_server dependency of wrap_module and launch it wsname = self.getDependencyName(self.wrappername) if (not self.dependencyLaunched(self.wrappername)): wsitem = self.getModuleInstance(ModuleType.WRAP_SERVER, wsname, self.args["wrap_server_args"]) wsitem.start() self.Router.wrapServers.append(wsitem) else: for elem in self.Router.wrapServers: if wsname == elem.name: wsitem = elem break # add wrap_module to wrap_server list wsitem.addWrapModule(wmitem) # Launch overlay module omitem = self.getModuleInstance(ModuleType.OVERLAY, self.overlayname, self.args["overlay_args"]) omitem.start() self.Router.overlayModules.append(omitem) targetoverlay = self.Router.overlayModules[0] if targetoverlay.hasInput: self.captureInput(targetoverlay) self.Router.join() self._LOGGING_ and self.logger.debug("[MísticaServer] Terminated") elif self.mode == MisticaMode.MULTI: # Launch prompt etc. # Before registering a wrapper or an overlay, we must make sure that there is no other # module with incompatible parameters (e.g 2 DNS base64-based wrap_modules) self.Router.inbox.put( Message("Mistica", 0, "sotp", 0, MessageType.SIGNAL, SignalType.TERMINATE)) print("Multi-handler mode is not implemented yet! use -h") exit(0)
def run_main(storage_path, skip_cert_verify=False, without_downloading_files=False): log_formatter = logging.Formatter( '%(asctime)s %(levelname)s {%(module)s} %(message)s', '%Y-%m-%d %H:%M:%S') log_file = os.path.join(storage_path, 'MoodleDownloader.log') log_handler = RotatingFileHandler(log_file, mode='a', maxBytes=1 * 1024 * 1024, backupCount=2, encoding=None, delay=0) log_handler.setFormatter(log_formatter) log_handler.setLevel(logging.DEBUG) app_log = logging.getLogger() app_log.setLevel(logging.DEBUG) app_log.addHandler(log_handler) logging.info('--- main started ---------------------') Log.info('Moodle Downloader starting...') if IS_DEBUG: logging.info( 'Debug-Mode detected. Errors will not be logged but instead re-risen.' ) debug_logger = logging.getLogger() debug_logger.setLevel(logging.ERROR) debug_logger.addHandler(ReRaiseOnError()) try: logging.debug('Loading config...') Log.debug('Loading config...') config = ConfigHelper(storage_path) config.load() except BaseException as e: logging.error( 'Error while trying to load the Configuration! Exiting...', extra={'exception': e}) Log.error('Error while trying to load the Configuration!') sys.exit(-1) r_client = False try: sentry_dsn = config.get_property('sentry_dsn') if sentry_dsn: sentry_sdk.init(sentry_dsn) except BaseException: pass mail_service = MailService(config) tg_service = TelegramService(config) console_service = ConsoleService(config) PathTools.filename_character_map = config.get_filename_character_map() try: if not IS_DEBUG: process_lock.lock(storage_path) moodle = MoodleService(config, storage_path, skip_cert_verify) logging.debug( 'Checking for changes for the configured Moodle-Account....') Log.debug('Checking for changes for the configured Moodle-Account...') changed_courses = moodle.fetch_state() logging.debug('Start downloading changed files...') Log.debug('Start downloading changed files...') if without_downloading_files: downloader = FakeDownloadService(changed_courses, moodle, storage_path) else: downloader = DownloadService(changed_courses, moodle, storage_path, skip_cert_verify) downloader.run() changed_courses_to_notify = moodle.recorder.changes_to_notify() if len(changed_courses_to_notify) > 0: console_service.notify_about_changes_in_moodle( changed_courses_to_notify) mail_service.notify_about_changes_in_moodle( changed_courses_to_notify) tg_service.notify_about_changes_in_moodle( changed_courses_to_notify) moodle.recorder.notified(changed_courses_to_notify) else: logging.info('No changes found for the configured Moodle-Account.') Log.warning('No changes found for the configured Moodle-Account.') process_lock.unlock(storage_path) logging.debug('All done. Exiting...') Log.success('All done. Exiting..') except BaseException as e: if not isinstance(e, process_lock.LockError): process_lock.unlock(storage_path) error_formatted = traceback.format_exc() logging.error(error_formatted, extra={'exception': e}) if r_client: sentry_sdk.capture_exception(e) short_error = '%s\r\n%s' % (str(e), traceback.format_exc(limit=1)) mail_service.notify_about_error(short_error) tg_service.notify_about_error(short_error) logging.debug('Exception-Handling completed. Exiting...', extra={'exception': e}) Log.critical('Exception:\n%s' % (error_formatted)) Log.error('The following error occurred during execution: %s' % (str(e))) sys.exit(-1)
def _dir_path(path): if os.path.isdir(path): return path else: raise argparse.ArgumentTypeError( '"%s" is not a valid path. Make sure the directory exists.' % (str(path))) # --- called at the program invocation: ------------------------------------- IS_DEBUG = False if 'pydevd' in sys.modules: IS_DEBUG = True Log.debug('[RUNNING IN DEBUG-MODE!]') parser = argparse.ArgumentParser(description=( 'Moodle Downloader 2 helps you download all the course files of your Moodle account.' )) group = parser.add_mutually_exclusive_group() group.add_argument( '--init', action='store_true', help=('Guides you trough the configuration of the' + ' software, including the activation of' + ' mail-notifications and obtainment of a' + ' login-token for your Moodle-Account. It' + ' does not fetch the current state of your' + ' Moodle-Account.'), )
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ For building an managing filters in gmail """ from utils.gmail import GMailLabelAPI, GMailFilterAPI from utils.filter_builder import GMailFilter from utils.yaml_organizer import YamlWrapper from utils.logger import Log log = Log('main-script') log.debug('Logging initiated') # Read in the YAML file # Debug points to the example yaml file in this repo # When True: points to the example yaml file in this repo. # When False (default): takes in 1st argument in script run (i.e., sys.argv[1]) gmail_filters = YamlWrapper().gmail_filters # Load tools & API services filter_tools = GMailFilter() log.debug('Initializing APIs') label_svc = GMailLabelAPI() filter_svc = GMailFilterAPI() # Get already-existing labels all_labels = label_svc.get_all_labels(label_type='user') label_name_list = [x['name'] for x in all_labels] label_id_list = [x['id'] for x in all_labels] # Get already-existing filters all_filters = filter_svc.list_filters() # Remove all the old filters
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ For cleaning & sorting entries in a YAML file """ from utils.yaml_organizer import YamlWrapper from utils.logger import Log log = Log('filter-cleaner') log.debug('Initializing script') # Read in the YAML file # Debug # When True: points to the example yaml file in this repo. # When False (default): takes in 1st argument in script run (i.e., sys.argv[1]) YamlWrapper(debug=False).sort_and_save() log.debug('Filter cleaning complete. Ending script.')
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ For building an managing filters in gmail """ from utils.filter_builder import GMailFilter from utils.yaml_organizer import YamlWrapper from utils.xml_builder import XMLBuilder from utils.logger import Log # Read in the YAML file # Debug # When True: points to the example yaml file in this repo. # When False (default): takes in 1st argument in script run (i.e., sys.argv[1]) log = Log('main-script') log.debug('Logging initiated') gmail_filters = YamlWrapper(debug=False).gmail_filters # Load tools & API services filter_tools = GMailFilter() xml_tools = XMLBuilder(gmail_filters) # Generate the xml & save to path # (defaults to ~/Documents/gmail_filters.xml) xml_tools.generate_xml() log.debug('XML file generated. Ending script.')