def _select_should_download_linked_files(self): """ Asks the user if linked files should be downloaded """ download_linked_files = self.config_helper.get_download_linked_files() print('') Log.info('In Moodle courses the teacher can also link to external' + ' files. This can be audio, video, text or anything else.' + ' In particular, the teacher can link to Youtube videos.') Log.debug('To download videos correctly you have to install ffmpeg. ') Log.error('These files can increase the download volume considerably.') Log.info('If you want to filter the external links by their domain,' + ' you can manually set a whitelist and a blacklist' + ' (https://github.com/C0D3D3V/Moodle-Downloader-2/' + 'wiki/Download-(external)-linked-files' + ' for more details).') Log.warning( 'Please note that the size of the external files is determined during the download, so the total size' + ' changes during the download.') print('') download_linked_files = cutie.prompt_yes_or_no( 'Would you like to download linked files of the courses you have selected?', default_is_yes=download_linked_files, ) self.config_helper.set_property('download_linked_files', download_linked_files)
def _select_should_download_descriptions(self): """ Asks the user if descriptions should be downloaded """ download_descriptions = self.config_helper.get_download_descriptions() print('') Log.info( 'In Moodle courses, descriptions can be added to all kinds' + ' of resources, such as files, tasks, assignments or simply' + ' free text. These descriptions are usually unnecessary to' + ' download because you have already read the information or' + ' know it from context. However, there are situations where' + ' it might be interesting to download these descriptions. The' + ' descriptions are created as Markdown files and can be' + ' deleted as desired.') Log.debug( 'Creating the description files does not take extra time, but they can be annoying' + ' if they only contain unnecessary information.') print('') download_descriptions = cutie.prompt_yes_or_no( Log.special_str( 'Would you like to download descriptions of the courses you have selected?' ), default_is_yes=download_descriptions, ) self.config_helper.set_property('download_descriptions', download_descriptions)
class WebSocket(object): """ WebSocket服务端 """ def __init__(self): from utils.logger import Log self.logger = Log().logger self.tcp_server = None self.bind() def bind(self): try: self.tcp_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.tcp_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.tcp_server.bind(("", 5201)) self.tcp_server.listen(128) font_color = ["\033[1;36m", "\033[0m"] print(f"""{font_color[0] if sys.platform != "win32" else ""}注意事项: 1. 手机端请求IP地址为如下监听地址,请先测试是否可以访问通 2. 用手机浏览器测试访问说明1中尝试过的IP地址,如访问通代表无问题 3. 以下IP获取到的IP仅做参考,如果全部访问不通,请检查防火墙开启5201端口或使用ipconfig/ifconfig查看本地其他IP 4. 记得更改手机端的请求地址,并授权软件短信权限和验证码获取权限{font_color[1] if sys.platform != "win32" else ""} """) for idx, val in enumerate(get_inter_ip()): self.logger.info( f"监听地址{idx+1}:\thttp://{val}:5201/publish?smsCode=123456") except: self.logger.error("监听失败,请查看是否有同端口脚本") # 等待时间30 轮询时间0.5 @limit_decor(30, 0.5) def listener(self, *args, **kwargs): """ 通过 socket 监听 """ while True: try: cs, ca = self.tcp_server.accept() recv_data = cs.recv(1024) try: a = str( re.search(r'smsCode=(\d+)', str(recv_data)).group(1)) self.logger.info(f'监听到京东验证码:\t{a}') return json.dumps({"sms_code": a}) except AttributeError: self.logger.warnning(f"监听到IP: \t{ca[0]}\t访问,但未获取到短信验证码") except: return ""
def _select_courses_to_download(self, courses: [Course]): """ Asks the user for the courses that should be downloaded. @param courses: All available courses """ download_course_ids = self.config_helper.get_download_course_ids() dont_download_course_ids = self.config_helper.get_dont_download_course_ids( ) print('') Log.info( 'To avoid downloading all the Moodle courses you are enrolled in, you can select which ones you want' + ' to download here. ') print('') choices = [] defaults = [] for i, course in enumerate(courses): choices.append(('%5i\t%s' % (course.id, course.fullname))) if ResultsHandler._should_download_course( course.id, download_course_ids, dont_download_course_ids): defaults.append(i) Log.special('Which of the courses should be downloaded?') Log.info( '[You can select with the space bar and confirm your selection with the enter key]' ) print('') selected_courses = cutie.select_multiple(options=choices, ticked_indices=defaults) download_course_ids = [] for i, course in enumerate(courses): if i in selected_courses: download_course_ids.append(course.id) self.config_helper.set_property('download_course_ids', download_course_ids) self.config_helper.remove_property('dont_download_course_ids')
class EasyOCR(object): """ EasyOCR识别类,用于帮助ios设备识别投屏后的短信验证码 """ def __init__(self): from utils.logger import Log self.logger = Log().logger @staticmethod def get_code_pic(_range, name='ios_code_pic.png'): """ 获取验证码图像 :param _range: :param name: :return: """ # 确定验证码的左上角和右下角坐标 code_pic = ImageGrab.grab(_range) code_pic.save(name) return code_pic def easy_ocr(self, _range, delay_time=5): """ easy ocr识别数字 :param delay_time: ocr识别延迟时间 :param _range: 验证码截图区域坐标(左x,左y,右x,右y) :return: 识别到的数字 """ global sms_code self.get_code_pic(_range) reader = easyocr.Reader(['ch_sim', 'en']) result = reader.readtext('ios_code_pic.png') find_all = re.findall(r'\'[\d]{6}\'', str(result)) if len(find_all) != 1: find_all = re.findall(r'([\d]{6})[\u3002]', str(result)) if len(find_all) != 1: find_all = re.findall(r'(您的验证码为[\d]{6})', str(result)) # 识别结果 self.logger.info("EasyOCR识别结果:" + str(result)) if len(find_all) == 1: code = find_all[0].strip("'") if sms_code == code: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.easy_ocr(_range, delay_time) else: sms_code = code return code else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.easy_ocr(_range, delay_time)
def _select_should_download_databases(self): """ Asks the user if databases should be downloaded """ download_databases = self.config_helper.get_download_databases() print('') Log.info('In the database module of Moodle data can be stored' + ' structured with information. Often it is also' + ' possible for students to upload data there. Because' + ' the implementation of the downloader has not yet been' + ' optimized at this point, it is optional to download the' + ' databases. Currently only files are downloaded, thumbails' + ' are ignored.') print('') download_databases = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to download databases of your courses?'), default_is_yes=download_databases) self.config_helper.set_property('download_databases', download_databases)
def _select_should_download_submissions(self): """ Asks the user if submissions should be downloaded """ download_submissions = self.config_helper.get_download_submissions() print('') Log.info('Submissions are files that you or a teacher have uploaded' + ' to your assignments. Moodle does not provide an' + ' interface for downloading information from all' + ' submissions to a course at once.') Log.warning( 'Therefore, it may be slow to monitor changes to submissions.') print('') download_submissions = cutie.prompt_yes_or_no( Log.special_str( 'Do you want to download submissions of your assignments?'), default_is_yes=download_submissions, ) self.config_helper.set_property('download_submissions', download_submissions)
class EasyOCR(object): """ EasyOCR识别类,用于帮助ios设备识别投屏后的短信验证码 """ def __init__(self, debug=False): from utils.logger import Log self.logger = Log().logger self.debug = debug def easy_ocr(self, _range_, delay_time=5): """ easy ocr识别数字 :param delay_time: ocr识别延迟时间 :param _range_: 验证码截图区域坐标(左x,左y,右x,右y) :return: 识别到的数字 """ global sms_code screenshot_save(_range_) reader = easyocr.Reader(['ch_sim', 'en']) ocr_ret = str(reader.readtext(captcha_screenshot)) # debug模式打印识别内容 if self.debug: self.logger.info(str(ocr_ret)) find_all = "" for rule in matching_rules: find_all = re.findall(rule, ocr_ret) if len(find_all) >= 1: break if len(find_all) == 1: code = find_all[0].strip("'") if sms_code == code: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.easy_ocr(_range_, delay_time) else: sms_code = code return code else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.easy_ocr(_range_, delay_time)
class JDyolocaptcha(object): """ yolov4类 """ def __init__(self, _config): self.CONFIDENCE_THRESHOLD = 0.8 # 置信阈值 self.NMS_THRESHOLD = 0.01 # 非极大值抑制 from utils.logger import Log self.logger = Log().logger weights = _config['yolov4_weights'] cfg = _config['yolov4_cfg'] if os.path.exists(weights): self.net = cv2.dnn.readNet(weights, cfg) else: self.logger.warning( f"找不到权重文件,当前工作目录{os.getcwd()} 应为{os.path.dirname(os.path.dirname(__file__))} 正在尝试更换工作目录" ) os.chdir(os.path.dirname(os.path.dirname(__file__))) if os.path.exists(weights): self.logger.info('已找到权重文件') self.net = cv2.dnn.readNet(weights, cfg) else: self.logger.error( f"找不到权重文件,请检查权重文件路径是否正确{os.getcwd()}/{weights},及时进行反馈") sys.exit(1) self.model = cv2.dnn_DetectionModel(self.net) size = (_config['yolov4_net_size'], _config['yolov4_net_size']) self.model.setInputParams(size=size, scale=1 / 255, swapRB=True) def base64_conversion(self, data): """ base64转Mat :param data: :return: """ imgData = base64.b64decode(data.replace("data:image/jpg;base64,", "")) nparr = np.frombuffer(imgData, np.uint8) return cv2.imdecode(nparr, cv2.IMREAD_COLOR) def img_merge(self, cpc, pcp): """ 将两张图合成为一张大图,节省一次识别次数 :param cpc: :param pcp: :return: """ img = np.zeros((206, 275, 3), np.uint8) img[0:170, 0:275] = cpc img[170:206, 167:275] = pcp return img def get_box(self, data): """ 获取坐标 """ def key(list): return list[1][1] data.sort(key=key, reverse=True) pcp_class = data[0][0] data.pop(0) return dict(data)[pcp_class] def detect(self, cpc, pcp): """ 识别验证码并返回坐标 :param cpc: :param pcp: :return: """ try: classes, scores, boxes = self.model.detect( self.img_merge(cpc, pcp), self.CONFIDENCE_THRESHOLD, self.NMS_THRESHOLD) # 将验证码进行识别 data = list(zip([c[0] for c in classes.tolist()], boxes.tolist())) x1, y1, x2, y2 = self.get_box(data) r = (x1 * 2 + x2) // 2, (y1 * 2 + y2) // 2 return True, r except: return False, (None, None) def JDyolo(self, cpc_img_path_base64, pcp_show_picture_path_base64): return self.detect( self.base64_conversion(cpc_img_path_base64), self.base64_conversion(pcp_show_picture_path_base64))
class BaiduFanYi(object): """ 百度翻译识别类,用于帮助ios设备识别投屏后的短信验证码 """ def __init__(self, _config, debug=False): from utils.logger import Log self.logger = Log().logger self.app_id = _config["baidu_fanyi_appid"] self.app_key = _config["baidu_fanyi_appkey"] self.debug = debug if self.app_id == "" or self.app_key == "": self.logger.warning("请在config.yaml中配置baidu翻译相关配置") sys.exit(1) @staticmethod def get_file_md5(file): return md5(file).hexdigest() def baidu_fanyi(self, _range_, delay_time=5): """ 百度ocr识别数字 :param delay_time: ocr识别延迟时间 :param _range_: 验证码截图区域坐标(左x,左y,右x,右y) :ocr_return: 识别到的数字 """ global sms_code screenshot_save(_range_) img = open(captcha_screenshot, 'rb').read() salt = random.randint(32768, 65536) sign = md5( (self.app_id + self.get_file_md5(img) + str(salt) + "APICUID" + "mac" + self.app_key).encode("utf-8")).hexdigest(), payload = { "from": "zh", "to": "zh", "appid": self.app_id, "salt": salt, "sign": sign, "cuid": "APICUID", "mac": "mac" } image = { "image": (os.path.basename(captcha_screenshot), img, "multipart/form-data") } ocr_ret = requests.post(baidu_api, params=payload, files=image).json() # debug模式打印识别内容 if self.debug: self.logger.info(ocr_ret) if ocr_ret["error_code"] == "0": ocr_ret = ocr_ret["data"] if ocr_ret["sumSrc"] == "": self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_fanyi(_range_, delay_time) fan_yi_ocr_ret = str(ocr_ret["sumSrc"]) find_all = "" for rule in matching_rules: find_all = re.findall(rule, fan_yi_ocr_ret) if len(find_all) >= 1: break if len(find_all) == 1: code = find_all[0].strip("'") if sms_code == code: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_fanyi(_range_, delay_time) else: sms_code = code return code else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_fanyi(_range_, delay_time) else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_fanyi(_range_, delay_time)
class MisticaServer(): def __init__(self, mode, key, verbose, moduleargs): # Get args and set attributes self.args = moduleargs # Logger params self.logger = Log('_server', verbose) if verbose > 0 else None self._LOGGING_ = False if self.logger is None else True # init Router self.key = key self.Router = Router(self.key, self.logger) self.Router.start() self.mode = mode self.procid = 0 if self.mode == MisticaMode.SINGLE: self.overlayname = self.args["overlay"] self.wrappername = self.args["wrapper"] # Checks if the wrap_server of a certain wrap_module is up and running. def dependencyLaunched(self, name): dname = self.getDependencyName(name) for elem in self.Router.wrapServers: if dname == elem.name: return True return False # Gets the name of the wrap_server associated to a wrap_module def getDependencyName(self, name): modulename = f'wrapper.server.wrap_module.{name}.module' try: module = import_module(modulename) except Exception: print(f"Failed to import module {name}") return "" return module.getServerDependency() # Returns a wrap_module, wrap_server or overlay module def getModuleInstance(self, type, name, args): self.procid += 1 if (type == ModuleType.WRAP_MODULE): mpath = f"wrapper.server.wrap_module.{name}.module" return import_module(mpath).getInstance(self.procid, self.Router.inbox, args, self.logger) elif (type == ModuleType.WRAP_SERVER): mpath = f"wrapper.server.wrap_server.{name}.module" return import_module(mpath).getInstance(self.procid, args, self.logger) else: mpath = f"overlay.server.{name}.module" return import_module(mpath).getInstance(self.procid, self.Router.inbox, self.mode, args, self.logger) def sigintDetect(self, signum, frame): self._LOGGING_ and self.logger.info("[Sotp] SIGINT detected") if (self.mode == MisticaMode.SINGLE): targetoverlay = self.Router.overlayModules[0] targetoverlay.inbox.put( Message('input', 0, self.Router.overlayModules[0].name, self.Router.overlayModules[0].id, MessageType.SIGNAL, SignalType.TERMINATE)) else: # TODO: Depends on who's on the foreground pass def captureInput(self, overlay): while True: if overlay.exit: break if system() != 'Windows': polling = poll() try: if system() == 'Windows': # Ugly loop for windows rawdata = stdin.buffer.raw.read(50000) sleep(0.1) else: # Nice loop for unix polling.register(stdin.buffer.raw.fileno(), POLLIN) polling.poll() rawdata = stdin.buffer.raw.read(50000) if rawdata == b'': # assume EOF self._LOGGING_ and self.logger.info( "[MísticaServer] Input is dead") self.Router.join() if rawdata and len(rawdata) > 0 and overlay.hasInput: overlay.inbox.put( Message('input', 0, 'overlay', overlay.id, MessageType.STREAM, rawdata)) except KeyboardInterrupt: self._LOGGING_ and self.logger.info( "[MísticaServer] CTRL+C detected. Passing to overlay") overlay.inbox.put( Message('input', 0, self.Router.overlayModules[0].name, self.Router.overlayModules[0].id, MessageType.SIGNAL, SignalType.TERMINATE)) break return def run(self): # If the mode is single-handler only a wrapper module and overlay module is used if self.mode == MisticaMode.SINGLE: # Launch wrap_module wmitem = self.getModuleInstance(ModuleType.WRAP_MODULE, self.wrappername, self.args["wrapper_args"]) wmitem.start() self.Router.wrapModules.append(wmitem) # Check wrap_server dependency of wrap_module and launch it wsname = self.getDependencyName(self.wrappername) if (not self.dependencyLaunched(self.wrappername)): wsitem = self.getModuleInstance(ModuleType.WRAP_SERVER, wsname, self.args["wrap_server_args"]) wsitem.start() self.Router.wrapServers.append(wsitem) else: for elem in self.Router.wrapServers: if wsname == elem.name: wsitem = elem break # add wrap_module to wrap_server list wsitem.addWrapModule(wmitem) # Launch overlay module omitem = self.getModuleInstance(ModuleType.OVERLAY, self.overlayname, self.args["overlay_args"]) omitem.start() self.Router.overlayModules.append(omitem) targetoverlay = self.Router.overlayModules[0] if targetoverlay.hasInput: self.captureInput(targetoverlay) self.Router.join() self._LOGGING_ and self.logger.debug("[MísticaServer] Terminated") elif self.mode == MisticaMode.MULTI: # Launch prompt etc. # Before registering a wrapper or an overlay, we must make sure that there is no other # module with incompatible parameters (e.g 2 DNS base64-based wrap_modules) self.Router.inbox.put( Message("Mistica", 0, "sotp", 0, MessageType.SIGNAL, SignalType.TERMINATE)) print("Multi-handler mode is not implemented yet! use -h") exit(0)
class BaiduOCR(object): """ 百度ocr识别类,用于帮助ios设备识别投屏后的短信验证码 """ def __init__(self, _config, debug=False): from utils.logger import Log self.logger = Log().logger app_id = _config["baidu_app_id"] api_key = _config["baidu_api_key"] secret_key = _config["baidu_secret_key"] self.debug = debug if app_id == "" or api_key == "" or secret_key == "": self.logger.warning("请在config.yaml中配置baidu ocr相关配置") sys.exit(1) self.client = AipOcr(app_id, api_key, secret_key) def baidu_ocr(self, _range_, delay_time=5): """ 百度ocr识别数字 :param delay_time: ocr识别延迟时间 :param _range_: 验证码截图区域坐标(左x,左y,右x,右y) :return: 识别到的数字 """ global sms_code screenshot_save(_range_) img = open(captcha_screenshot, 'rb').read() ocr_ret = self.client.basicGeneral(img) # debug模式打印识别内容 if self.debug: self.logger.info(ocr_ret) if "words_result" in ocr_ret: if len(ocr_ret["words_result"]) == 0: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_ocr(_range_, delay_time) ocr_ret = str(ocr_ret["words_result"]) find_all = "" for rule in matching_rules: find_all = re.findall(rule, ocr_ret) if len(find_all) >= 1: break if len(find_all) >= 1: code = find_all[0].strip("'") if sms_code == code: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_ocr(_range_, delay_time) else: sms_code = code return code else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_ocr(_range_, delay_time) else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_ocr(_range_, delay_time)
def run_init(storage_path, use_sso=False, skip_cert_verify=False): config = ConfigHelper(storage_path) if config.is_present(): do_override_input = cutie.prompt_yes_or_no( Log.error_str('Do you want to override the existing config?')) if not do_override_input: sys.exit(0) MailService(config).interactively_configure() TelegramService(config).interactively_configure() do_sentry = cutie.prompt_yes_or_no( 'Do you want to configure Error Reporting via Sentry?') if do_sentry: sentry_dsn = input('Please enter your Sentry DSN: ') config.set_property('sentry_dsn', sentry_dsn) moodle = MoodleService(config, storage_path, skip_cert_verify) if use_sso: moodle.interactively_acquire_sso_token() else: moodle.interactively_acquire_token() if os.name != 'nt': Log.info( 'On Windows many characters are forbidden in filenames and paths, if you want, these characters can be' + ' automatically removed from filenames.') Log.warning( 'If you want to view the downloaded files on Windows this is important!' ) default_windows_map = cutie.prompt_yes_or_no( 'Do you want to load the default filename character map for windows?' ) if default_windows_map: config.set_default_filename_character_map(True) else: config.set_default_filename_character_map(False) else: config.set_default_filename_character_map(True) Log.success('Configuration finished and saved!') if os.name != 'nt': if storage_path == '.': Log.info( ' To set a cron-job for this program on your Unix-System:\n' + ' 1. `crontab -e`\n' + ' 2. Add `*/15 * * * * cd %s && python3 %smain.py`\n' % (os.getcwd(), os.path.join(os.path.dirname(os.path.realpath(__file__)), '')) + ' 3. Save and you\'re done!') else: Log.info( ' To set a cron-job for this program on your Unix-System:\n' + ' 1. `crontab -e`\n' + ' 2. Add `*/15 * * * *' + ' cd %s && python3 %smain.py --path %s`\n' % (os.getcwd(), os.path.join(os.path.dirname(os.path.realpath(__file__)), ''), storage_path) + ' 3. Save and you\'re done!') print('') Log.info( 'You can always do the additional configuration later with the --config option.' ) do_config = cutie.prompt_yes_or_no( 'Do you want to make additional configurations now?') if do_config: run_configure(storage_path, skip_cert_verify) print('') Log.success('All set and ready to go!')
class BaiduFanYi(object): """ 百度翻译识别类,用于帮助ios设备识别投屏后的短信验证码 """ def __init__(self, _config, debug=False): from utils.logger import Log self.logger = Log().logger self.app_id = _config["baidu_fanyi_appid"] self.app_key = _config["baidu_fanyi_appkey"] self.debug = debug if self.app_id == "" or self.app_key == "": self.logger.warning("请在config.yaml中配置baidu翻译相关配置") sys.exit(1) @staticmethod def get_code_pic(_range, name='ios_code_pic.png'): """ 获取验证码图像 :param _range: :param name: :return: """ # 确定验证码的左上角和右下角坐标 code_pic = ImageGrab.grab(_range) code_pic.save(name) return code_pic @staticmethod def get_file_md5(file): return md5(file).hexdigest() def baidu_fanyi(self, _range, delay_time=5): """ 百度ocr识别数字 :param delay_time: ocr识别延迟时间 :param _range: 验证码截图区域坐标(左x,左y,右x,右y) :return: 识别到的数字 """ global sms_code self.get_code_pic(_range) img = open('ios_code_pic.png', 'rb').read() salt = random.randint(32768, 65536) sign = md5( (self.app_id + self.get_file_md5(img) + str(salt) + "APICUID" + "mac" + self.app_key).encode("utf-8")).hexdigest(), payload = { "from": "zh", "to": "zh", "appid": self.app_id, "salt": salt, "sign": sign, "cuid": "APICUID", "mac": "mac" } image = { "image": (os.path.basename("ios_code_pic.png"), img, "multipart/form-data") } ret = requests.post(baidu_api, params=payload, files=image).json() # debug模式打印识别内容 if self.debug: self.logger.info(ret) if ret["error_code"] == "0": ret = ret["data"] if ret["sumSrc"] == "": self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_fanyi(_range, delay_time) fan_yi_ret = str(ret["sumSrc"]) find_all = re.findall(r'\'[\d]{6}\'', fan_yi_ret) if len(find_all) != 1: find_all = re.findall(r'([\d]{6})[\u3002]', fan_yi_ret) if len(find_all) != 1: find_all = re.findall(r'(您的验证码为[\d]{6})', fan_yi_ret) if len(find_all) == 1: code = find_all[0].strip("'") if sms_code == code: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_fanyi(_range, delay_time) else: sms_code = code return code else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_fanyi(_range, delay_time) else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_fanyi(_range, delay_time)
class MisticaClient(object): def __init__(self, key, args, verbose): self.name = type(self).__name__ self.wrapper = None self.overlay = None self.qsotp = Queue() self.qdata = Queue() self.sem = Semaphore(1) self.sotp_sem = Semaphore(1) self.released = False # Overlay and Wrapper args self.overlayname = args["overlay"] self.wrappername = args["wrapper"] self.overlayargs = args["overlay_args"] self.wrapperargs = args["wrapper_args"] # Mistica Client args self.key = key # Arguments depended of overlay used self.tag = None # Arguments depended of wrapper used self.max_size = None self.poll_delay = None self.response_timeout = None self.max_retries = None # Logger parameters self.logger = Log('_client', verbose) if verbose > 0 else None self._LOGGING_ = False if self.logger is None else True def doWrapper(self): self.sem.acquire() self._LOGGING_ and self.logger.info(f"[Wrapper] Initializing...") self.wrapper = [ x for x in ClientWrapper.__subclasses__() if x.NAME == self.wrappername ][0](self.qsotp, self.wrapperargs, self.logger) self.wrapper.start() # setting sotp arguments depending on the wrapper to be used self.max_size = self.wrapper.max_size self.response_timeout = self.wrapper.response_timeout self.poll_delay = self.wrapper.poll_delay self.max_retries = self.wrapper.max_retries self.sem.release() def doSotp(self): try: self.sem.acquire() self._LOGGING_ and self.logger.info( f"[{self.name}] Initializing...") self.sem.release() self.sotp_sem.acquire() i = 1 s = ClientWorker(self.key, self.max_retries, self.max_size, self.tag, self.overlayname, self.wrappername, self.qdata, self.logger) dataThread = Thread(target=s.dataEntry, args=(self.qsotp, )) dataThread.start() while not s.exit: answers = [] self._LOGGING_ and self.logger.debug( f"[{self.name}] Iteration nº{i} Status: {s.st} Seq: {s.seqnumber}/65535" ) if s.wait_reply: timeout = self.response_timeout else: timeout = self.poll_delay try: dataEntry = self.qsotp.get(True, timeout) answers = s.Entrypoint(dataEntry) except Empty: if s.wait_reply: answers = s.lookForRetries() else: answers = s.getPollRequest() for answer in answers: self._LOGGING_ and self.logger.debug( f"[{self.name}] Header Sent: {answer.printHeader()}") if answer.receiver == self.wrapper.name: self._LOGGING_ and self.logger.debug_all( f"[{self.name}] Retries {s.retries}/{self.max_retries}" ) self.wrapper.inbox.put(answer) elif answer.receiver == self.overlay.name: self.overlay.inbox.put(answer) else: raise Exception( f"Invalid answer to {answer.receiver} in client loop" ) if self.released == False and s.sotp_first_push: self._LOGGING_ and self.logger.debug( f"[{self.name}] Initialized! Unblocked sem") self.sotp_sem.release() self.released = True i += 1 pass dataThread.join() self._LOGGING_ and self.logger.info(f"[DataThread] Terminated") self._LOGGING_ and self.logger.info(f"[{s.name}] Terminated") except Exception as e: print(e) self._LOGGING_ and self.logger.exception( f"[ClientWorker] Exception in doSotp: {e}") self.overlay.inbox.put( Message("clientworker", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) self.wrapper.inbox.put( Message("clientworker", 0, self.wrappername, 0, MessageType.SIGNAL, SignalType.TERMINATE)) def doOverlay(self): self.sem.acquire() self._LOGGING_ and self.logger.info(f"[Overlay] Initializing...") self.overlay = [ x for x in ClientOverlay.__subclasses__() if x.NAME == self.overlayname ][0](self.qsotp, self.qdata, self.overlayargs, self.logger) self.overlay.start() # setting sotp arguments depending on the overlay to be used self.tag = self.overlay.tag self.sem.release() def captureInput(self): self.sem.acquire() self._LOGGING_ and self.logger.info(f"[Input] Initializing...") self.sem.release() if system() != 'Windows': polling = poll() while True: if self.overlay.exit: break try: if system() == 'Windows': # Ugly loop for windows rawdata = stdin.buffer.raw.read(300000) sleep(0.1) else: # Nice loop for unix polling.register(stdin.buffer.raw.fileno(), POLLIN) polling.poll() rawdata = stdin.buffer.raw.read(300000) if rawdata == b'': # assume EOF self.sotp_sem.acquire() self._LOGGING_ and self.logger.debug( f"[Input] SOTP initialized, sending terminate because input recv EOF" ) self.overlay.inbox.put( Message("input", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) self.sotp_sem.release() break if rawdata and len(rawdata) > 0 and self.overlay.hasInput: self.overlay.inbox.put( Message("input", 0, self.overlayname, 0, MessageType.STREAM, rawdata)) except KeyboardInterrupt: self._LOGGING_ and self.logger.debug( "[Input] CTRL+C detected. Passing to wrapper") self.overlay.inbox.put( Message("input", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) break self._LOGGING_ and self.logger.info(f"[Input] Terminated") return def captureExit(self, signal_received, frame): self._LOGGING_ and self.logger.debug( "[Input] CTRL+C detected. Passing to overlay") self.overlay.inbox.put( Message("input", 0, self.overlayname, 0, MessageType.SIGNAL, SignalType.TERMINATE)) def run(self): try: self.doWrapper() self.doOverlay() sotpThread = Thread(target=self.doSotp) sotpThread.start() if self.overlay.hasInput: self.captureInput() else: signal(SIGINT, self.captureExit) # Crappy loop for windows if system() == 'Windows': while not self.overlay.exit: sleep(0.5) # Nice sync primitive for unix else: sotpThread.join() except Exception as e: self._LOGGING_ and self.logger.exception( f"Exception at run(): {e}") finally: self._LOGGING_ and self.logger.debug(f"[{self.name}] Terminated")
class AliYunOCR(object): """ 阿里云OCR识别类,用于帮助ios设备识别投屏后的短信验证码 """ def __init__(self, _config, debug=False): from utils.logger import Log self.logger = Log().logger self.api_url = "https://ocrapi-advanced.taobao.com/ocrservice/advanced" self.debug = debug appcode = _config["aliyun_appcode"] if appcode == "": self.logger.warning("请在config.yaml中配置aliyun_appcode") sys.exit(1) self.appcode = appcode def post_url(self, img): headers = { "Authorization": "APPCODE %s" % self.appcode, "Content-Type": "application/json; charset=UTF-8" } img_base64 = base64.b64encode(img).decode() payload = {"img": img_base64} resp = requests.request("POST", url=self.api_url, headers=headers, data=json.dumps(payload)) if resp.status_code != 200: self.logger.warning("阿里云OCR请求错误,错误原因:" + resp.text) self.logger.warning("阿里云OCR请求错误,大概率是次数用光,如非次数用光请将该问题反馈给开发者,程序即将退出") sys.exit(1) else: ocr_ret = json.loads(resp.text)["content"].strip(" ") return ocr_ret def aliyun_ocr(self, _range_, delay_time=5): """ 阿里云OCR识别数字 :param _range_: 验证码截图区域坐标(左x,左y,右x,右y) :param delay_time: ocr识别延迟时间 :return: 识别到的数字 """ global sms_code screenshot_save(_range_) img = open(captcha_screenshot, 'rb').read() ocr_ret = self.post_url(img) # debug模式打印识别内容 if self.debug: self.logger.info(ocr_ret) if ocr_ret != "": find_all = "" for rule in matching_rules: find_all = re.findall(rule, ocr_ret) if len(find_all) >= 1: break if len(find_all) == 1: code = find_all[0].strip("'") if sms_code == code: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.aliyun_ocr(_range_, delay_time) else: sms_code = code return code else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.aliyun_ocr(_range_, delay_time) else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.aliyun_ocr(_range_, delay_time)
class BaiduOCR(object): """ 百度ocr识别类,用于帮助ios设备识别投屏后的短信验证码 """ def __init__(self, _config): from utils.logger import Log self.logger = Log().logger app_id = _config["baidu_app_id"] api_key = _config["baidu_api_key"] secret_key = _config["baidu_secret_key"] if app_id == "" or api_key == "" or secret_key == "": self.logger.warning("请在config.yaml中配置baidu ocr相关配置") sys.exit(1) self.client = AipOcr(app_id, api_key, secret_key) @staticmethod def get_code_pic(_range, name='ios_code_pic.png'): """ 获取验证码图像 :param _range: :param name: :return: """ # 确定验证码的左上角和右下角坐标 code_pic = ImageGrab.grab(_range) code_pic.save(name) return code_pic def baidu_ocr(self, _range, delay_time=5): """ 百度ocr识别数字 :param delay_time: ocr识别延迟时间 :param _range: 验证码截图区域坐标(左x,左y,右x,右y) :return: 识别到的数字 """ global sms_code self.get_code_pic(_range) img = open('ios_code_pic.png', 'rb').read() ret = self.client.basicGeneral(img) # 加这个是为了很多人不知道OCR为啥识别不到,如果介意请注释 self.logger.info(ret) if "words_result" in ret: if len(ret["words_result"]) == 0: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_ocr(_range, delay_time) ocr_ret = str(ret["words_result"]) find_all = re.findall(r'\'[\d]{6}\'', ocr_ret) if len(find_all) != 1: find_all = re.findall(r'([\d]{6})[\u3002]', ocr_ret) if len(find_all) != 1: find_all = re.findall(r'(您的验证码为[\d]{6})', ocr_ret) if len(find_all) == 1: code = find_all[0].strip("'") if sms_code == code: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_ocr(_range, delay_time) else: sms_code = code return code else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_ocr(_range, delay_time) else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.baidu_ocr(_range, delay_time)
post_response_dic["headers"] = post_response.headers logger.info(post_response_dic) return post_response_dic else: post_response_fail["status_code"] = post_response.status_code post_response_fail["text"] = post_response.text logger.info(post_response_fail) return post_response_fail # 如果需要处理token def requestToken(self, tokenStrDic): for tokenStr in tokenStrDic.keys(): self.header_app # 判断需要使用的请求类型 def requestEstimate(self, methodRequest, urlRequest, dataRequest=None): if methodRequest == "post" or methodRequest == "Post" or methodRequest == "POST": resultRequst = self.post_request(url_post=urlRequest, post_data=dataRequest) elif methodRequest == "get" or methodRequest == "Get" or methodRequest == "GET": resultRequst = self.get_request(url_get=urlRequest, pydata=dataRequest) return resultRequst if __name__ == "__main__": ac = ApiClassification() logger.info(ac.header_post) result = ac.post_request(url_post=ac.url_post, post_data=ac.data) print(result.text)
class AliYunOCR(object): """ 阿里云OCR识别类,用于帮助ios设备识别投屏后的短信验证码 """ def __init__(self, _config, debug=False): from utils.logger import Log self.logger = Log().logger self.api_url = "https://ocrapi-advanced.taobao.com/ocrservice/advanced" self.debug = debug appcode = _config["aliyun_appcode"] if appcode == "": self.logger.warning("请在config.yaml中配置aliyun_appcode") sys.exit(1) self.appcode = appcode @staticmethod def get_code_pic(_range, name='ios_code_pic.png'): """ 获取验证码图像 :param _range: :param name: :return: """ # 确定验证码的左上角和右下角坐标 code_pic = ImageGrab.grab(_range) code_pic.save(name) return code_pic def post_url(self, img): headers = { "Authorization": "APPCODE %s" % self.appcode, "Content-Type": "application/json; charset=UTF-8" } img_base64 = base64.b64encode(img).decode() payload = {"img": img_base64} resp = requests.request("POST", url=self.api_url, headers=headers, data=json.dumps(payload)) if resp.status_code != 200: self.logger.warning("阿里云OCR请求错误,错误原因:" + resp.text) self.logger.warning("阿里云OCR请求错误,大概率是次数用光,如非次数用光请将该问题反馈给开发者,程序即将退出") sys.exit(1) else: ocr_ret = json.loads(resp.text)["content"].strip(" ") return ocr_ret def aliyun_ocr(self, _range, delay_time=5): """ 阿里云OCR识别数字 :param _range: 验证码截图区域坐标(左x,左y,右x,右y) :param delay_time: ocr识别延迟时间 :return: 识别到的数字 """ global sms_code self.get_code_pic(_range) img = open('ios_code_pic.png', 'rb').read() ocr_ret = self.post_url(img) # debug模式打印识别内容 if self.debug: self.logger.info(ocr_ret) if ocr_ret != "": find_all = re.findall(r'\'[\d]{6}\'', ocr_ret) if len(find_all) != 1: find_all = re.findall(r'([\d]{6})[\u3002]', ocr_ret) if len(find_all) != 1: find_all = re.findall(r'(您的验证码为[\d]{6})', ocr_ret) if len(find_all) != 1 and len(ocr_ret) == 6: find_all = re.findall(r'[\d]{6}', ocr_ret) if len(find_all) == 1: code = find_all[0].strip("'") if sms_code == code: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.aliyun_ocr(_range, delay_time) else: sms_code = code return code else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.aliyun_ocr(_range, delay_time) else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.aliyun_ocr(_range, delay_time)
def _set_options_of_courses(self, courses: [Course]): """ Let the user set special options for every single course """ download_course_ids = self.config_helper.get_download_course_ids() dont_download_course_ids = self.config_helper.get_dont_download_course_ids( ) print('') Log.info( 'You can set special settings for every single course.\n' + ' You can set these options:\n' + '- A different name for the course\n' + '- If a directory structure should be created for the course' + ' [create_directory_structure (cfs)].') print('') while True: choices = [] choices_courses = [] options_of_courses = self.config_helper.get_options_of_courses() choices.append('None') for course in courses: if ResultsHandler._should_download_course( course.id, download_course_ids, dont_download_course_ids): current_course_settings = options_of_courses.get( str(course.id), None) # create default settings if current_course_settings is None: current_course_settings = { 'original_name': course.fullname, 'overwrite_name_with': None, 'create_directory_structure': True, } # create list of options overwrite_name_with = current_course_settings.get( 'overwrite_name_with', None) create_directory_structure = current_course_settings.get( 'create_directory_structure', True) if overwrite_name_with is not None and overwrite_name_with != course.fullname: choices.append( ('%5i\t%s (%s) cfs=%s' % (course.id, overwrite_name_with, course.fullname, create_directory_structure))) else: choices.append( ('%5i\t%s cfs=%s' % (course.id, course.fullname, create_directory_structure))) choices_courses.append(course) print('') Log.special( 'For which of the following course do you want to change the settings?' ) print('[Confirm your selection with the Enter key]') print('') selected_course = cutie.select(options=choices) if selected_course == 0: break else: self._change_settings_of(choices_courses[selected_course - 1], options_of_courses)
class Animotes: def __init__(self, bot): self.bot = bot self.channel_logger = ChannelLogger(bot) self.log = Log(__name__) async def on_message(self, message): if not message.author.bot and sql.is_user_animote_user(message.author.id): channel = message.channel content = emote_corrector(self, message) if content: await message.delete() await channel.send(content=content) @commands.command(aliases=['unregister']) async def register(self, ctx): if not sql.is_user_animote_user(ctx.author.id): sql.add_animote_user(ctx.author.id) message = getlang(ctx.guild.id, "animotes.member_opt_in") else: sql.delete_animote_user(ctx.author.id) message = getlang(ctx.guild.id, "animotes.member_opt_out") await ctx.message.author.send(content=message) @commands.command(aliases=['unregisterserver']) @commands.guild_only() async def registerguild(self, ctx): if not ctx.author.permissions_in(ctx.channel).manage_emojis: raise NoPermission if not sql.is_server_emojishare_server(ctx.guild.id): sql.update_guild(ctx.guild.id, emojishare=1) self.log.info("Guild {0.name} (ID: {0.id}) has opted in to emoji sharing.".format(ctx.guild)) await self.channel_logger.log_to_channel("Guild **{0.name}** (ID: `{0.id}`) has opted in to emoji sharing." .format(ctx.guild)) message = getlang(ctx.guild.id, "animotes.guild_opt_in") else: sql.update_guild(ctx.guild.id, emojishare=0) self.log.info("Guild {0.name} (ID: {0.id}) has opted out of emoji sharing.".format(ctx.guild)) await self.channel_logger.log_to_channel("Guild **{0.name}** (ID: `{0.id}`) has opted out of emoji sharing." .format(ctx.guild)) message = getlang(ctx.guild.id, "animotes.guild_opt_out") await ctx.send(message) @commands.command() async def listemotes(self, ctx): guilds = [] blacklist = [] whitelist = [] opted_in = sql.is_server_emojishare_server(ctx.guild.id) whitelist.append(ctx.guild.id) # Emoji from this server are allowed automatically for e in self.bot.emojis: if e.animated: # No emoji from blacklisted servers if e.guild_id in blacklist: continue # Do not list cross-server emoji if this server has not opted in if e.guild_id != ctx.guild.id and not opted_in: continue # Blacklist servers that have not themselves opted in if not (e.guild_id in whitelist or sql.is_server_emojishare_server(e.guild_id)): blacklist.append(e.guild_id) continue # If passed all checks, ensure this server is whitelisted so we can skip future opt-in checks if e.guild_id not in whitelist: whitelist.append(e.guild_id) if not any(x['id'] for x in guilds if x['id'] == e.guild_id): guild = next(x for x in self.bot.guilds if x.id == e.guild_id) guilds.append({'id': e.guild_id, 'name': guild.name, 'animojis': []}) pos = next(i for i, x in enumerate(guilds) if x['id'] == e.guild_id) guilds[pos]['animojis'].append(str(e)) for g in guilds: msg = "**{0}**:".format(g['name']) msg += "\n" for e in g['animojis']: msg += e await ctx.send(msg)
class CnOCR(object): """ CnOCR识别类,用于帮助ios设备识别投屏后的短信验证码 """ def __init__(self): from utils.logger import Log self.logger = Log().logger @staticmethod def get_code_pic(_range, name='ios_code_pic.png'): """ 获取验证码图像 :param _range: :param name: :return: """ # 确定验证码的左上角和右下角坐标 code_pic = ImageGrab.grab(_range) code_pic.save(name) return code_pic def cn_ocr(self, _range, delay_time=5): """ cn ocr识别数字 :param delay_time: ocr识别延迟时间 :param _range: 验证码截图区域坐标(左x,左y,右x,右y) :return: 识别到的数字 """ global sms_code self.get_code_pic(_range) cn_ocr = CnOcr(model_name="conv-lite-fc", context="cpu", root="conv-lite-fc") ret = cn_ocr.ocr("ios_code_pic.png") result = "" for v in ret: result += "".join(v) find_all = re.findall(r'\'[\d]{6}\'', str(result)) if len(find_all) != 1: find_all = re.findall(r'([\d]{6})[\u3002]', str(result)) if len(find_all) != 1: find_all = re.findall(r'(您的验证码为[\d]{6})', str(result)) # 识别结果 self.logger.info("CnOCR识别结果:" + result) if len(find_all) == 1: code = find_all[0].strip("'") if sms_code == code: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.cn_ocr(_range, delay_time) else: sms_code = code return code else: self.logger.info("暂未获取到最新验证码,%d秒后重试" % delay_time) time.sleep(delay_time) return self.cn_ocr(_range, delay_time)
def run_main(storage_path, skip_cert_verify=False, without_downloading_files=False): logging.basicConfig( filename=os.path.join(storage_path, 'MoodleDownloader.log'), level=logging.DEBUG, format='%(asctime)s %(levelname)s {%(module)s} %(message)s', datefmt='%Y-%m-%d %H:%M:%S') logging.info('--- main started ---------------------') Log.info('Moodle Downloader starting...') if IS_DEBUG: logging.info( 'Debug-Mode detected. Errors will not be logged but instead' + ' re-risen.') debug_logger = logging.getLogger() debug_logger.setLevel(logging.ERROR) debug_logger.addHandler(ReRaiseOnError()) try: logging.debug('Loading config...') Log.debug('Loading config...') config = ConfigHelper(storage_path) config.load() except BaseException as e: logging.error('Error while trying to load the Configuration! ' + 'Exiting...', extra={'exception': e}) Log.error('Error while trying to load the Configuration!') sys.exit(-1) r_client = False try: sentry_dsn = config.get_property('sentry_dsn') if sentry_dsn: sentry_sdk.init(sentry_dsn) except BaseException: pass mail_service = MailService(config) console_service = ConsoleService(config) try: moodle = MoodleService(config, storage_path, skip_cert_verify) logging.debug( 'Checking for changes for the configured Moodle-Account....') Log.debug('Checking for changes for the configured Moodle-Account...') changed_courses = moodle.fetch_state() diff_count = 0 logging.debug('Start downloading changed files...') Log.debug('Start downloading changed files...') if (without_downloading_files): downloader = FakeDownloadService(changed_courses, moodle, storage_path) else: downloader = DownloadService(changed_courses, moodle, storage_path) downloader.run() changed_courses_to_notify = moodle.recorder.changes_to_notify() for course in changed_courses: diff_count += len(course.files) if diff_count > 0: logging.info( '%s changes found for the configured Moodle-Account.' % (diff_count)) Log.success('%s changes found for the configured Moodle-Account.' % (diff_count)) console_service.notify_about_changes_in_moodle(changed_courses) else: logging.info('No changes found for the configured Moodle-Account.') Log.warning('No changes found for the configured Moodle-Account.') if (len(changed_courses_to_notify) > 0): mail_service.notify_about_changes_in_moodle( changed_courses_to_notify) moodle.recorder.notified(changed_courses_to_notify) logging.debug('All done. Exiting...') Log.success('All done. Exiting..') except BaseException as e: error_formatted = traceback.format_exc() logging.error(error_formatted, extra={'exception': e}) if r_client: sentry_sdk.capture_exception(e) mail_service.notify_about_error(str(e)) logging.debug('Exception-Handling completed. Exiting...', extra={'exception': e}) Log.critical('Exception:\n%s' % (error_formatted)) Log.error('The following error occurred during execution: %s' % (str(e))) sys.exit(-1)
def run_main(storage_path, skip_cert_verify=False, without_downloading_files=False): log_formatter = logging.Formatter( '%(asctime)s %(levelname)s {%(module)s} %(message)s', '%Y-%m-%d %H:%M:%S') log_file = os.path.join(storage_path, 'MoodleDownloader.log') log_handler = RotatingFileHandler(log_file, mode='a', maxBytes=1 * 1024 * 1024, backupCount=2, encoding=None, delay=0) log_handler.setFormatter(log_formatter) log_handler.setLevel(logging.DEBUG) app_log = logging.getLogger() app_log.setLevel(logging.DEBUG) app_log.addHandler(log_handler) logging.info('--- main started ---------------------') Log.info('Moodle Downloader starting...') if IS_DEBUG: logging.info( 'Debug-Mode detected. Errors will not be logged but instead re-risen.' ) debug_logger = logging.getLogger() debug_logger.setLevel(logging.ERROR) debug_logger.addHandler(ReRaiseOnError()) try: logging.debug('Loading config...') Log.debug('Loading config...') config = ConfigHelper(storage_path) config.load() except BaseException as e: logging.error( 'Error while trying to load the Configuration! Exiting...', extra={'exception': e}) Log.error('Error while trying to load the Configuration!') sys.exit(-1) r_client = False try: sentry_dsn = config.get_property('sentry_dsn') if sentry_dsn: sentry_sdk.init(sentry_dsn) except BaseException: pass mail_service = MailService(config) tg_service = TelegramService(config) console_service = ConsoleService(config) PathTools.filename_character_map = config.get_filename_character_map() try: if not IS_DEBUG: process_lock.lock(storage_path) moodle = MoodleService(config, storage_path, skip_cert_verify) logging.debug( 'Checking for changes for the configured Moodle-Account....') Log.debug('Checking for changes for the configured Moodle-Account...') changed_courses = moodle.fetch_state() logging.debug('Start downloading changed files...') Log.debug('Start downloading changed files...') if without_downloading_files: downloader = FakeDownloadService(changed_courses, moodle, storage_path) else: downloader = DownloadService(changed_courses, moodle, storage_path, skip_cert_verify) downloader.run() changed_courses_to_notify = moodle.recorder.changes_to_notify() if len(changed_courses_to_notify) > 0: console_service.notify_about_changes_in_moodle( changed_courses_to_notify) mail_service.notify_about_changes_in_moodle( changed_courses_to_notify) tg_service.notify_about_changes_in_moodle( changed_courses_to_notify) moodle.recorder.notified(changed_courses_to_notify) else: logging.info('No changes found for the configured Moodle-Account.') Log.warning('No changes found for the configured Moodle-Account.') process_lock.unlock(storage_path) logging.debug('All done. Exiting...') Log.success('All done. Exiting..') except BaseException as e: if not isinstance(e, process_lock.LockError): process_lock.unlock(storage_path) error_formatted = traceback.format_exc() logging.error(error_formatted, extra={'exception': e}) if r_client: sentry_sdk.capture_exception(e) short_error = '%s\r\n%s' % (str(e), traceback.format_exc(limit=1)) mail_service.notify_about_error(short_error) tg_service.notify_about_error(short_error) logging.debug('Exception-Handling completed. Exiting...', extra={'exception': e}) Log.critical('Exception:\n%s' % (error_formatted)) Log.error('The following error occurred during execution: %s' % (str(e))) sys.exit(-1)
while True: for shop in shops: log_path = os.path.join(path, 'log', shop['url_name']) if not os.path.exists(log_path): os.makedirs(log_path) nowdate = time.strftime("%Y-%m-%d", time.localtime(time.time())) logfile = shop['name'] + nowdate logger = Log(logfile, log_path, '%(asctime)s - %(message)s') if Step > 3: Step = 1 if (Step == 1): #长期没有更新的数据 sql = "select link_id,SpiderDate,stockid from prices_tb where shop_id='%s' ORDER BY SpiderDate ASC limit 100" % ( shop['shopid']) logger.info("Step 1:更新时间按升序排列,处理前面的数据:") Step = Step + 1 else: if (Step == 2): #处理超过一周没有更新的数据 sql = "select link_id,SpiderDate,stockid from prices_tb where shop_id='%s' and DATE_SUB(CURDATE(), INTERVAL 7 DAY) >= date(SpiderDate) ORDER BY SpiderDate ASC limit 300" % ( shop['shopid']) logger.info("Step 2:处理大于7天没有更新的数据:") Step = Step + 1 else: if (Step == 3): sql = "select link_id,SpiderDate,stockid from prices_tb where shop_id='%s' ORDER BY last_time DEC limit 100" % ( shop['shopid']) logger.info("Step 3:处理人工修改时间的数据:") Step = Step + 1
class AnalysisJson: """swagger自动生成测试用例""" def __init__(self, url_json): self.url_json = url_json r = requests.get(self.url_json + '/v2/api-docs').json() print(r) self.title = r['info']['description'] write_data(r, '{}.json'.format(self.title)) self.interface_params = {} self.log = Log() self.row = 2 # 写入excel起始行数 self.num = 1 # case id global title_list, json_path if self.check_data(r): self.json_path = os.path.abspath( os.path.dirname(os.path.dirname(__file__)) ) + '\\case_generate' + '\\data_new' + '\\{}_data.json'.format( self.title) # json file path,执行多个url的情况,区分生成的json文件 self.data = r['paths'] # paths中的数据是有用的 title_list.append(self.title) def check_data(self, r): """检查返回的数据是否是dict""" if not isinstance(r, dict): self.log.info('swagger return json error.') return False else: return True def retrieve_data(self): """主函数""" global body_name, method for k, v in self.data.items(): method_list = [] for _k, _v in v.items(): interface = {} if (checkJSON().isExtend(v, 'deprecated')): if not _v['deprecated']: # 接口是否被弃用 method_list.append(_k) api = k # api地址 if len(method_list) > 1: # api地址下的请求方式不止一个的情况 for i in range(len(method_list)): body_name = api.replace( '/', '_') + '_' * i # json文件对应参数名称,excel中body名称 method = method_list[ -1] # 请求方式 同一个api地址,不同请求方式 else: body_name = api.replace('/', '_') method = _k self.interface_params = self.retrieve_excel( _v, interface, api) else: print(_v) if (checkJSON().isExtend(_v, 'description')): self.log.info( 'interface path: {}, case name: {}, is deprecated.' .format(k, _v['description'])) break else: self.log.info('无key键:description') else: self.log.info('无key键:deprecated') if self.interface_params: write_data(self.interface_params, self.json_path) # 参数写入json文件 def retrieve_excel(self, _v, interface, api): """解析参数,拼接为dict--准备完成写入excel的数据""" parameters = _v.get('parameters') # 未解析的参数字典 if not parameters: # 确保参数字典存在 parameters = {} case_name = _v['summary'] # 接口名称 tags = _v['tags'][0] # 标签名称 params_dict = self.retrieve_params(parameters) # 处理接口参数,拼成dict形式 if params_dict and parameters != {}: # 单个或多个参数 interface['row_num'] = self.row # 写入excel时的所在行 interface['id'] = 'test_' + str(self.num) # case id interface['tags'] = tags # 标签名称 interface['name'] = case_name _type = 'json' # 参数获取方式 interface['method'] = method # 请求方式 interface['url'] = self.url_json + api # 拼接完成接口url interface['headers'] = 'yes' # 是否传header interface['body'] = body_name interface['type'] = _type self.num += 1 self.row += 1 self.interface_params[body_name] = params_dict self.write_excel(interface, excel_path) # 参数写入excel else: # 没有参数 _type = 'data_old' interface['name'] = case_name interface['row_num'] = self.row interface['id'] = 'test_' + str(self.num) interface['tags'] = tags interface['method'] = method interface['url'] = self.url_json + api interface['headers'] = 'yes' interface['body'] = '' interface['type'] = _type self.num += 1 self.row += 1 self.interface_params[body_name] = params_dict self.write_excel(interface, excel_path) return self.interface_params def retrieve_params(self, parameters): """处理参数,转为dict""" params = '' _in = '' for each in parameters: _in += each.get('in') + '\n' # 参数传递位置 params += each.get('name') + '\n' # 参数 _in = _in.strip('\n') _in_list = _in.split('\n') params = params.strip('\n') params_list = params.split('\n') del_list = params_list.copy() for i in range(len(_in_list)): if _in_list[i] == 'header': params_list.remove(del_list[i]) # 只保存在body传的参数 test_list = params_list.copy() params_dict = dict(zip(params_list, test_list)) # 把list转为dict return params_dict def write_excel(self, interface, filename): """把dict中的值写入对应的excel行中""" wt = Write_excel(filename, self.title) try: wt.write(interface['row_num'], 1, interface['id']) wt.write(interface['row_num'], 2, interface['tags']) wt.write(interface['row_num'], 3, interface['name']) wt.write(interface['row_num'], 4, interface['method']) wt.write(interface['row_num'], 5, interface['url']) wt.write(interface['row_num'], 7, interface['headers']) wt.write(interface['row_num'], 8, interface['body']) wt.write(interface['row_num'], 10, interface['type']) self.log.info( 'Interface case id {},write to excel file successfully!'. format(interface['id'])) except Exception as e: self.log.info( 'Failure of interface use case to write to excel file! error:{}\n' .format(e)) return