def _select_should_download_linked_files(self):
"""
Asks the user if linked files should be downloaded
"""
download_linked_files = self.config_helper.get_download_linked_files()
print('')
Log.info('In Moodle courses the teacher can also link to external' +
' files. This can be audio, video, text or anything else.' +
' In particular, the teacher can link to Youtube videos.')
Log.debug('To download videos correctly you have to install ffmpeg. ')
Log.error('These files can increase the download volume considerably.')
Log.info('If you want to filter the external links by their domain,' +
' you can manually set a whitelist and a blacklist' +
' (https://github.com/C0D3D3V/Moodle-Downloader-2/' +
'wiki/Download-(external)-linked-files' +
' for more details).')
Log.warning(
'Please note that the size of the external files is determined during the download, so the total size'
+ ' changes during the download.')
print('')
download_linked_files = cutie.prompt_yes_or_no(
'Would you like to download linked files of the courses you have selected?',
default_is_yes=download_linked_files,
)
self.config_helper.set_property('download_linked_files',
download_linked_files)
def _select_should_download_descriptions(self):
"""
Asks the user if descriptions should be downloaded
"""
download_descriptions = self.config_helper.get_download_descriptions()
print('')
Log.info(
'In Moodle courses, descriptions can be added to all kinds' +
' of resources, such as files, tasks, assignments or simply' +
' free text. These descriptions are usually unnecessary to' +
' download because you have already read the information or' +
' know it from context. However, there are situations where' +
' it might be interesting to download these descriptions. The' +
' descriptions are created as Markdown files and can be' +
' deleted as desired.')
Log.debug(
'Creating the description files does not take extra time, but they can be annoying'
+ ' if they only contain unnecessary information.')
print('')
download_descriptions = cutie.prompt_yes_or_no(
Log.special_str(
'Would you like to download descriptions of the courses you have selected?'
),
default_is_yes=download_descriptions,
)
self.config_helper.set_property('download_descriptions',
download_descriptions)
class WebSocket(object):
"""
WebSocket服务端
"""
def __init__(self):
from utils.logger import Log
self.logger = Log().logger
self.tcp_server = None
self.bind()
def bind(self):
try:
self.tcp_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.tcp_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
1)
self.tcp_server.bind(("", 5201))
self.tcp_server.listen(128)
font_color = ["\033[1;36m", "\033[0m"]
print(f"""{font_color[0] if sys.platform != "win32" else ""}注意事项：
1. 手机端请求IP地址为如下监听地址，请先测试是否可以访问通
2. 用手机浏览器测试访问说明1中尝试过的IP地址，如访问通代表无问题
3. 以下IP获取到的IP仅做参考，如果全部访问不通，请检查防火墙开启5201端口或使用ipconfig/ifconfig查看本地其他IP
4. 记得更改手机端的请求地址，并授权软件短信权限和验证码获取权限{font_color[1] if sys.platform != "win32" else ""}
""")
for idx, val in enumerate(get_inter_ip()):
self.logger.info(
f"监听地址{idx+1}:\thttp://{val}:5201/publish?smsCode=123456")
except:
self.logger.error("监听失败,请查看是否有同端口脚本")
# 等待时间30 轮询时间0.5
@limit_decor(30, 0.5)
def listener(self, *args, **kwargs):
"""
通过 socket 监听
"""
while True:
try:
cs, ca = self.tcp_server.accept()
recv_data = cs.recv(1024)
try:
a = str(
re.search(r'smsCode=(\d+)', str(recv_data)).group(1))
self.logger.info(f'监听到京东验证码:\t{a}')
return json.dumps({"sms_code": a})
except AttributeError:
self.logger.warnning(f"监听到IP: \t{ca[0]}\t访问，但未获取到短信验证码")
except:
return ""
def _select_courses_to_download(self, courses: [Course]):
"""
Asks the user for the courses that should be downloaded.
@param courses: All available courses
"""
download_course_ids = self.config_helper.get_download_course_ids()
dont_download_course_ids = self.config_helper.get_dont_download_course_ids(
)
print('')
Log.info(
'To avoid downloading all the Moodle courses you are enrolled in, you can select which ones you want'
+ ' to download here. ')
print('')
choices = []
defaults = []
for i, course in enumerate(courses):
choices.append(('%5i\t%s' % (course.id, course.fullname)))
if ResultsHandler._should_download_course(
course.id, download_course_ids, dont_download_course_ids):
defaults.append(i)
Log.special('Which of the courses should be downloaded?')
Log.info(
'[You can select with the space bar and confirm your selection with the enter key]'
)
print('')
selected_courses = cutie.select_multiple(options=choices,
ticked_indices=defaults)
download_course_ids = []
for i, course in enumerate(courses):
if i in selected_courses:
download_course_ids.append(course.id)
self.config_helper.set_property('download_course_ids',
download_course_ids)
self.config_helper.remove_property('dont_download_course_ids')
class EasyOCR(object):
"""
EasyOCR识别类，用于帮助ios设备识别投屏后的短信验证码
"""
def __init__(self):
from utils.logger import Log
self.logger = Log().logger
@staticmethod
def get_code_pic(_range, name='ios_code_pic.png'):
"""
获取验证码图像
:param _range:
:param name:
:return:
"""
# 确定验证码的左上角和右下角坐标
code_pic = ImageGrab.grab(_range)
code_pic.save(name)
return code_pic
def easy_ocr(self, _range, delay_time=5):
"""
easy ocr识别数字
:param delay_time: ocr识别延迟时间
:param _range: 验证码截图区域坐标(左x,左y,右x,右y)
:return: 识别到的数字
"""
global sms_code
self.get_code_pic(_range)
reader = easyocr.Reader(['ch_sim', 'en'])
result = reader.readtext('ios_code_pic.png')
find_all = re.findall(r'\'[\d]{6}\'', str(result))
if len(find_all) != 1:
find_all = re.findall(r'([\d]{6})[\u3002]', str(result))
if len(find_all) != 1:
find_all = re.findall(r'(您的验证码为[\d]{6})', str(result))
# 识别结果
self.logger.info("EasyOCR识别结果：" + str(result))
if len(find_all) == 1:
code = find_all[0].strip("'")
if sms_code == code:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.easy_ocr(_range, delay_time)
else:
sms_code = code
return code
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.easy_ocr(_range, delay_time)
def _select_should_download_databases(self):
"""
Asks the user if databases should be downloaded
"""
download_databases = self.config_helper.get_download_databases()
print('')
Log.info('In the database module of Moodle data can be stored' +
' structured with information. Often it is also' +
' possible for students to upload data there. Because' +
' the implementation of the downloader has not yet been' +
' optimized at this point, it is optional to download the' +
' databases. Currently only files are downloaded, thumbails' +
' are ignored.')
print('')
download_databases = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to download databases of your courses?'),
default_is_yes=download_databases)
self.config_helper.set_property('download_databases',
download_databases)
def _select_should_download_submissions(self):
"""
Asks the user if submissions should be downloaded
"""
download_submissions = self.config_helper.get_download_submissions()
print('')
Log.info('Submissions are files that you or a teacher have uploaded' +
' to your assignments. Moodle does not provide an' +
' interface for downloading information from all' +
' submissions to a course at once.')
Log.warning(
'Therefore, it may be slow to monitor changes to submissions.')
print('')
download_submissions = cutie.prompt_yes_or_no(
Log.special_str(
'Do you want to download submissions of your assignments?'),
default_is_yes=download_submissions,
)
self.config_helper.set_property('download_submissions',
download_submissions)
class EasyOCR(object):
"""
EasyOCR识别类，用于帮助ios设备识别投屏后的短信验证码
"""
def __init__(self, debug=False):
from utils.logger import Log
self.logger = Log().logger
self.debug = debug
def easy_ocr(self, _range_, delay_time=5):
"""
easy ocr识别数字
:param delay_time: ocr识别延迟时间
:param _range_: 验证码截图区域坐标(左x,左y,右x,右y)
:return: 识别到的数字
"""
global sms_code
screenshot_save(_range_)
reader = easyocr.Reader(['ch_sim', 'en'])
ocr_ret = str(reader.readtext(captcha_screenshot))
# debug模式打印识别内容
if self.debug:
self.logger.info(str(ocr_ret))
find_all = ""
for rule in matching_rules:
find_all = re.findall(rule, ocr_ret)
if len(find_all) >= 1:
break
if len(find_all) == 1:
code = find_all[0].strip("'")
if sms_code == code:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.easy_ocr(_range_, delay_time)
else:
sms_code = code
return code
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.easy_ocr(_range_, delay_time)
class JDyolocaptcha(object):
"""
yolov4类
"""
def __init__(self, _config):
self.CONFIDENCE_THRESHOLD = 0.8 # 置信阈值
self.NMS_THRESHOLD = 0.01 # 非极大值抑制
from utils.logger import Log
self.logger = Log().logger
weights = _config['yolov4_weights']
cfg = _config['yolov4_cfg']
if os.path.exists(weights):
self.net = cv2.dnn.readNet(weights, cfg)
else:
self.logger.warning(
f"找不到权重文件，当前工作目录{os.getcwd()} 应为{os.path.dirname(os.path.dirname(__file__))} 正在尝试更换工作目录"
)
os.chdir(os.path.dirname(os.path.dirname(__file__)))
if os.path.exists(weights):
self.logger.info('已找到权重文件')
self.net = cv2.dnn.readNet(weights, cfg)
else:
self.logger.error(
f"找不到权重文件，请检查权重文件路径是否正确{os.getcwd()}/{weights}，及时进行反馈")
sys.exit(1)
self.model = cv2.dnn_DetectionModel(self.net)
size = (_config['yolov4_net_size'], _config['yolov4_net_size'])
self.model.setInputParams(size=size, scale=1 / 255, swapRB=True)
def base64_conversion(self, data):
"""
base64转Mat
:param data:
:return:
"""
imgData = base64.b64decode(data.replace("data:image/jpg;base64,", ""))
nparr = np.frombuffer(imgData, np.uint8)
return cv2.imdecode(nparr, cv2.IMREAD_COLOR)
def img_merge(self, cpc, pcp):
"""
将两张图合成为一张大图，节省一次识别次数
:param cpc:
:param pcp:
:return:
"""
img = np.zeros((206, 275, 3), np.uint8)
img[0:170, 0:275] = cpc
img[170:206, 167:275] = pcp
return img
def get_box(self, data):
"""
获取坐标
"""
def key(list):
return list[1][1]
data.sort(key=key, reverse=True)
pcp_class = data[0][0]
data.pop(0)
return dict(data)[pcp_class]
def detect(self, cpc, pcp):
"""
识别验证码并返回坐标
:param cpc:
:param pcp:
:return:
"""
try:
classes, scores, boxes = self.model.detect(
self.img_merge(cpc, pcp), self.CONFIDENCE_THRESHOLD,
self.NMS_THRESHOLD) # 将验证码进行识别
data = list(zip([c[0] for c in classes.tolist()], boxes.tolist()))
x1, y1, x2, y2 = self.get_box(data)
r = (x1 * 2 + x2) // 2, (y1 * 2 + y2) // 2
return True, r
except:
return False, (None, None)
def JDyolo(self, cpc_img_path_base64, pcp_show_picture_path_base64):
return self.detect(
self.base64_conversion(cpc_img_path_base64),
self.base64_conversion(pcp_show_picture_path_base64))
class BaiduFanYi(object):
"""
百度翻译识别类，用于帮助ios设备识别投屏后的短信验证码
"""
def __init__(self, _config, debug=False):
from utils.logger import Log
self.logger = Log().logger
self.app_id = _config["baidu_fanyi_appid"]
self.app_key = _config["baidu_fanyi_appkey"]
self.debug = debug
if self.app_id == "" or self.app_key == "":
self.logger.warning("请在config.yaml中配置baidu翻译相关配置")
sys.exit(1)
@staticmethod
def get_file_md5(file):
return md5(file).hexdigest()
def baidu_fanyi(self, _range_, delay_time=5):
"""
百度ocr识别数字
:param delay_time: ocr识别延迟时间
:param _range_: 验证码截图区域坐标(左x,左y,右x,右y)
:ocr_return: 识别到的数字
"""
global sms_code
screenshot_save(_range_)
img = open(captcha_screenshot, 'rb').read()
salt = random.randint(32768, 65536)
sign = md5(
(self.app_id + self.get_file_md5(img) + str(salt) + "APICUID" +
"mac" + self.app_key).encode("utf-8")).hexdigest(),
payload = {
"from": "zh",
"to": "zh",
"appid": self.app_id,
"salt": salt,
"sign": sign,
"cuid": "APICUID",
"mac": "mac"
}
image = {
"image":
(os.path.basename(captcha_screenshot), img, "multipart/form-data")
}
ocr_ret = requests.post(baidu_api, params=payload, files=image).json()
# debug模式打印识别内容
if self.debug:
self.logger.info(ocr_ret)
if ocr_ret["error_code"] == "0":
ocr_ret = ocr_ret["data"]
if ocr_ret["sumSrc"] == "":
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_fanyi(_range_, delay_time)
fan_yi_ocr_ret = str(ocr_ret["sumSrc"])
find_all = ""
for rule in matching_rules:
find_all = re.findall(rule, fan_yi_ocr_ret)
if len(find_all) >= 1:
break
if len(find_all) == 1:
code = find_all[0].strip("'")
if sms_code == code:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_fanyi(_range_, delay_time)
else:
sms_code = code
return code
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_fanyi(_range_, delay_time)
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_fanyi(_range_, delay_time)
class MisticaServer():
def __init__(self, mode, key, verbose, moduleargs):
# Get args and set attributes
self.args = moduleargs
# Logger params
self.logger = Log('_server', verbose) if verbose > 0 else None
self._LOGGING_ = False if self.logger is None else True
# init Router
self.key = key
self.Router = Router(self.key, self.logger)
self.Router.start()
self.mode = mode
self.procid = 0
if self.mode == MisticaMode.SINGLE:
self.overlayname = self.args["overlay"]
self.wrappername = self.args["wrapper"]
# Checks if the wrap_server of a certain wrap_module is up and running.
def dependencyLaunched(self, name):
dname = self.getDependencyName(name)
for elem in self.Router.wrapServers:
if dname == elem.name:
return True
return False
# Gets the name of the wrap_server associated to a wrap_module
def getDependencyName(self, name):
modulename = f'wrapper.server.wrap_module.{name}.module'
try:
module = import_module(modulename)
except Exception:
print(f"Failed to import module {name}")
return ""
return module.getServerDependency()
# Returns a wrap_module, wrap_server or overlay module
def getModuleInstance(self, type, name, args):
self.procid += 1
if (type == ModuleType.WRAP_MODULE):
mpath = f"wrapper.server.wrap_module.{name}.module"
return import_module(mpath).getInstance(self.procid,
self.Router.inbox, args,
self.logger)
elif (type == ModuleType.WRAP_SERVER):
mpath = f"wrapper.server.wrap_server.{name}.module"
return import_module(mpath).getInstance(self.procid, args,
self.logger)
else:
mpath = f"overlay.server.{name}.module"
return import_module(mpath).getInstance(self.procid,
self.Router.inbox,
self.mode, args,
self.logger)
def sigintDetect(self, signum, frame):
self._LOGGING_ and self.logger.info("[Sotp] SIGINT detected")
if (self.mode == MisticaMode.SINGLE):
targetoverlay = self.Router.overlayModules[0]
targetoverlay.inbox.put(
Message('input', 0, self.Router.overlayModules[0].name,
self.Router.overlayModules[0].id, MessageType.SIGNAL,
SignalType.TERMINATE))
else:
# TODO: Depends on who's on the foreground
pass
def captureInput(self, overlay):
while True:
if overlay.exit:
break
if system() != 'Windows':
polling = poll()
try:
if system() == 'Windows':
# Ugly loop for windows
rawdata = stdin.buffer.raw.read(50000)
sleep(0.1)
else:
# Nice loop for unix
polling.register(stdin.buffer.raw.fileno(), POLLIN)
polling.poll()
rawdata = stdin.buffer.raw.read(50000)
if rawdata == b'': # assume EOF
self._LOGGING_ and self.logger.info(
"[MísticaServer] Input is dead")
self.Router.join()
if rawdata and len(rawdata) > 0 and overlay.hasInput:
overlay.inbox.put(
Message('input', 0, 'overlay', overlay.id,
MessageType.STREAM, rawdata))
except KeyboardInterrupt:
self._LOGGING_ and self.logger.info(
"[MísticaServer] CTRL+C detected. Passing to overlay")
overlay.inbox.put(
Message('input', 0, self.Router.overlayModules[0].name,
self.Router.overlayModules[0].id,
MessageType.SIGNAL, SignalType.TERMINATE))
break
return
def run(self):
# If the mode is single-handler only a wrapper module and overlay module is used
if self.mode == MisticaMode.SINGLE:
# Launch wrap_module
wmitem = self.getModuleInstance(ModuleType.WRAP_MODULE,
self.wrappername,
self.args["wrapper_args"])
wmitem.start()
self.Router.wrapModules.append(wmitem)
# Check wrap_server dependency of wrap_module and launch it
wsname = self.getDependencyName(self.wrappername)
if (not self.dependencyLaunched(self.wrappername)):
wsitem = self.getModuleInstance(ModuleType.WRAP_SERVER, wsname,
self.args["wrap_server_args"])
wsitem.start()
self.Router.wrapServers.append(wsitem)
else:
for elem in self.Router.wrapServers:
if wsname == elem.name:
wsitem = elem
break
# add wrap_module to wrap_server list
wsitem.addWrapModule(wmitem)
# Launch overlay module
omitem = self.getModuleInstance(ModuleType.OVERLAY,
self.overlayname,
self.args["overlay_args"])
omitem.start()
self.Router.overlayModules.append(omitem)
targetoverlay = self.Router.overlayModules[0]
if targetoverlay.hasInput:
self.captureInput(targetoverlay)
self.Router.join()
self._LOGGING_ and self.logger.debug("[MísticaServer] Terminated")
elif self.mode == MisticaMode.MULTI:
# Launch prompt etc.
# Before registering a wrapper or an overlay, we must make sure that there is no other
# module with incompatible parameters (e.g 2 DNS base64-based wrap_modules)
self.Router.inbox.put(
Message("Mistica", 0, "sotp", 0, MessageType.SIGNAL,
SignalType.TERMINATE))
print("Multi-handler mode is not implemented yet! use -h")
exit(0)
class BaiduOCR(object):
"""
百度ocr识别类，用于帮助ios设备识别投屏后的短信验证码
"""
def __init__(self, _config, debug=False):
from utils.logger import Log
self.logger = Log().logger
app_id = _config["baidu_app_id"]
api_key = _config["baidu_api_key"]
secret_key = _config["baidu_secret_key"]
self.debug = debug
if app_id == "" or api_key == "" or secret_key == "":
self.logger.warning("请在config.yaml中配置baidu ocr相关配置")
sys.exit(1)
self.client = AipOcr(app_id, api_key, secret_key)
def baidu_ocr(self, _range_, delay_time=5):
"""
百度ocr识别数字
:param delay_time: ocr识别延迟时间
:param _range_: 验证码截图区域坐标(左x,左y,右x,右y)
:return: 识别到的数字
"""
global sms_code
screenshot_save(_range_)
img = open(captcha_screenshot, 'rb').read()
ocr_ret = self.client.basicGeneral(img)
# debug模式打印识别内容
if self.debug:
self.logger.info(ocr_ret)
if "words_result" in ocr_ret:
if len(ocr_ret["words_result"]) == 0:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_ocr(_range_, delay_time)
ocr_ret = str(ocr_ret["words_result"])
find_all = ""
for rule in matching_rules:
find_all = re.findall(rule, ocr_ret)
if len(find_all) >= 1:
break
if len(find_all) >= 1:
code = find_all[0].strip("'")
if sms_code == code:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_ocr(_range_, delay_time)
else:
sms_code = code
return code
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_ocr(_range_, delay_time)
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_ocr(_range_, delay_time)
def run_init(storage_path, use_sso=False, skip_cert_verify=False):
config = ConfigHelper(storage_path)
if config.is_present():
do_override_input = cutie.prompt_yes_or_no(
Log.error_str('Do you want to override the existing config?'))
if not do_override_input:
sys.exit(0)
MailService(config).interactively_configure()
TelegramService(config).interactively_configure()
do_sentry = cutie.prompt_yes_or_no(
'Do you want to configure Error Reporting via Sentry?')
if do_sentry:
sentry_dsn = input('Please enter your Sentry DSN: ')
config.set_property('sentry_dsn', sentry_dsn)
moodle = MoodleService(config, storage_path, skip_cert_verify)
if use_sso:
moodle.interactively_acquire_sso_token()
else:
moodle.interactively_acquire_token()
if os.name != 'nt':
Log.info(
'On Windows many characters are forbidden in filenames and paths, if you want, these characters can be'
+ ' automatically removed from filenames.')
Log.warning(
'If you want to view the downloaded files on Windows this is important!'
)
default_windows_map = cutie.prompt_yes_or_no(
'Do you want to load the default filename character map for windows?'
)
if default_windows_map:
config.set_default_filename_character_map(True)
else:
config.set_default_filename_character_map(False)
else:
config.set_default_filename_character_map(True)
Log.success('Configuration finished and saved!')
if os.name != 'nt':
if storage_path == '.':
Log.info(
' To set a cron-job for this program on your Unix-System:\n' +
' 1. `crontab -e`\n' +
' 2. Add `*/15 * * * * cd %s && python3 %smain.py`\n' %
(os.getcwd(),
os.path.join(os.path.dirname(os.path.realpath(__file__)), ''))
+ ' 3. Save and you\'re done!')
else:
Log.info(
' To set a cron-job for this program on your Unix-System:\n' +
' 1. `crontab -e`\n' + ' 2. Add `*/15 * * * *' +
' cd %s && python3 %smain.py --path %s`\n' %
(os.getcwd(),
os.path.join(os.path.dirname(os.path.realpath(__file__)), ''),
storage_path) + ' 3. Save and you\'re done!')
print('')
Log.info(
'You can always do the additional configuration later with the --config option.'
)
do_config = cutie.prompt_yes_or_no(
'Do you want to make additional configurations now?')
if do_config:
run_configure(storage_path, skip_cert_verify)
print('')
Log.success('All set and ready to go!')
class BaiduFanYi(object):
"""
百度翻译识别类，用于帮助ios设备识别投屏后的短信验证码
"""
def __init__(self, _config, debug=False):
from utils.logger import Log
self.logger = Log().logger
self.app_id = _config["baidu_fanyi_appid"]
self.app_key = _config["baidu_fanyi_appkey"]
self.debug = debug
if self.app_id == "" or self.app_key == "":
self.logger.warning("请在config.yaml中配置baidu翻译相关配置")
sys.exit(1)
@staticmethod
def get_code_pic(_range, name='ios_code_pic.png'):
"""
获取验证码图像
:param _range:
:param name:
:return:
"""
# 确定验证码的左上角和右下角坐标
code_pic = ImageGrab.grab(_range)
code_pic.save(name)
return code_pic
@staticmethod
def get_file_md5(file):
return md5(file).hexdigest()
def baidu_fanyi(self, _range, delay_time=5):
"""
百度ocr识别数字
:param delay_time: ocr识别延迟时间
:param _range: 验证码截图区域坐标(左x,左y,右x,右y)
:return: 识别到的数字
"""
global sms_code
self.get_code_pic(_range)
img = open('ios_code_pic.png', 'rb').read()
salt = random.randint(32768, 65536)
sign = md5(
(self.app_id + self.get_file_md5(img) + str(salt) + "APICUID" +
"mac" + self.app_key).encode("utf-8")).hexdigest(),
payload = {
"from": "zh",
"to": "zh",
"appid": self.app_id,
"salt": salt,
"sign": sign,
"cuid": "APICUID",
"mac": "mac"
}
image = {
"image":
(os.path.basename("ios_code_pic.png"), img, "multipart/form-data")
}
ret = requests.post(baidu_api, params=payload, files=image).json()
# debug模式打印识别内容
if self.debug:
self.logger.info(ret)
if ret["error_code"] == "0":
ret = ret["data"]
if ret["sumSrc"] == "":
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_fanyi(_range, delay_time)
fan_yi_ret = str(ret["sumSrc"])
find_all = re.findall(r'\'[\d]{6}\'', fan_yi_ret)
if len(find_all) != 1:
find_all = re.findall(r'([\d]{6})[\u3002]', fan_yi_ret)
if len(find_all) != 1:
find_all = re.findall(r'(您的验证码为[\d]{6})', fan_yi_ret)
if len(find_all) == 1:
code = find_all[0].strip("'")
if sms_code == code:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_fanyi(_range, delay_time)
else:
sms_code = code
return code
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_fanyi(_range, delay_time)
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_fanyi(_range, delay_time)
class MisticaClient(object):
def __init__(self, key, args, verbose):
self.name = type(self).__name__
self.wrapper = None
self.overlay = None
self.qsotp = Queue()
self.qdata = Queue()
self.sem = Semaphore(1)
self.sotp_sem = Semaphore(1)
self.released = False
# Overlay and Wrapper args
self.overlayname = args["overlay"]
self.wrappername = args["wrapper"]
self.overlayargs = args["overlay_args"]
self.wrapperargs = args["wrapper_args"]
# Mistica Client args
self.key = key
# Arguments depended of overlay used
self.tag = None
# Arguments depended of wrapper used
self.max_size = None
self.poll_delay = None
self.response_timeout = None
self.max_retries = None
# Logger parameters
self.logger = Log('_client', verbose) if verbose > 0 else None
self._LOGGING_ = False if self.logger is None else True
def doWrapper(self):
self.sem.acquire()
self._LOGGING_ and self.logger.info(f"[Wrapper] Initializing...")
self.wrapper = [
x for x in ClientWrapper.__subclasses__()
if x.NAME == self.wrappername
][0](self.qsotp, self.wrapperargs, self.logger)
self.wrapper.start()
# setting sotp arguments depending on the wrapper to be used
self.max_size = self.wrapper.max_size
self.response_timeout = self.wrapper.response_timeout
self.poll_delay = self.wrapper.poll_delay
self.max_retries = self.wrapper.max_retries
self.sem.release()
def doSotp(self):
try:
self.sem.acquire()
self._LOGGING_ and self.logger.info(
f"[{self.name}] Initializing...")
self.sem.release()
self.sotp_sem.acquire()
i = 1
s = ClientWorker(self.key, self.max_retries, self.max_size,
self.tag, self.overlayname, self.wrappername,
self.qdata, self.logger)
dataThread = Thread(target=s.dataEntry, args=(self.qsotp, ))
dataThread.start()
while not s.exit:
answers = []
self._LOGGING_ and self.logger.debug(
f"[{self.name}] Iteration nº{i} Status: {s.st} Seq: {s.seqnumber}/65535"
)
if s.wait_reply:
timeout = self.response_timeout
else:
timeout = self.poll_delay
try:
dataEntry = self.qsotp.get(True, timeout)
answers = s.Entrypoint(dataEntry)
except Empty:
if s.wait_reply:
answers = s.lookForRetries()
else:
answers = s.getPollRequest()
for answer in answers:
self._LOGGING_ and self.logger.debug(
f"[{self.name}] Header Sent: {answer.printHeader()}")
if answer.receiver == self.wrapper.name:
self._LOGGING_ and self.logger.debug_all(
f"[{self.name}] Retries {s.retries}/{self.max_retries}"
)
self.wrapper.inbox.put(answer)
elif answer.receiver == self.overlay.name:
self.overlay.inbox.put(answer)
else:
raise Exception(
f"Invalid answer to {answer.receiver} in client loop"
)
if self.released == False and s.sotp_first_push:
self._LOGGING_ and self.logger.debug(
f"[{self.name}] Initialized! Unblocked sem")
self.sotp_sem.release()
self.released = True
i += 1
pass
dataThread.join()
self._LOGGING_ and self.logger.info(f"[DataThread] Terminated")
self._LOGGING_ and self.logger.info(f"[{s.name}] Terminated")
except Exception as e:
print(e)
self._LOGGING_ and self.logger.exception(
f"[ClientWorker] Exception in doSotp: {e}")
self.overlay.inbox.put(
Message("clientworker", 0, self.overlayname, 0,
MessageType.SIGNAL, SignalType.TERMINATE))
self.wrapper.inbox.put(
Message("clientworker", 0, self.wrappername, 0,
MessageType.SIGNAL, SignalType.TERMINATE))
def doOverlay(self):
self.sem.acquire()
self._LOGGING_ and self.logger.info(f"[Overlay] Initializing...")
self.overlay = [
x for x in ClientOverlay.__subclasses__()
if x.NAME == self.overlayname
][0](self.qsotp, self.qdata, self.overlayargs, self.logger)
self.overlay.start()
# setting sotp arguments depending on the overlay to be used
self.tag = self.overlay.tag
self.sem.release()
def captureInput(self):
self.sem.acquire()
self._LOGGING_ and self.logger.info(f"[Input] Initializing...")
self.sem.release()
if system() != 'Windows':
polling = poll()
while True:
if self.overlay.exit:
break
try:
if system() == 'Windows':
# Ugly loop for windows
rawdata = stdin.buffer.raw.read(300000)
sleep(0.1)
else:
# Nice loop for unix
polling.register(stdin.buffer.raw.fileno(), POLLIN)
polling.poll()
rawdata = stdin.buffer.raw.read(300000)
if rawdata == b'': # assume EOF
self.sotp_sem.acquire()
self._LOGGING_ and self.logger.debug(
f"[Input] SOTP initialized, sending terminate because input recv EOF"
)
self.overlay.inbox.put(
Message("input", 0, self.overlayname, 0,
MessageType.SIGNAL, SignalType.TERMINATE))
self.sotp_sem.release()
break
if rawdata and len(rawdata) > 0 and self.overlay.hasInput:
self.overlay.inbox.put(
Message("input", 0, self.overlayname, 0,
MessageType.STREAM, rawdata))
except KeyboardInterrupt:
self._LOGGING_ and self.logger.debug(
"[Input] CTRL+C detected. Passing to wrapper")
self.overlay.inbox.put(
Message("input", 0, self.overlayname, 0,
MessageType.SIGNAL, SignalType.TERMINATE))
break
self._LOGGING_ and self.logger.info(f"[Input] Terminated")
return
def captureExit(self, signal_received, frame):
self._LOGGING_ and self.logger.debug(
"[Input] CTRL+C detected. Passing to overlay")
self.overlay.inbox.put(
Message("input", 0, self.overlayname, 0, MessageType.SIGNAL,
SignalType.TERMINATE))
def run(self):
try:
self.doWrapper()
self.doOverlay()
sotpThread = Thread(target=self.doSotp)
sotpThread.start()
if self.overlay.hasInput:
self.captureInput()
else:
signal(SIGINT, self.captureExit)
# Crappy loop for windows
if system() == 'Windows':
while not self.overlay.exit:
sleep(0.5)
# Nice sync primitive for unix
else:
sotpThread.join()
except Exception as e:
self._LOGGING_ and self.logger.exception(
f"Exception at run(): {e}")
finally:
self._LOGGING_ and self.logger.debug(f"[{self.name}] Terminated")
class AliYunOCR(object):
"""
阿里云OCR识别类，用于帮助ios设备识别投屏后的短信验证码
"""
def __init__(self, _config, debug=False):
from utils.logger import Log
self.logger = Log().logger
self.api_url = "https://ocrapi-advanced.taobao.com/ocrservice/advanced"
self.debug = debug
appcode = _config["aliyun_appcode"]
if appcode == "":
self.logger.warning("请在config.yaml中配置aliyun_appcode")
sys.exit(1)
self.appcode = appcode
def post_url(self, img):
headers = {
"Authorization": "APPCODE %s" % self.appcode,
"Content-Type": "application/json; charset=UTF-8"
}
img_base64 = base64.b64encode(img).decode()
payload = {"img": img_base64}
resp = requests.request("POST", url=self.api_url, headers=headers, data=json.dumps(payload))
if resp.status_code != 200:
self.logger.warning("阿里云OCR请求错误，错误原因：" + resp.text)
self.logger.warning("阿里云OCR请求错误，大概率是次数用光，如非次数用光请将该问题反馈给开发者，程序即将退出")
sys.exit(1)
else:
ocr_ret = json.loads(resp.text)["content"].strip(" ")
return ocr_ret
def aliyun_ocr(self, _range_, delay_time=5):
"""
阿里云OCR识别数字
:param _range_: 验证码截图区域坐标(左x,左y,右x,右y)
:param delay_time: ocr识别延迟时间
:return: 识别到的数字
"""
global sms_code
screenshot_save(_range_)
img = open(captcha_screenshot, 'rb').read()
ocr_ret = self.post_url(img)
# debug模式打印识别内容
if self.debug:
self.logger.info(ocr_ret)
if ocr_ret != "":
find_all = ""
for rule in matching_rules:
find_all = re.findall(rule, ocr_ret)
if len(find_all) >= 1:
break
if len(find_all) == 1:
code = find_all[0].strip("'")
if sms_code == code:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.aliyun_ocr(_range_, delay_time)
else:
sms_code = code
return code
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.aliyun_ocr(_range_, delay_time)
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.aliyun_ocr(_range_, delay_time)
class BaiduOCR(object):
"""
百度ocr识别类，用于帮助ios设备识别投屏后的短信验证码
"""
def __init__(self, _config):
from utils.logger import Log
self.logger = Log().logger
app_id = _config["baidu_app_id"]
api_key = _config["baidu_api_key"]
secret_key = _config["baidu_secret_key"]
if app_id == "" or api_key == "" or secret_key == "":
self.logger.warning("请在config.yaml中配置baidu ocr相关配置")
sys.exit(1)
self.client = AipOcr(app_id, api_key, secret_key)
@staticmethod
def get_code_pic(_range, name='ios_code_pic.png'):
"""
获取验证码图像
:param _range:
:param name:
:return:
"""
# 确定验证码的左上角和右下角坐标
code_pic = ImageGrab.grab(_range)
code_pic.save(name)
return code_pic
def baidu_ocr(self, _range, delay_time=5):
"""
百度ocr识别数字
:param delay_time: ocr识别延迟时间
:param _range: 验证码截图区域坐标(左x,左y,右x,右y)
:return: 识别到的数字
"""
global sms_code
self.get_code_pic(_range)
img = open('ios_code_pic.png', 'rb').read()
ret = self.client.basicGeneral(img)
# 加这个是为了很多人不知道OCR为啥识别不到，如果介意请注释
self.logger.info(ret)
if "words_result" in ret:
if len(ret["words_result"]) == 0:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_ocr(_range, delay_time)
ocr_ret = str(ret["words_result"])
find_all = re.findall(r'\'[\d]{6}\'', ocr_ret)
if len(find_all) != 1:
find_all = re.findall(r'([\d]{6})[\u3002]', ocr_ret)
if len(find_all) != 1:
find_all = re.findall(r'(您的验证码为[\d]{6})', ocr_ret)
if len(find_all) == 1:
code = find_all[0].strip("'")
if sms_code == code:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_ocr(_range, delay_time)
else:
sms_code = code
return code
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_ocr(_range, delay_time)
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.baidu_ocr(_range, delay_time)
post_response_dic["headers"] = post_response.headers
logger.info(post_response_dic)
return post_response_dic
else:
post_response_fail["status_code"] = post_response.status_code
post_response_fail["text"] = post_response.text
logger.info(post_response_fail)
return post_response_fail
# 如果需要处理token
def requestToken(self, tokenStrDic):
for tokenStr in tokenStrDic.keys():
self.header_app
# 判断需要使用的请求类型
def requestEstimate(self, methodRequest, urlRequest, dataRequest=None):
if methodRequest == "post" or methodRequest == "Post" or methodRequest == "POST":
resultRequst = self.post_request(url_post=urlRequest,
post_data=dataRequest)
elif methodRequest == "get" or methodRequest == "Get" or methodRequest == "GET":
resultRequst = self.get_request(url_get=urlRequest,
pydata=dataRequest)
return resultRequst
if __name__ == "__main__":
ac = ApiClassification()
logger.info(ac.header_post)
result = ac.post_request(url_post=ac.url_post, post_data=ac.data)
print(result.text)
class AliYunOCR(object):
"""
阿里云OCR识别类，用于帮助ios设备识别投屏后的短信验证码
"""
def __init__(self, _config, debug=False):
from utils.logger import Log
self.logger = Log().logger
self.api_url = "https://ocrapi-advanced.taobao.com/ocrservice/advanced"
self.debug = debug
appcode = _config["aliyun_appcode"]
if appcode == "":
self.logger.warning("请在config.yaml中配置aliyun_appcode")
sys.exit(1)
self.appcode = appcode
@staticmethod
def get_code_pic(_range, name='ios_code_pic.png'):
"""
获取验证码图像
:param _range:
:param name:
:return:
"""
# 确定验证码的左上角和右下角坐标
code_pic = ImageGrab.grab(_range)
code_pic.save(name)
return code_pic
def post_url(self, img):
headers = {
"Authorization": "APPCODE %s" % self.appcode,
"Content-Type": "application/json; charset=UTF-8"
}
img_base64 = base64.b64encode(img).decode()
payload = {"img": img_base64}
resp = requests.request("POST",
url=self.api_url,
headers=headers,
data=json.dumps(payload))
if resp.status_code != 200:
self.logger.warning("阿里云OCR请求错误，错误原因：" + resp.text)
self.logger.warning("阿里云OCR请求错误，大概率是次数用光，如非次数用光请将该问题反馈给开发者，程序即将退出")
sys.exit(1)
else:
ocr_ret = json.loads(resp.text)["content"].strip(" ")
return ocr_ret
def aliyun_ocr(self, _range, delay_time=5):
"""
阿里云OCR识别数字
:param _range: 验证码截图区域坐标(左x,左y,右x,右y)
:param delay_time: ocr识别延迟时间
:return: 识别到的数字
"""
global sms_code
self.get_code_pic(_range)
img = open('ios_code_pic.png', 'rb').read()
ocr_ret = self.post_url(img)
# debug模式打印识别内容
if self.debug:
self.logger.info(ocr_ret)
if ocr_ret != "":
find_all = re.findall(r'\'[\d]{6}\'', ocr_ret)
if len(find_all) != 1:
find_all = re.findall(r'([\d]{6})[\u3002]', ocr_ret)
if len(find_all) != 1:
find_all = re.findall(r'(您的验证码为[\d]{6})', ocr_ret)
if len(find_all) != 1 and len(ocr_ret) == 6:
find_all = re.findall(r'[\d]{6}', ocr_ret)
if len(find_all) == 1:
code = find_all[0].strip("'")
if sms_code == code:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.aliyun_ocr(_range, delay_time)
else:
sms_code = code
return code
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.aliyun_ocr(_range, delay_time)
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.aliyun_ocr(_range, delay_time)
def _set_options_of_courses(self, courses: [Course]):
"""
Let the user set special options for every single course
"""
download_course_ids = self.config_helper.get_download_course_ids()
dont_download_course_ids = self.config_helper.get_dont_download_course_ids(
)
print('')
Log.info(
'You can set special settings for every single course.\n' +
' You can set these options:\n' +
'- A different name for the course\n' +
'- If a directory structure should be created for the course' +
' [create_directory_structure (cfs)].')
print('')
while True:
choices = []
choices_courses = []
options_of_courses = self.config_helper.get_options_of_courses()
choices.append('None')
for course in courses:
if ResultsHandler._should_download_course(
course.id, download_course_ids,
dont_download_course_ids):
current_course_settings = options_of_courses.get(
str(course.id), None)
# create default settings
if current_course_settings is None:
current_course_settings = {
'original_name': course.fullname,
'overwrite_name_with': None,
'create_directory_structure': True,
}
# create list of options
overwrite_name_with = current_course_settings.get(
'overwrite_name_with', None)
create_directory_structure = current_course_settings.get(
'create_directory_structure', True)
if overwrite_name_with is not None and overwrite_name_with != course.fullname:
choices.append(
('%5i\t%s (%s) cfs=%s' %
(course.id, overwrite_name_with, course.fullname,
create_directory_structure)))
else:
choices.append(
('%5i\t%s cfs=%s' % (course.id, course.fullname,
create_directory_structure)))
choices_courses.append(course)
print('')
Log.special(
'For which of the following course do you want to change the settings?'
)
print('[Confirm your selection with the Enter key]')
print('')
selected_course = cutie.select(options=choices)
if selected_course == 0:
break
else:
self._change_settings_of(choices_courses[selected_course - 1],
options_of_courses)
class Animotes:
def __init__(self, bot):
self.bot = bot
self.channel_logger = ChannelLogger(bot)
self.log = Log(__name__)
async def on_message(self, message):
if not message.author.bot and sql.is_user_animote_user(message.author.id):
channel = message.channel
content = emote_corrector(self, message)
if content:
await message.delete()
await channel.send(content=content)
@commands.command(aliases=['unregister'])
async def register(self, ctx):
if not sql.is_user_animote_user(ctx.author.id):
sql.add_animote_user(ctx.author.id)
message = getlang(ctx.guild.id, "animotes.member_opt_in")
else:
sql.delete_animote_user(ctx.author.id)
message = getlang(ctx.guild.id, "animotes.member_opt_out")
await ctx.message.author.send(content=message)
@commands.command(aliases=['unregisterserver'])
@commands.guild_only()
async def registerguild(self, ctx):
if not ctx.author.permissions_in(ctx.channel).manage_emojis:
raise NoPermission
if not sql.is_server_emojishare_server(ctx.guild.id):
sql.update_guild(ctx.guild.id, emojishare=1)
self.log.info("Guild {0.name} (ID: {0.id}) has opted in to emoji sharing.".format(ctx.guild))
await self.channel_logger.log_to_channel("Guild **{0.name}** (ID: `{0.id}`) has opted in to emoji sharing."
.format(ctx.guild))
message = getlang(ctx.guild.id, "animotes.guild_opt_in")
else:
sql.update_guild(ctx.guild.id, emojishare=0)
self.log.info("Guild {0.name} (ID: {0.id}) has opted out of emoji sharing.".format(ctx.guild))
await self.channel_logger.log_to_channel("Guild **{0.name}** (ID: `{0.id}`) has opted out of emoji sharing."
.format(ctx.guild))
message = getlang(ctx.guild.id, "animotes.guild_opt_out")
await ctx.send(message)
@commands.command()
async def listemotes(self, ctx):
guilds = []
blacklist = []
whitelist = []
opted_in = sql.is_server_emojishare_server(ctx.guild.id)
whitelist.append(ctx.guild.id) # Emoji from this server are allowed automatically
for e in self.bot.emojis:
if e.animated:
# No emoji from blacklisted servers
if e.guild_id in blacklist:
continue
# Do not list cross-server emoji if this server has not opted in
if e.guild_id != ctx.guild.id and not opted_in:
continue
# Blacklist servers that have not themselves opted in
if not (e.guild_id in whitelist or sql.is_server_emojishare_server(e.guild_id)):
blacklist.append(e.guild_id)
continue
# If passed all checks, ensure this server is whitelisted so we can skip future opt-in checks
if e.guild_id not in whitelist:
whitelist.append(e.guild_id)
if not any(x['id'] for x in guilds if x['id'] == e.guild_id):
guild = next(x for x in self.bot.guilds if x.id == e.guild_id)
guilds.append({'id': e.guild_id, 'name': guild.name, 'animojis': []})
pos = next(i for i, x in enumerate(guilds) if x['id'] == e.guild_id)
guilds[pos]['animojis'].append(str(e))
for g in guilds:
msg = "**{0}**:".format(g['name'])
msg += "\n"
for e in g['animojis']:
msg += e
await ctx.send(msg)
class CnOCR(object):
"""
CnOCR识别类，用于帮助ios设备识别投屏后的短信验证码
"""
def __init__(self):
from utils.logger import Log
self.logger = Log().logger
@staticmethod
def get_code_pic(_range, name='ios_code_pic.png'):
"""
获取验证码图像
:param _range:
:param name:
:return:
"""
# 确定验证码的左上角和右下角坐标
code_pic = ImageGrab.grab(_range)
code_pic.save(name)
return code_pic
def cn_ocr(self, _range, delay_time=5):
"""
cn ocr识别数字
:param delay_time: ocr识别延迟时间
:param _range: 验证码截图区域坐标(左x,左y,右x,右y)
:return: 识别到的数字
"""
global sms_code
self.get_code_pic(_range)
cn_ocr = CnOcr(model_name="conv-lite-fc",
context="cpu",
root="conv-lite-fc")
ret = cn_ocr.ocr("ios_code_pic.png")
result = ""
for v in ret:
result += "".join(v)
find_all = re.findall(r'\'[\d]{6}\'', str(result))
if len(find_all) != 1:
find_all = re.findall(r'([\d]{6})[\u3002]', str(result))
if len(find_all) != 1:
find_all = re.findall(r'(您的验证码为[\d]{6})', str(result))
# 识别结果
self.logger.info("CnOCR识别结果：" + result)
if len(find_all) == 1:
code = find_all[0].strip("'")
if sms_code == code:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.cn_ocr(_range, delay_time)
else:
sms_code = code
return code
else:
self.logger.info("暂未获取到最新验证码，%d秒后重试" % delay_time)
time.sleep(delay_time)
return self.cn_ocr(_range, delay_time)
def run_main(storage_path,
skip_cert_verify=False,
without_downloading_files=False):
logging.basicConfig(
filename=os.path.join(storage_path, 'MoodleDownloader.log'),
level=logging.DEBUG,
format='%(asctime)s %(levelname)s {%(module)s} %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logging.info('--- main started ---------------------')
Log.info('Moodle Downloader starting...')
if IS_DEBUG:
logging.info(
'Debug-Mode detected. Errors will not be logged but instead' +
' re-risen.')
debug_logger = logging.getLogger()
debug_logger.setLevel(logging.ERROR)
debug_logger.addHandler(ReRaiseOnError())
try:
logging.debug('Loading config...')
Log.debug('Loading config...')
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error('Error while trying to load the Configuration! ' +
'Exiting...',
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(-1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
console_service = ConsoleService(config)
try:
moodle = MoodleService(config, storage_path, skip_cert_verify)
logging.debug(
'Checking for changes for the configured Moodle-Account....')
Log.debug('Checking for changes for the configured Moodle-Account...')
changed_courses = moodle.fetch_state()
diff_count = 0
logging.debug('Start downloading changed files...')
Log.debug('Start downloading changed files...')
if (without_downloading_files):
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path)
downloader.run()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
for course in changed_courses:
diff_count += len(course.files)
if diff_count > 0:
logging.info(
'%s changes found for the configured Moodle-Account.' %
(diff_count))
Log.success('%s changes found for the configured Moodle-Account.' %
(diff_count))
console_service.notify_about_changes_in_moodle(changed_courses)
else:
logging.info('No changes found for the configured Moodle-Account.')
Log.warning('No changes found for the configured Moodle-Account.')
if (len(changed_courses_to_notify) > 0):
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
mail_service.notify_about_error(str(e))
logging.debug('Exception-Handling completed. Exiting...',
extra={'exception': e})
Log.critical('Exception:\n%s' % (error_formatted))
Log.error('The following error occurred during execution: %s' %
(str(e)))
sys.exit(-1)
def run_main(storage_path,
skip_cert_verify=False,
without_downloading_files=False):
log_formatter = logging.Formatter(
'%(asctime)s %(levelname)s {%(module)s} %(message)s',
'%Y-%m-%d %H:%M:%S')
log_file = os.path.join(storage_path, 'MoodleDownloader.log')
log_handler = RotatingFileHandler(log_file,
mode='a',
maxBytes=1 * 1024 * 1024,
backupCount=2,
encoding=None,
delay=0)
log_handler.setFormatter(log_formatter)
log_handler.setLevel(logging.DEBUG)
app_log = logging.getLogger()
app_log.setLevel(logging.DEBUG)
app_log.addHandler(log_handler)
logging.info('--- main started ---------------------')
Log.info('Moodle Downloader starting...')
if IS_DEBUG:
logging.info(
'Debug-Mode detected. Errors will not be logged but instead re-risen.'
)
debug_logger = logging.getLogger()
debug_logger.setLevel(logging.ERROR)
debug_logger.addHandler(ReRaiseOnError())
try:
logging.debug('Loading config...')
Log.debug('Loading config...')
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error(
'Error while trying to load the Configuration! Exiting...',
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(-1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
tg_service = TelegramService(config)
console_service = ConsoleService(config)
PathTools.filename_character_map = config.get_filename_character_map()
try:
if not IS_DEBUG:
process_lock.lock(storage_path)
moodle = MoodleService(config, storage_path, skip_cert_verify)
logging.debug(
'Checking for changes for the configured Moodle-Account....')
Log.debug('Checking for changes for the configured Moodle-Account...')
changed_courses = moodle.fetch_state()
logging.debug('Start downloading changed files...')
Log.debug('Start downloading changed files...')
if without_downloading_files:
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path,
skip_cert_verify)
downloader.run()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
if len(changed_courses_to_notify) > 0:
console_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
tg_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
else:
logging.info('No changes found for the configured Moodle-Account.')
Log.warning('No changes found for the configured Moodle-Account.')
process_lock.unlock(storage_path)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
if not isinstance(e, process_lock.LockError):
process_lock.unlock(storage_path)
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
short_error = '%s\r\n%s' % (str(e), traceback.format_exc(limit=1))
mail_service.notify_about_error(short_error)
tg_service.notify_about_error(short_error)
logging.debug('Exception-Handling completed. Exiting...',
extra={'exception': e})
Log.critical('Exception:\n%s' % (error_formatted))
Log.error('The following error occurred during execution: %s' %
(str(e)))
sys.exit(-1)
while True:
for shop in shops:
log_path = os.path.join(path, 'log', shop['url_name'])
if not os.path.exists(log_path):
os.makedirs(log_path)
nowdate = time.strftime("%Y-%m-%d", time.localtime(time.time()))
logfile = shop['name'] + nowdate
logger = Log(logfile, log_path, '%(asctime)s - %(message)s')
if Step > 3:
Step = 1
if (Step == 1):
#长期没有更新的数据
sql = "select link_id,SpiderDate,stockid from prices_tb where shop_id='%s' ORDER BY SpiderDate ASC limit 100" % (
shop['shopid'])
logger.info("Step 1:更新时间按升序排列，处理前面的数据:")
Step = Step + 1
else:
if (Step == 2):
#处理超过一周没有更新的数据
sql = "select link_id,SpiderDate,stockid from prices_tb where shop_id='%s' and DATE_SUB(CURDATE(), INTERVAL 7 DAY) >= date(SpiderDate) ORDER BY SpiderDate ASC limit 300" % (
shop['shopid'])
logger.info("Step 2:处理大于7天没有更新的数据:")
Step = Step + 1
else:
if (Step == 3):
sql = "select link_id,SpiderDate,stockid from prices_tb where shop_id='%s' ORDER BY last_time DEC limit 100" % (
shop['shopid'])
logger.info("Step 3:处理人工修改时间的数据:")
Step = Step + 1
class AnalysisJson:
"""swagger自动生成测试用例"""
def __init__(self, url_json):
self.url_json = url_json
r = requests.get(self.url_json + '/v2/api-docs').json()
print(r)
self.title = r['info']['description']
write_data(r, '{}.json'.format(self.title))
self.interface_params = {}
self.log = Log()
self.row = 2 # 写入excel起始行数
self.num = 1 # case id
global title_list, json_path
if self.check_data(r):
self.json_path = os.path.abspath(
os.path.dirname(os.path.dirname(__file__))
) + '\\case_generate' + '\\data_new' + '\\{}_data.json'.format(
self.title) # json file path，执行多个url的情况，区分生成的json文件
self.data = r['paths'] # paths中的数据是有用的
title_list.append(self.title)
def check_data(self, r):
"""检查返回的数据是否是dict"""
if not isinstance(r, dict):
self.log.info('swagger return json error.')
return False
else:
return True
def retrieve_data(self):
"""主函数"""
global body_name, method
for k, v in self.data.items():
method_list = []
for _k, _v in v.items():
interface = {}
if (checkJSON().isExtend(v, 'deprecated')):
if not _v['deprecated']: # 接口是否被弃用
method_list.append(_k)
api = k # api地址
if len(method_list) > 1: # api地址下的请求方式不止一个的情况
for i in range(len(method_list)):
body_name = api.replace(
'/',
'_') + '_' * i # json文件对应参数名称，excel中body名称
method = method_list[
-1] # 请求方式 同一个api地址，不同请求方式
else:
body_name = api.replace('/', '_')
method = _k
self.interface_params = self.retrieve_excel(
_v, interface, api)
else:
print(_v)
if (checkJSON().isExtend(_v, 'description')):
self.log.info(
'interface path: {}, case name: {}, is deprecated.'
.format(k, _v['description']))
break
else:
self.log.info('无key键:description')
else:
self.log.info('无key键:deprecated')
if self.interface_params:
write_data(self.interface_params, self.json_path) # 参数写入json文件
def retrieve_excel(self, _v, interface, api):
"""解析参数，拼接为dict--准备完成写入excel的数据"""
parameters = _v.get('parameters') # 未解析的参数字典
if not parameters: # 确保参数字典存在
parameters = {}
case_name = _v['summary'] # 接口名称
tags = _v['tags'][0] # 标签名称
params_dict = self.retrieve_params(parameters) # 处理接口参数，拼成dict形式
if params_dict and parameters != {}: # 单个或多个参数
interface['row_num'] = self.row # 写入excel时的所在行
interface['id'] = 'test_' + str(self.num) # case id
interface['tags'] = tags # 标签名称
interface['name'] = case_name
_type = 'json' # 参数获取方式
interface['method'] = method # 请求方式
interface['url'] = self.url_json + api # 拼接完成接口url
interface['headers'] = 'yes' # 是否传header
interface['body'] = body_name
interface['type'] = _type
self.num += 1
self.row += 1
self.interface_params[body_name] = params_dict
self.write_excel(interface, excel_path) # 参数写入excel
else: # 没有参数
_type = 'data_old'
interface['name'] = case_name
interface['row_num'] = self.row
interface['id'] = 'test_' + str(self.num)
interface['tags'] = tags
interface['method'] = method
interface['url'] = self.url_json + api
interface['headers'] = 'yes'
interface['body'] = ''
interface['type'] = _type
self.num += 1
self.row += 1
self.interface_params[body_name] = params_dict
self.write_excel(interface, excel_path)
return self.interface_params
def retrieve_params(self, parameters):
"""处理参数，转为dict"""
params = ''
_in = ''
for each in parameters:
_in += each.get('in') + '\n' # 参数传递位置
params += each.get('name') + '\n' # 参数
_in = _in.strip('\n')
_in_list = _in.split('\n')
params = params.strip('\n')
params_list = params.split('\n')
del_list = params_list.copy()
for i in range(len(_in_list)):
if _in_list[i] == 'header':
params_list.remove(del_list[i]) # 只保存在body传的参数
test_list = params_list.copy()
params_dict = dict(zip(params_list, test_list)) # 把list转为dict
return params_dict
def write_excel(self, interface, filename):
"""把dict中的值写入对应的excel行中"""
wt = Write_excel(filename, self.title)
try:
wt.write(interface['row_num'], 1, interface['id'])
wt.write(interface['row_num'], 2, interface['tags'])
wt.write(interface['row_num'], 3, interface['name'])
wt.write(interface['row_num'], 4, interface['method'])
wt.write(interface['row_num'], 5, interface['url'])
wt.write(interface['row_num'], 7, interface['headers'])
wt.write(interface['row_num'], 8, interface['body'])
wt.write(interface['row_num'], 10, interface['type'])
self.log.info(
'Interface case id {},write to excel file successfully!'.
format(interface['id']))
except Exception as e:
self.log.info(
'Failure of interface use case to write to excel file! error:{}\n'
.format(e))
return
