def _select_should_download_linked_files(self):
"""
Asks the user if linked files should be downloaded
"""
download_linked_files = self.config_helper.get_download_linked_files()
print('')
Log.info('In Moodle courses the teacher can also link to external' +
' files. This can be audio, video, text or anything else.' +
' In particular, the teacher can link to Youtube videos.')
Log.debug('To download videos correctly you have to install ffmpeg. ')
Log.error('These files can increase the download volume considerably.')
Log.info('If you want to filter the external links by their domain,' +
' you can manually set a whitelist and a blacklist' +
' (https://github.com/C0D3D3V/Moodle-Downloader-2/' +
'wiki/Download-(external)-linked-files' +
' for more details).')
Log.warning(
'Please note that the size of the external files is determined during the download, so the total size'
+ ' changes during the download.')
print('')
download_linked_files = cutie.prompt_yes_or_no(
'Would you like to download linked files of the courses you have selected?',
default_is_yes=download_linked_files,
)
self.config_helper.set_property('download_linked_files',
download_linked_files)
def _log_failures(self):
"""
Logs errors if any have occurred.
"""
print('')
if len(self.report['failure']) > 0:
Log.warning('Error while trying to download files, look at the log for more details.')
for url_target in self.report['failure']:
logging.error('Error while trying to download file: %s' % (url_target))
Log.error('%s\t%s' % (url_target.file.content_filename, url_target.error))
class WebSocket(object):
"""
WebSocket服务端
"""
def __init__(self):
from utils.logger import Log
self.logger = Log().logger
self.tcp_server = None
self.bind()
def bind(self):
try:
self.tcp_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.tcp_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
1)
self.tcp_server.bind(("", 5201))
self.tcp_server.listen(128)
font_color = ["\033[1;36m", "\033[0m"]
print(f"""{font_color[0] if sys.platform != "win32" else ""}注意事项：
1. 手机端请求IP地址为如下监听地址，请先测试是否可以访问通
2. 用手机浏览器测试访问说明1中尝试过的IP地址，如访问通代表无问题
3. 以下IP获取到的IP仅做参考，如果全部访问不通，请检查防火墙开启5201端口或使用ipconfig/ifconfig查看本地其他IP
4. 记得更改手机端的请求地址，并授权软件短信权限和验证码获取权限{font_color[1] if sys.platform != "win32" else ""}
""")
for idx, val in enumerate(get_inter_ip()):
self.logger.info(
f"监听地址{idx+1}:\thttp://{val}:5201/publish?smsCode=123456")
except:
self.logger.error("监听失败,请查看是否有同端口脚本")
# 等待时间30 轮询时间0.5
@limit_decor(30, 0.5)
def listener(self, *args, **kwargs):
"""
通过 socket 监听
"""
while True:
try:
cs, ca = self.tcp_server.accept()
recv_data = cs.recv(1024)
try:
a = str(
re.search(r'smsCode=(\d+)', str(recv_data)).group(1))
self.logger.info(f'监听到京东验证码:\t{a}')
return json.dumps({"sms_code": a})
except AttributeError:
self.logger.warnning(f"监听到IP: \t{ca[0]}\t访问，但未获取到短信验证码")
except:
return ""
class TuJian(object):
"""
图鉴验证类
图鉴打码地址：https://www.ttshitu.com
"""
def __init__(self, _config):
from utils.logger import Log
self.logger = Log().logger
self.username = _config["tj_username"]
self.password = _config["tj_password"]
def post_pic(self, im, type_id):
"""
提交图片
:return:
"""
base64_data = base64.b64encode(im)
b64 = base64_data.decode()
data = {
"username": self.username,
"password": self.password,
"typeid": type_id,
"image": b64
}
ret = json.loads(
requests.post("http://api.ttshitu.com/predict", json=data).text)
if ret['success']:
return ret["data"]
else:
self.logger.error(ret["message"])
sys.exit(1)
@staticmethod
def report_error(pid):
"""
提交错误图片ID
:param pid:
:return:
"""
data = {"id": pid}
ret = json.loads(
requests.post("http://api.ttshitu.com/reporterror.json",
json=data).text)
if ret['success']:
return "报错成功"
else:
return ret["message"]
def receive_token() -> str:
"""
Starts an HTTP server to receive the SSO token from browser.
It waits till a token was received.
"""
server_address = ('localhost', 80)
try:
httpd = HTTPServer(server_address, TransferServer)
except PermissionError:
Log.error('Permission denied: Please start the' +
' downloader once with administrator rights, so that it' +
' can wait on port 80 for the token.')
exit(1)
extracted_token = None
while extracted_token is None:
httpd.handle_request()
extracted_token = extract_token(TransferServer.received_token)
httpd.server_close()
return extracted_token
def run_main(storage_path,
skip_cert_verify=False,
without_downloading_files=False):
logging.basicConfig(
filename=os.path.join(storage_path, 'MoodleDownloader.log'),
level=logging.DEBUG,
format='%(asctime)s %(levelname)s {%(module)s} %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logging.info('--- main started ---------------------')
Log.info('Moodle Downloader starting...')
if IS_DEBUG:
logging.info(
'Debug-Mode detected. Errors will not be logged but instead' +
' re-risen.')
debug_logger = logging.getLogger()
debug_logger.setLevel(logging.ERROR)
debug_logger.addHandler(ReRaiseOnError())
try:
logging.debug('Loading config...')
Log.debug('Loading config...')
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error('Error while trying to load the Configuration! ' +
'Exiting...',
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(-1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
console_service = ConsoleService(config)
try:
moodle = MoodleService(config, storage_path, skip_cert_verify)
logging.debug(
'Checking for changes for the configured Moodle-Account....')
Log.debug('Checking for changes for the configured Moodle-Account...')
changed_courses = moodle.fetch_state()
diff_count = 0
logging.debug('Start downloading changed files...')
Log.debug('Start downloading changed files...')
if (without_downloading_files):
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path)
downloader.run()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
for course in changed_courses:
diff_count += len(course.files)
if diff_count > 0:
logging.info(
'%s changes found for the configured Moodle-Account.' %
(diff_count))
Log.success('%s changes found for the configured Moodle-Account.' %
(diff_count))
console_service.notify_about_changes_in_moodle(changed_courses)
else:
logging.info('No changes found for the configured Moodle-Account.')
Log.warning('No changes found for the configured Moodle-Account.')
if (len(changed_courses_to_notify) > 0):
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
mail_service.notify_about_error(str(e))
logging.debug('Exception-Handling completed. Exiting...',
extra={'exception': e})
Log.critical('Exception:\n%s' % (error_formatted))
Log.error('The following error occurred during execution: %s' %
(str(e)))
sys.exit(-1)
class JDyolocaptcha(object):
"""
yolov4类
"""
def __init__(self, _config):
self.CONFIDENCE_THRESHOLD = 0.8 # 置信阈值
self.NMS_THRESHOLD = 0.01 # 非极大值抑制
from utils.logger import Log
self.logger = Log().logger
weights = _config['yolov4_weights']
cfg = _config['yolov4_cfg']
if os.path.exists(weights):
self.net = cv2.dnn.readNet(weights, cfg)
else:
self.logger.warning(
f"找不到权重文件，当前工作目录{os.getcwd()} 应为{os.path.dirname(os.path.dirname(__file__))} 正在尝试更换工作目录"
)
os.chdir(os.path.dirname(os.path.dirname(__file__)))
if os.path.exists(weights):
self.logger.info('已找到权重文件')
self.net = cv2.dnn.readNet(weights, cfg)
else:
self.logger.error(
f"找不到权重文件，请检查权重文件路径是否正确{os.getcwd()}/{weights}，及时进行反馈")
sys.exit(1)
self.model = cv2.dnn_DetectionModel(self.net)
size = (_config['yolov4_net_size'], _config['yolov4_net_size'])
self.model.setInputParams(size=size, scale=1 / 255, swapRB=True)
def base64_conversion(self, data):
"""
base64转Mat
:param data:
:return:
"""
imgData = base64.b64decode(data.replace("data:image/jpg;base64,", ""))
nparr = np.frombuffer(imgData, np.uint8)
return cv2.imdecode(nparr, cv2.IMREAD_COLOR)
def img_merge(self, cpc, pcp):
"""
将两张图合成为一张大图，节省一次识别次数
:param cpc:
:param pcp:
:return:
"""
img = np.zeros((206, 275, 3), np.uint8)
img[0:170, 0:275] = cpc
img[170:206, 167:275] = pcp
return img
def get_box(self, data):
"""
获取坐标
"""
def key(list):
return list[1][1]
data.sort(key=key, reverse=True)
pcp_class = data[0][0]
data.pop(0)
return dict(data)[pcp_class]
def detect(self, cpc, pcp):
"""
识别验证码并返回坐标
:param cpc:
:param pcp:
:return:
"""
try:
classes, scores, boxes = self.model.detect(
self.img_merge(cpc, pcp), self.CONFIDENCE_THRESHOLD,
self.NMS_THRESHOLD) # 将验证码进行识别
data = list(zip([c[0] for c in classes.tolist()], boxes.tolist()))
x1, y1, x2, y2 = self.get_box(data)
r = (x1 * 2 + x2) // 2, (y1 * 2 + y2) // 2
return True, r
except:
return False, (None, None)
def JDyolo(self, cpc_img_path_base64, pcp_show_picture_path_base64):
return self.detect(
self.base64_conversion(cpc_img_path_base64),
self.base64_conversion(pcp_show_picture_path_base64))
def run_main(storage_path,
skip_cert_verify=False,
without_downloading_files=False):
log_formatter = logging.Formatter(
'%(asctime)s %(levelname)s {%(module)s} %(message)s',
'%Y-%m-%d %H:%M:%S')
log_file = os.path.join(storage_path, 'MoodleDownloader.log')
log_handler = RotatingFileHandler(log_file,
mode='a',
maxBytes=1 * 1024 * 1024,
backupCount=2,
encoding=None,
delay=0)
log_handler.setFormatter(log_formatter)
log_handler.setLevel(logging.DEBUG)
app_log = logging.getLogger()
app_log.setLevel(logging.DEBUG)
app_log.addHandler(log_handler)
logging.info('--- main started ---------------------')
Log.info('Moodle Downloader starting...')
if IS_DEBUG:
logging.info(
'Debug-Mode detected. Errors will not be logged but instead re-risen.'
)
debug_logger = logging.getLogger()
debug_logger.setLevel(logging.ERROR)
debug_logger.addHandler(ReRaiseOnError())
try:
logging.debug('Loading config...')
Log.debug('Loading config...')
config = ConfigHelper(storage_path)
config.load()
except BaseException as e:
logging.error(
'Error while trying to load the Configuration! Exiting...',
extra={'exception': e})
Log.error('Error while trying to load the Configuration!')
sys.exit(-1)
r_client = False
try:
sentry_dsn = config.get_property('sentry_dsn')
if sentry_dsn:
sentry_sdk.init(sentry_dsn)
except BaseException:
pass
mail_service = MailService(config)
tg_service = TelegramService(config)
console_service = ConsoleService(config)
PathTools.filename_character_map = config.get_filename_character_map()
try:
if not IS_DEBUG:
process_lock.lock(storage_path)
moodle = MoodleService(config, storage_path, skip_cert_verify)
logging.debug(
'Checking for changes for the configured Moodle-Account....')
Log.debug('Checking for changes for the configured Moodle-Account...')
changed_courses = moodle.fetch_state()
logging.debug('Start downloading changed files...')
Log.debug('Start downloading changed files...')
if without_downloading_files:
downloader = FakeDownloadService(changed_courses, moodle,
storage_path)
else:
downloader = DownloadService(changed_courses, moodle, storage_path,
skip_cert_verify)
downloader.run()
changed_courses_to_notify = moodle.recorder.changes_to_notify()
if len(changed_courses_to_notify) > 0:
console_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
mail_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
tg_service.notify_about_changes_in_moodle(
changed_courses_to_notify)
moodle.recorder.notified(changed_courses_to_notify)
else:
logging.info('No changes found for the configured Moodle-Account.')
Log.warning('No changes found for the configured Moodle-Account.')
process_lock.unlock(storage_path)
logging.debug('All done. Exiting...')
Log.success('All done. Exiting..')
except BaseException as e:
if not isinstance(e, process_lock.LockError):
process_lock.unlock(storage_path)
error_formatted = traceback.format_exc()
logging.error(error_formatted, extra={'exception': e})
if r_client:
sentry_sdk.capture_exception(e)
short_error = '%s\r\n%s' % (str(e), traceback.format_exc(limit=1))
mail_service.notify_about_error(short_error)
tg_service.notify_about_error(short_error)
logging.debug('Exception-Handling completed. Exiting...',
extra={'exception': e})
Log.critical('Exception:\n%s' % (error_formatted))
Log.error('The following error occurred during execution: %s' %
(str(e)))
sys.exit(-1)
