def compute_validation_map(epoch,
iteration,
yolact_net,
dataset,
log: Log = None,
wandb=None):
with torch.no_grad():
yolact_net.eval()
start = time.time()
print()
print("Computing validation mAP (this may take a while)...",
flush=True)
val_info = eval_script.evaluate(yolact_net, dataset, train_mode=True)
end = time.time()
if log is not None:
log.log('val',
val_info,
elapsed=(end - start),
epoch=epoch,
iter=iteration)
if wandb is not None:
wandb.log({
"Box Accuracy": val_info['box']['all'],
"Mask Accuracy": val_info['mask']['all']
})
yolact_net.train()
def _select_should_download_linked_files(self):
"""
Asks the user if linked files should be downloaded
"""
download_linked_files = self.config_helper.get_download_linked_files()
print('')
Log.info('In Moodle courses the teacher can also link to external' +
' files. This can be audio, video, text or anything else.' +
' In particular, the teacher can link to Youtube videos.')
Log.debug('To download videos correctly you have to install ffmpeg. ')
Log.error('These files can increase the download volume considerably.')
Log.info('If you want to filter the external links by their domain,' +
' you can manually set a whitelist and a blacklist' +
' (https://github.com/C0D3D3V/Moodle-Downloader-2/' +
'wiki/Download-(external)-linked-files' +
' for more details).')
Log.warning(
'Please note that the size of the external files is determined during the download, so the total size'
+ ' changes during the download.')
print('')
download_linked_files = cutie.prompt_yes_or_no(
'Would you like to download linked files of the courses you have selected?',
default_is_yes=download_linked_files,
)
self.config_helper.set_property('download_linked_files',
download_linked_files)
def run_change_notification_mail(storage_path):
config = ConfigHelper(storage_path)
config.load()
MailService(config).interactively_configure()
Log.success('Configuration successfully updated!')
def run_change_notification_telegram(storage_path):
config = ConfigHelper(storage_path)
config.load()
TelegramService(config).interactively_configure()
Log.success('Telegram Configuration successfully updated!')
def _select_should_download_descriptions(self):
"""
Asks the user if descriptions should be downloaded
"""
download_descriptions = self.config_helper.get_download_descriptions()
print('')
Log.info(
'In Moodle courses, descriptions can be added to all kinds' +
' of resources, such as files, tasks, assignments or simply' +
' free text. These descriptions are usually unnecessary to' +
' download because you have already read the information or' +
' know it from context. However, there are situations where' +
' it might be interesting to download these descriptions. The' +
' descriptions are created as Markdown files and can be' +
' deleted as desired.')
Log.debug(
'Creating the description files does not take extra time, but they can be annoying'
+ ' if they only contain unnecessary information.')
print('')
download_descriptions = cutie.prompt_yes_or_no(
Log.special_str(
'Would you like to download descriptions of the courses you have selected?'
),
default_is_yes=download_descriptions,
)
self.config_helper.set_property('download_descriptions',
download_descriptions)
def compute_validation_map(epoch,
iteration,
yolact_net,
dataset,
log: Log = None):
with torch.no_grad():
yolact_net.eval()
start = time.time()
print()
print("Computing validation mAP (this may take a while)...",
flush=True)
val_info = eval_script.evaluate(yolact_net, dataset, train_mode=True)
mapData.append([
epoch, val_info['box']['all'], val_info['box'][55],
val_info['box'][65], val_info['box'][75], val_info['box'][85],
val_info['box'][95], val_info['mask']['all'], val_info['mask'][55],
val_info['mask'][65], val_info['mask'][75], val_info['mask'][85],
val_info['mask'][95]
])
end = time.time()
if log is not None:
log.log('val',
val_info,
elapsed=(end - start),
epoch=epoch,
iter=iteration)
yolact_net.train()
return val_info
def __init__(self, key, args, verbose):
self.name = type(self).__name__
self.wrapper = None
self.overlay = None
self.qsotp = Queue()
self.qdata = Queue()
self.sem = Semaphore(1)
self.sotp_sem = Semaphore(1)
self.released = False
# Overlay and Wrapper args
self.overlayname = args["overlay"]
self.wrappername = args["wrapper"]
self.overlayargs = args["overlay_args"]
self.wrapperargs = args["wrapper_args"]
# Mistica Client args
self.key = key
# Arguments depended of overlay used
self.tag = None
# Arguments depended of wrapper used
self.max_size = None
self.poll_delay = None
self.response_timeout = None
self.max_retries = None
# Logger parameters
self.logger = Log('_client', verbose) if verbose > 0 else None
self._LOGGING_ = False if self.logger is None else True
def _get_control_tree(self, process_name):
'''获取指定进程中的所有控件树
:param process_name: 进程名
:type process_name: string
'''
driver = self._get_driver(process_name)
Log.i('ControlManager',
'get control tree in process %s' % process_name)
result = driver._get_control_tree('', -1)
Log.i('ControlManager', 'get control tree complete')
for key in result.keys():
if not result[key]:
result.pop(key)
continue
if not 'Visible' in result[key]:
print result[key]
if not result[key]['Visible']:
# 过滤掉不可见的窗口
print 'ignor window %s' % key
result.pop(key)
pattern = re.compile(r'^(.+)#(\d+)$')
output_result = {} # 窗口名相同的放到list中
for key in result.keys():
window_title = key
ret = pattern.match(key)
if ret: window_title = ret.group(1)
if not window_title in output_result:
output_result[window_title] = [process_name]
output_result[window_title].append(result.pop(key))
return output_result
def compute_validation_map(epoch,
iteration,
yolact_net,
dataset,
log: Log = None):
'''
- 훈련이 종료되었을 때 호출되는 함수이다.
- yolact모델을 eval모드로 바꾼 뒤, 최종 mAP를 측정한다.
- 함수가 끝난 뒤에는 다시 train모드로 바꾼다.
'''
with torch.no_grad():
yolact_net.eval()
start = time.time()
print()
print("Computing validation mAP (this may take a while)...",
flush=True)
val_info = eval_script.evaluate(yolact_net, dataset, train_mode=True)
end = time.time()
if log is not None:
log.log('val',
val_info,
elapsed=(end - start),
epoch=epoch,
iter=iteration)
yolact_net.train()
def write_data(res, json_path):
"""把处理后的参数写入json文件"""
if isinstance(res, dict):
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(res, f, indent=4)
Log().info('Interface Params Total：{} ,write to json file successfully!\n'.format(len(res)))
else:
Log().info('{} Params is not dict.\n'.format(write_data.__name__))
def run_manage_database(storage_path):
config = ConfigHelper(storage_path)
config.load() # because we want to only manage configured courses
offline_service = OfflineService(config, storage_path)
offline_service.interactively_manage_database()
Log.success('All done.')
def run_configure(storage_path, skip_cert_verify=False):
config = ConfigHelper(storage_path)
config.load() # because we do not want to override the other settings
ConfigService(config, storage_path,
skip_cert_verify).interactively_acquire_config()
Log.success('Configuration successfully updated!')
def __init__(self, _config, debug=False):
from utils.logger import Log
self.logger = Log().logger
self.app_id = _config["baidu_fanyi_appid"]
self.app_key = _config["baidu_fanyi_appkey"]
self.debug = debug
if self.app_id == "" or self.app_key == "":
self.logger.warning("请在config.yaml中配置baidu翻译相关配置")
sys.exit(1)
def on_put(self, req, resp, phone):
"""
Handles the PUT request updating an specific phone number.
The req.body should contain all the parameters that the object allows
because it will validate them as if it was an insert
Args:
req (Falcon HTTP Request): Request received by the server
resp (Falcon HTTP Response): Response constructed by the server
phone (str): Phone number to be updated in the database
Returns:
None
"""
Log(f"PUT Request Received: Searching {phone}", req=req)
resp.status = falcon.HTTP_400
resp.content_type = "application/json"
data = None
try:
data = json.loads(req.bounded_stream.read(), encoding='utf-8')
except json.decoder.JSONDecodeError as err:
Log(f"Error reading body: {err}", req=req, level=3)
resp.body = json.dumps({"msg": f"Error reading body: {err}"},
ensure_ascii=False)
resp.body += "\n"
return
person = PersonOperator.searchBy(key="phone", value=str(phone))
if len(person) != 1:
resp.body = json.dumps(
{"msg": f"The person with the phone {phone} wasn't found"})
resp.body += "\n"
return
person = None
validator = Validate()
isValid = validator.fullValidation(data)
if isValid["ok"] is not True:
resp.body = json.dumps({"msg": isValid["msg"]}, ensure_ascii=False)
resp.body += "\n"
return
update = PersonOperator.write(values=data, onPhone=str(phone))
if update["ok"] is not True:
resp.body = json.dumps({
"msg":
f"There was an error updating the records: {update['err']}"
})
else:
resp.status = falcon.HTTP_200
resp.body = json.dumps(
{"msg": f"{data['name']} was updated successfully"})
resp.body += "\n"
def __init__(self, _config, debug=False):
from utils.logger import Log
self.logger = Log().logger
self.api_url = "https://ocrapi-advanced.taobao.com/ocrservice/advanced"
self.debug = debug
appcode = _config["aliyun_appcode"]
if appcode == "":
self.logger.warning("请在config.yaml中配置aliyun_appcode")
sys.exit(1)
self.appcode = appcode
def _change_settings_of(self, course: Course, options_of_courses: {}):
"""
Ask for a new Name for the course.
Then asks if a file structure should be created.
"""
current_course_settings = options_of_courses.get(str(course.id), None)
# create default settings
if current_course_settings is None:
current_course_settings = {
'original_name': course.fullname,
'overwrite_name_with': None,
'create_directory_structure': True,
}
changed = False
# Ask for new name
overwrite_name_with = input(
Log.special_str(
'Enter a new name for this Course [leave blank for "%s"]: ' %
(course.fullname, )))
if overwrite_name_with == '':
overwrite_name_with = None
if (overwrite_name_with != course.fullname
and current_course_settings.get('overwrite_name_with',
None) != overwrite_name_with):
current_course_settings.update(
{'overwrite_name_with': overwrite_name_with})
changed = True
# Ask if a file structure should be created
create_directory_structure = current_course_settings.get(
'create_directory_structure', True)
create_directory_structure = cutie.prompt_yes_or_no(
Log.special_str(
'Should a directory structure be created for this course?'),
default_is_yes=create_directory_structure,
)
if create_directory_structure is not current_course_settings.get(
'create_directory_structure', True):
changed = True
current_course_settings.update(
{'create_directory_structure': create_directory_structure})
if changed:
options_of_courses.update(
{str(course.id): current_course_settings})
self.config_helper.set_property('options_of_courses',
options_of_courses)
def _log_failures(self):
"""
Logs errors if any have occurred.
"""
print('')
if len(self.report['failure']) > 0:
Log.warning('Error while trying to download files, look at the log for more details.')
for url_target in self.report['failure']:
logging.error('Error while trying to download file: %s' % (url_target))
Log.error('%s\t%s' % (url_target.file.content_filename, url_target.error))
def _get_windows_data(self):
'''获取windows数据并解析
'''
result = self._device.adb.run_shell_cmd('dumpsys window windows')
result = result.replace('\r', '')
# print result
windows = []
window = None
p1 = re.compile(r'^ Window #(\d+) Window{(\w{6,8}) (.*)}:$')
p2 = re.compile(r'Window{(\w{6,8}) (u0 ){0,1}(\S+).*}')
p3 = re.compile(r'mShownFrame=\[([-\d\.]+),([-\d\.]+)\]\[([-\d\.]+),([-\d\.]+)\]')
for line in result.split('\n')[1:]:
# print repr(line)
ret = p1.match(line)
if ret:
title = ret.group(3)
if ' ' in title:
items = title.split(' ')
if 'u0' == items[0]: title = items[1]
else: title = items[0]
window = Window(self, int(ret.group(1)), ret.group(2), title) # 此逻辑可能有bug
windows.append(window)
elif 'mCurrentFocus' in line:
ret = p2.search(line)
if ret:
title = ret.group(3)
self._current_window = Window(self, 0, ret.group(1), title)
else:
Log.w('WindowManager', line)
self._current_window = None
elif 'mInputMethodTarget' in line:
ret = p2.search(line)
self._current_input_target = Window(self, 0, ret.group(1), ret.group(2) if ret.group(2) and len(ret.group(2)) > 5 else ret.group(3))
elif line.startswith(' '):
if 'mShownFrame' in line:
ret = p3.search(line)
window['x'] = int(float(ret.group(1)))
window['y'] = int(float(ret.group(2)))
elif 'mAttachedWindow' in line:
ret = p2.search(line)
window['mAttachedWindow'] = Window(self, 0, ret.group(1), ret.group(2) if ret.group(2) and len(ret.group(2)) > 5 else ret.group(3))
else:
items = line.split(' ')
for item in items:
if not '=' in item: continue
pos = item.find('=')
key = item[:pos]
val = item[pos + 1:]
if key in ['package', 'w', 'h']:
window[key] = val
else:
pass
return windows
def __init__(self, _config):
from utils.logger import Log
self.logger = Log().logger
app_id = _config["baidu_app_id"]
api_key = _config["baidu_api_key"]
secret_key = _config["baidu_secret_key"]
if app_id == "" or api_key == "" or secret_key == "":
self.logger.warning("请在config.yaml中配置baidu ocr相关配置")
sys.exit(1)
self.client = AipOcr(app_id, api_key, secret_key)
def update_control_tree(process_name):
'''更新控件树
'''
try:
control_tree = self._get_control_tree(process_name)
except:
Log.ex('ControlManager',
'get control tree in %s failed' % process_name)
return
result.update(control_tree) # 相同窗口名的窗口必然在同一进程中，所以不会冲突
process_list.append(process_name)
def notify_about_changes_in_moodle(self, changes: [Course]) -> None:
"""
Creates a terminal output about the downloaded changes.
@param changes: A list of changed courses with changed files.
"""
RESET_SEQ = '\033[0m'
COLOR_SEQ = '\033[1;%dm'
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
print('\n')
diff_count = 0
for course in changes:
diff_count += len(course.files)
if diff_count > 0:
logging.info('%s changes found for the configured Moodle-Account.' % (diff_count))
Log.success('%s changes found for the configured Moodle-Account.' % (diff_count))
for course in changes:
if len(course.files) == 0:
continue
print(COLOR_SEQ % BLUE + course.fullname + RESET_SEQ)
for file in course.files:
if file.modified:
print(COLOR_SEQ % YELLOW + '≠\t' + file.saved_to + RESET_SEQ)
elif file.moved:
if file.new_file is not None:
print(
COLOR_SEQ % CYAN
+ '<->\t'
+ (
file.saved_to
+ RESET_SEQ
+ COLOR_SEQ % GREEN
+ ' ==> '
+ file.new_file.saved_to
+ RESET_SEQ
)
)
else:
print(COLOR_SEQ % CYAN + '<->\t' + file.saved_to + RESET_SEQ)
elif file.deleted:
print(COLOR_SEQ % MAGENTA + '-\t' + file.saved_to + RESET_SEQ)
else:
print(COLOR_SEQ % GREEN + '+\t' + file.saved_to + RESET_SEQ)
print('\n')
def run_new_token(storage_path, use_sso=False, skip_cert_verify=False):
config = ConfigHelper(storage_path)
config.load() # because we do not want to override the other settings
moodle = MoodleService(config, storage_path, skip_cert_verify)
if use_sso:
moodle.interactively_acquire_sso_token(use_stored_url=True)
else:
moodle.interactively_acquire_token(use_stored_url=True)
Log.success('New Token successfully saved!')
def _log_failures(self):
"""
Logs errors if any have occurred.
"""
print('')
if (len(self.report['failure']) > 0):
Log.warning(
'Error while trying to download files,' +
' look at the log for more details.')
for url_target in self.report['failure']:
logging.error('Error while trying to download file:' +
' %s' % (url_target))
def compute_validation_map(epoch, iteration, yolact_net, dataset, log:Log=None):
with torch.no_grad():
yolact_net.eval()
start = time.time()
print('\n')
print("Computing validation mAP (this may take a while)...", flush=True)
val_info = eval_script.evaluate(yolact_net, dataset, train_mode=True)
end = time.time()
if log is not None:
log.log('val', val_info, elapsed=(end - start), epoch=epoch, iter=iteration)
yolact_net.train()
def __init__(self, mode, key, verbose, moduleargs):
# Get args and set attributes
self.args = moduleargs
# Logger params
self.logger = Log('_server', verbose) if verbose > 0 else None
self._LOGGING_ = False if self.logger is None else True
# init Router
self.key = key
self.Router = Router(self.key, self.logger)
self.Router.start()
self.mode = mode
self.procid = 0
if self.mode == MisticaMode.SINGLE:
self.overlayname = self.args["overlay"]
self.wrappername = self.args["wrapper"]
class TuJian(object):
"""
图鉴验证类
图鉴打码地址：https://www.ttshitu.com
"""
def __init__(self, _config):
from utils.logger import Log
self.logger = Log().logger
self.username = _config["tj_username"]
self.password = _config["tj_password"]
def post_pic(self, im, type_id):
"""
提交图片
:return:
"""
base64_data = base64.b64encode(im)
b64 = base64_data.decode()
data = {
"username": self.username,
"password": self.password,
"typeid": type_id,
"image": b64
}
ret = json.loads(
requests.post("http://api.ttshitu.com/predict", json=data).text)
if ret['success']:
return ret["data"]
else:
self.logger.error(ret["message"])
sys.exit(1)
@staticmethod
def report_error(pid):
"""
提交错误图片ID
:param pid:
:return:
"""
data = {"id": pid}
ret = json.loads(
requests.post("http://api.ttshitu.com/reporterror.json",
json=data).text)
if ret['success']:
return "报错成功"
else:
return ret["message"]
def on_get(self, req, resp, mode, value):
"""
Handles the GET for search of persons and gives a list of people that match the search
Args:
req (Falcon HTTP Request): Request received by the server
resp (Falcon HTTP Response): Response constructed by the server
mode (str): The search method to be used (name, lastname, phone, email)
value (str, int): The value to be search
Returns:
None
"""
Log(f"GET Request Received", req=req)
resp.content_type = "application/json"
if mode not in getattr(fields, "field"):
resp.status = falcon.HTTP_400
resp.body = json.dumps({
"msg": "Search method not allowed"
})
else:
resp.status = falcon.HTTP_200
resp.body = json.dumps(PersonOperator.searchBy(key=mode, value=value), ensure_ascii=False)
resp.body += "\n"
def log_uncaught_exceptions(ex_cls, ex, tb):
import traceback
err_msg = ''.join(traceback.format_tb(tb))
msg = ex.message
if isinstance(msg, unicode): msg = msg.encode('utf8')
err_msg += msg
if not isinstance(err_msg, unicode):
err_msg = err_msg.decode('utf8')
err_msg += u'\n'
Log.e('UncaughtException', err_msg)
dlg = wx.MessageDialog(None,
err_msg,
u'WaterInflowPredictor出现异常',
style=wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
dlg.Destroy()
def on_delete(self, req, resp, phone):
"""
Handles the DELETE request. It deletes an specific phone number
Args:
req (Falcon HTTP Request): Request received by the server
resp (Falcon HTTP Response): Response constructed by the server
phone (str): Phone number to be deleted in the database
Returns:
None
"""
Log(f"DELETE Request Received: Searching {phone}", req=req)
resp.status = falcon.HTTP_400
resp.content_type = "application/json"
person = PersonOperator.searchBy(key="phone", value=str(phone))
if len(person) != 1:
resp.body = json.dumps(
{"msg": f"The person with the phone {phone} wasn't found"})
resp.body += "\n"
return
person = person[0]
delete = PersonOperator.delete(onPhone=str(phone))
if delete is True:
resp.status = falcon.HTTP_200
resp.body = json.dumps(
{"msg": f"{person['name']} was deleted successfully"})
else:
resp.body = json.dumps(
{"msg": f"{person['name']} couldn't be deleted"})
resp.body += "\n"
def __try_fetch_projects(cluster: dict) -> bool:
cluster_id: str = cluster.get('id')
cluster_name: str = cluster.get('name')
log: Log = Log.get_singleton()
log.info(
"Seeking for all projects inside cluster "
f"'{cluster_name}' [ID: {cluster_id}]...",
origin='Rancher')
status_code: int = 0
content: dict = {}
path: str = f"{app_config['static']['clusters']}/{cluster_id}/projects"
status_code, content = RancherMediator.__get_response(path)
if status_code != 200:
RancherMediator.__handle_bad_response(status_code, content)
return False
projects: list = []
data: list = content.get('data')
for i in range(len(data)):
project_data: dict = data[i]
project_id: str = project_data.get('id')
project_name: str = project_data.get('name')
project_links: dict = project_data.get('links')
project: dict = {
'id': project_id,
'name': project_name,
'links': project_links
}
projects.append(project)
cluster['projects'] = projects
return True
