def choose_time(self):
log.info("Change time")
x, y = self.__start_time_pos()
auto.Click(x + 250, y)
auto.SetCursorPos(x + 250, y + 80)
auto.WheelDown(3)
auto.Click(x + 500, y)
def choose_duration(self):
log.info("Change duration")
x, y = self.__meeting_duration_pos()
auto.Click(x + 160, y)
auto.SetCursorPos(x + 120, y + 100)
auto.WheelDown(3)
sleep(1)
auto.Click(x + 500, y)
def video_restore_from_fullscreen(self):
"""恢复先前视频窗口"""
time.sleep(10)
x, y = auto.GetCursorPos()
auto.Click(x - 800, y + 300)
self.hjtMeetingWindow.WindowControl(searchDepth=1, AutomationId="CLayoutBackgroundDlg.CLayoutPeopleSettingDlg.CLayoutCoverDlg")\
.ButtonControl(searchDepth=4, AutomationId="CLayoutBackgroundDlg.CLayoutPeopleSettingDlg.CLayoutCoverDlg.CLayoutToolbarDlg.m_pWgtTitleBar.m_pRightWidget.m_pBtnFullScreen_Exit").Click()
auto.Click(x - 100, y + 50)
def go_to_meeting_page(self):
self.hexMeetHJTWindow.GroupControl(searchDepth=1, AutomationId="CHomeDlg.m_pWgtOperationBar").ListControl(
searchDepth=2, ClassName="ev_app::custom_controls::CHomeMenuListView").Click()
x, y = auto.GetCursorPos()
# auto.Click(x, y - 70)
# high DPI
auto.Click(x, y - 55)
def save_content(LAST_MESS_TEMP):
""" 复制聊天记录到指定文件中
:return:
"""
dialog_box = auto.WindowControl(Name='中泰证券二部技术讨论', searchDepth=1)
dialog_box.SetActive()
dialog_box.Maximize()
message_win = dialog_box.ListControl(Name='消息', searchDepth=13)
message_win.Click()
auto.Click(800, 800)
message_win.SendKeys('{Ctrl}A')
message_win.SendKeys('{Ctrl}C')
df = pd.read_clipboard(sep=r"\s+", encoding='utf-8', error_bad_lines=False)
df.to_csv('message_tmp.txt', index=False, sep=' ', encoding='utf_8_sig')
# 查找未写入文件的内容
k = 0
with open('message_tmp.txt', 'r', encoding='utf_8_sig') as fp:
readlines = fp.readlines()
for i, line in enumerate(readlines):
if line == LAST_MESS_TEMP:
k = i
break
LAST_MESS_TEMP = readlines[-1].strip()
# 将未写入文件的内容写入文件
with open('message.txt', 'a+', encoding='utf_8_sig') as fp:
readlines = fp.readlines()
for i in range(k, len(readlines)):
fp.write(readlines[i])
return LAST_MESS_TEMP
def return_from_reserve_meeting(self):
"""无法取得返回的控件，只能通过像素点"""
log.info("Go back the main reserve meeting page")
self.participants_page.TextControl(searchDepth=7,
Name="会议基本信息:").Click()
x, y = auto.GetCursorPos()
auto.Click(x - 60, y - 60)
def append_password(self, password):
self.login_windows.TextControl(searchDepth=2, Name="加入会议").Click()
x, y = auto.GetCursorPos()
# Normal PC auto.Click(x + 55, y + 95)
# Hihg DPI
auto.Click(x + 45, y + 75)
auto.SendKeys(password)
def fill_in_conference_id(self, conference_id):
"""暂时改变不了值"""
self.login_windows.TextControl(searchDepth=2, Name="加入会议").Click()
x, y = auto.GetCursorPos()
# Normal PC auto.Click(x + 55, y + 95)
# Hihg DPI
auto.Click(x + 45, y + 75)
auto.SendKeys("{BACK}" * 40)
auto.SendKeys(conference_id)
def go_to_setting_page(self):
self.log.info("go_to_setting_page")
self.hexMeetHJTWindow.GroupControl(
searchDepth=1,
AutomationId="CHomeDlg.m_pWgtOperationBar").ListControl(
searchDepth=2,
ClassName="ev_app::custom_controls::CHomeMenuListView").Click(
)
x, y = auto.GetCursorPos()
auto.Click(x, y + 70)
def show_media_statistics(self):
log.info("show media statistics")
self.meetingWindowControl.ButtonControl(
searchDepth=2,
AutomationId=
"CLayoutBackgroundDlg.m_pWgtTitleBar.m_pLeftWidget.m_pBtnNetworkQuality"
).Click()
# 点击完后，把鼠标移到视频，以便移到鼠标调出工具条
x, y = auto.GetCursorPos()
auto.Click(x, y + 40)
def chang_bandwidth(self, value):
"""
values are
384K,512K,768K,1M,1.5M,2M,3M,4M
"""
self.log.info("chang_bandwidth")
self.go_to_setting_page()
sleep(3)
self.setting_page.TextControl(searchDepth=9, Name="呼叫速率").Click()
x, y = auto.GetCursorPos()
auto.Click(x, y + 40)
self.setting_page.ListItemControl(searchDepth=11, Name=value).Click()
def go_to_meeting_page(self):
log.info("Go to the meeting page")
self.hexMeetWindow.GroupControl(
searchDepth=1,
AutomationId="CHomeDlg.m_pWgtOperationBar").ListControl(
searchDepth=2,
ClassName="ev_app::custom_controls::CHomeMenuListView").Click(
)
x, y = auto.GetCursorPos()
# auto.Click(x, y - 70)
# high DPI
auto.Click(x, y - 55)
sleep(4)
self.click_upgrade_notice()
def monitor_click(qq_button):
""" 模拟点击操作
:param qq_button:
:return:
"""
qq_button.SetFocus()
qq_button.SendKeys('{Enter}')
qq_button.SendKeys('{Enter}')
qq_button.SendKeys('{Enter}')
qq_button.Click(waitTime=1.5)
x, y = win32gui.GetCursorPos()
print(x, y)
qq_button.SendKeys('{Enter}')
auto.Click(1696, 1080)
def SaveButton(self):
while 1:
try:
locating = auto.DataGridControl(
AutomationId='DataGrid1').HeaderControl(
).HeaderItemControl().TextControl()
locating_x = locating.GetClickablePoint()[0]
locating_y = locating.GetClickablePoint()[1] - 40
auto.Click(locating_x, locating_y)
self._log.info(f"点击保存按钮")
break
except Exception as e:
tab = auto.TabControl(AutomationId='mainTab').TabItemControl(
Name='emd.ViewModel.ColGridViewModel')
tab.DoubleClick()
self._log.error(f"输入对象丢失，正在重新查找，报错信息为{e}")
continue
def terminate_call_in_full_mode(self):
"""结束会议"""
log.info("Terminate the call in fullscreen view mode")
time.sleep(10)
x, y = auto.GetCursorPos()
# Normal PC auto.Click(x+800, y-300)
# High DPI
auto.Click(x + 800, y - 300)
self.meetingControlToolBar.ButtonControl(
searchDepth=1,
AutomationId=
"CLayoutBackgroundDlg.CLayoutPeopleSettingDlg.CLayoutCoverDlg.CLayoutToolbarDlg.m_pBtnHangup"
).Click()
time.sleep(2)
auto.WindowControl(searchDepth=1,
ClassName="ev_app::views::AlertDlg").ButtonControl(
searchDepth=2, Name="离开会议").Click()
def chang_language(self, language):
"""
languages are
English,简体中文
"""
self.log.info("chang_language")
self.go_to_setting_page()
sleep(3)
if self.setting_page.TextControl(searchDepth=9, Name="语言").Exists():
self.setting_page.TextControl(searchDepth=9, Name="语言").Click()
else:
self.setting_page.TextControl(searchDepth=9,
Name="Language").Click()
x, y = auto.GetCursorPos()
auto.Click(x, y + 40)
self.setting_page.ListItemControl(searchDepth=11,
Name=language).Click()
def join_a_meeting_from_panel_with_password_one_line(
self, password, mute="false"):
self.go_to_meeting_page()
sleep(10)
self.hexMeetHJTWindow.GroupControl(
searchDepth=1, AutomationId="CHomeDlg.m_pWgtContext").TextControl(
searchDepth=9, Name="加入会议").Click()
sleep(1)
self.hexMeetHJTWindow.WindowControl(
searchDepth=1,
ClassName="ev_app::views::CJoinConfDlg").TextControl(
searchDepth=2, Name="加入会议").Click()
self.mute_audio(mute)
self.mute_camera(mute)
# 通过移动鼠标到会议号码后面并添加密码
x, y = auto.GetCursorPos()
# auto.Click(x+50, y+50)
# hig DPI
auto.Click(x + 40, y + 40)
auto.SendKeys("*" + password + "{ENTER}")
def get_new_msg(self):
self.dialog_box.SetActive()
self.dialog_box.Maximize()
message_win = self.dialog_box.ListControl(Name='消息', searchDepth=13)
message_win.Click()
auto.Click(800, 800)
message_win.SendKeys('{Ctrl}A')
message_win.SendKeys('{Ctrl}C')
df = pd.read_clipboard(sep=r"\s+", encoding='utf-8', error_bad_lines=False)
if not os.path.exists("message_tmp.txt"):
df.to_csv('message_tmp.txt', index=False, sep=' ', encoding='utf_8_sig')
with open("message_tmp.txt", "r", encoding='utf_8_sig') as f:
ori_lines = f.readlines()
df.to_csv('message_tmp.txt', index=False, sep=' ', encoding='utf_8_sig')
with open("message_tmp.txt", "r", encoding='utf_8_sig') as f:
cur_lines = f.readlines()
new_lines = []
for line in cur_lines:
if line not in ori_lines:
new_lines.append(line)
return new_lines
def choose_date(self):
log.info("Change date")
x, y = self.__start_time_pos()
auto.Click(x + 160, y)
def __meeting_page_click(self):
self.__contact_page_click()
x, y = auto.GetCursorPos()
auto.Click(x, y - 70)
def ClickByXY(self, X, Y):
try:
sleep(2)
automation.Click(X, Y)
except Exception as e:
print("坐标点击异常" + '||原因：' + str(e))
def doStep(taskName, instanceCount=0, run_mode=False):
time.sleep(1)
global app
global dc
# global run_mode
# Load Action-Object Data
ACTION_OBJECT_DB_FILE = './action_object_db/action_object_db.csv'
with open(ACTION_OBJECT_DB_FILE, newline='') as csvfile:
action_object_data = list(csv.reader(csvfile, delimiter=','))
# print(action_object_data)
# while True:
# command = getCommand('What is the name of your task?')
command = taskName.lower()
doc = nlp(command)
token_text = [token.text for token in doc]
# ['This', 'is', 'a', 'text']
print(token_text)
token_pos = [token.pos_ for token in doc]
# ['VERB', 'DET', 'NOUN', 'PUNCT']
print(token_pos)
token_dep = [token.dep_ for token in doc]
# ['ROOT', 'det', 'dobj', 'punct']
print(token_dep)
# if commandReceived == "switch to voice":
# prompt('Switching to voice mode')
# manual_mode = False
# if commandReceived == "switch to manual mode":
# prompt('Switching to manual mode')
# manual_mode = True
if command == "switch to instruction mode":
prompt('Entering instruction mode...')
run_mode = False
instructionMode()
return '', '', 0
elif command == "help":
prompt(
"\n\nYou can say:\nswitch to instruction mode\nwhat is this\nhelp\nexit, shut up, shutdown, goodbye, stop it\nshow instruction list\n\n\nopen a/the/my***\nclick on/the/a\nclick on it\ngo to ***\n\n"
)
return '', '', 0
# if commandReceived == "listen":
# commandReceived = getCommand("What do you want me to do?")
# doStep(commandReceived, 0, False)
elif command == "show instruction list":
instructionList = os.listdir(INSTRUCTIONS_FOLDER)
print('\n\nInstruction List')
print('======================')
for instructionIndex, taskName in enumerate(instructionList):
santizedTaskName = taskName.replace('_', ' ').replace('.csv', '')
print(santizedTaskName)
print('\n\n')
return '', '', 0
# if commandReceived == "do this":
# commandReceived = getCommand("What is the name of the task?")
# prompt('Finding instructions from memory...')
# runInstructions(commandReceived)
elif command == "what is this":
currentMouseX, currentMouseY = auto.GetCursorPos()
control = auto.ControlFromPoint(currentMouseX, currentMouseY)
for c, d in auto.WalkControl(control, True, 1000):
print(c.ControlTypeName, c.ClassName, c.AutomationId,
c.BoundingRectangle, c.Name,
'0x{0:X}({0})'.format(c.NativeWindowHandle))
return '', '', 0
elif command == "exit" or command == "shut up" or command == "shutdown" or command == "goodbye" or command == "stop it":
prompt('Thank you for using virtual employee. Goodbye.')
sys.exit()
if len(token_text) > 0:
verbid = -1
dobjid = -1
first_word = token_text[0].lower().strip()
# override behavior during TYPE command:
# if first_word == 'type'
for textid, text in enumerate(token_text):
if (token_pos[textid] == 'VERB' and
(token_dep[textid] == 'ROOT' or token_dep[textid]
== 'amod')) or (token_pos[textid] == 'ADJ'
and token_dep[textid] == 'amod'):
verbid = textid
if token_dep[textid] == 'dobj' or token_dep[textid] == 'pobj' or (
token_pos[textid] == 'NOUN' and token_dep[textid] == 'ROOT'
) or (token_pos[textid] == 'NOUN' and token_dep[textid]
== 'advcl') or (token_pos[textid] == 'VERB'
and token_dep[textid] == 'advcl') or (
token_pos[textid] == 'VERB'
and token_dep[textid] == 'advmod') or (
token_pos[textid] == 'VERB'
and token_dep[textid] == 'xcomp'
) or (token_pos[textid] == 'VERB'
and token_dep[textid] == 'pcomp'):
dobjid = textid
if verbid == -1:
prompt(
'There is no action in your instruction step. Please start with a verb.'
)
else:
if dobjid == -1:
prompt(
'There is no object to act on. Please state a recipient of the action.'
)
else:
action_from_command = str(token_text[verbid].lower().strip())
object_from_command = str(token_text[dobjid].lower().strip())
# prompt('Instruction valid.')
objectInstanceCount = 1 # Counts how many times it should match before selecting the right control
# Look for built-in actions first
for action, dobject, subprocess_command, requirement in action_object_data:
# print('token_text[verbid]',token_text[verbid])
# print('action',action)
# print('token_text[dobjid]', token_text[dobjid])
# print('dobject', dobject)
# print('token_text[verbid] == action', token_text[verbid] == action)
# print('token_text[dobjid] == dobject', token_text[dobjid] == dobject)
action_from_db = str(action.lower().strip())
object_from_db = str(dobject.lower().strip())
if action_from_command == action_from_db and object_from_command == object_from_db:
# prompt('Instruction found. [' + subprocess_command + ']')
if action_from_command == 'open':
# print(subprocess_command)
subprocess.Popen(subprocess_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
time.sleep(2)
control = auto.GetRootControl()
# Get Target Window
detectControls = ['WindowControl', 'PaneControl']
for c, d in auto.WalkControl(control, True, 1000):
if c.ControlTypeName in detectControls:
# print(c.ControlTypeName, c.ClassName, c.AutomationId, c.BoundingRectangle, c.Name, '0x{0:X}({0})'.format(c.NativeWindowHandle))
if object_from_db.lower() in c.Name.lower(
):
dc.Clear()
dc.SetPen(wx.Pen("blue"))
dc.SetBrush(
wx.Brush("blue", wx.TRANSPARENT))
ControlBoundingRect = c.BoundingRectangle
# TODO: calculate dynamic width/height based on screensize using wx at header
xScale = 1 #1.25
yScale = 1 #1.25
width = (
ControlBoundingRect.right * xScale
) - (ControlBoundingRect.left * xScale)
height = (
ControlBoundingRect.bottom * yScale
) - (ControlBoundingRect.top * yScale)
dc.DrawRectangle(
(ControlBoundingRect.left *
xScale),
(ControlBoundingRect.top * yScale),
width, height)
break
control = getActiveWindowControls()
# Then process all other commands here
if action_from_command == 'go':
prompt(action_from_command + ' to ' + object_from_command)
control = getActiveWindowControls()
# Get controls of the target window
# detectControls = ['TextControl', 'ButtonControl', 'EditControl']
possibleControlMatch = []
if object_from_command == 'dropdown':
possibleControlMatch = ['▾']
detectControls = [
'HyperlinkControl', 'EditControl', 'ButtonControl',
'ComboBoxControl', 'ListItemControl', 'ListControl',
'TextControl'
]
for c, d in auto.WalkControl(control, True, 1000):
# print(c.ControlTypeName, c.ClassName, c.AutomationId, c.BoundingRectangle, c.Name, '0x{0:X}({0})'.format(c.NativeWindowHandle))
if object_from_command in c.Name.lower().strip(
) or object_from_command in possibleControlMatch:
if c.ControlTypeName in detectControls:
# if object_from_db.lower() in c.Name.lower():
dc.Clear()
dc.SetPen(wx.Pen("green"))
dc.SetBrush(wx.Brush("green", wx.TRANSPARENT))
ControlBoundingRect = c.BoundingRectangle
xScale = 1 #1.25
yScale = 1 #1.25
width = (ControlBoundingRect.right *
xScale) - (ControlBoundingRect.left *
xScale)
height = (ControlBoundingRect.bottom *
yScale) - (ControlBoundingRect.top *
yScale)
dc.DrawRectangle(
(ControlBoundingRect.left * xScale),
(ControlBoundingRect.top * yScale), width,
height)
controlCenterX = int(ControlBoundingRect.left +
(width / 2))
controlCenterY = int(ControlBoundingRect.top +
(height / 2))
auto.SetCursorPos(controlCenterX,
controlCenterY)
if run_mode == False:
# confirmAction = getCommand('This one?')
commandReceived = ''
commandReceived = getCommand("This one?")
while commandReceived == '':
# print('waiting...')
continue
print('commandReceived', commandReceived)
if commandReceived in yesConfirmations:
break
else:
objectInstanceCount += 1
continue
else:
if int(objectInstanceCount) == int(
instanceCount):
break
else:
objectInstanceCount += 1
continue
elif action_from_command == 'click':
# prompt(action_from_command + ' on ' + object_from_command)
if object_from_command == 'it':
currentMouseX, currentMouseY = auto.GetCursorPos()
auto.Click(currentMouseX, currentMouseY)
else:
control = getActiveWindowControls()
# Get controls of the target window
# detectControls = ['TextControl', 'ButtonControl', 'EditControl']
detectControls = [
'HyperlinkControl', 'EditControl', 'ButtonControl',
'ComboBoxControl', 'ListItemControl',
'ListControl', 'TextControl'
]
for c, d in auto.WalkControl(control, True, 1000):
if object_from_command in c.Name.lower().strip():
# print(c.ControlTypeName, c.ClassName, c.AutomationId, c.BoundingRectangle, c.Name, '0x{0:X}({0})'.format(c.NativeWindowHandle))
# if c.ControlTypeName in detectControls:
# if object_from_db.lower() in c.Name.lower():
dc.Clear()
dc.SetPen(wx.Pen("green"))
dc.SetBrush(wx.Brush("green", wx.TRANSPARENT))
ControlBoundingRect = c.BoundingRectangle
xScale = 1 #1.25
yScale = 1 #1.25
width = (ControlBoundingRect.right *
xScale) - (ControlBoundingRect.left *
xScale)
height = (ControlBoundingRect.bottom *
yScale) - (ControlBoundingRect.top *
yScale)
dc.DrawRectangle(
(ControlBoundingRect.left * xScale),
(ControlBoundingRect.top * yScale), width,
height)
controlCenterX = int(ControlBoundingRect.left +
(width / 2))
controlCenterY = int(ControlBoundingRect.top +
(height / 2))
auto.SetCursorPos(controlCenterX,
controlCenterY)
if run_mode == False:
commandReceived = getCommand('This one?')
if commandReceived in yesConfirmations:
auto.Click(controlCenterX,
controlCenterY)
break
else:
objectInstanceCount += 1
continue
else:
print('objectInstanceCount',
objectInstanceCount)
print('instanceCount', instanceCount)
if int(objectInstanceCount) == int(
instanceCount):
print('It goes here 3')
# auto.Click(controlCenterX, controlCenterY)
currentMouseX, currentMouseY = auto.GetCursorPos(
)
auto.Click(currentMouseX,
currentMouseY)
# time.sleep(2)
break
else:
print('It goes here 2')
objectInstanceCount += 1
continue
return action_from_command, object_from_command, objectInstanceCount
return '', '', 0
def click_by_position(self, x, y):
try:
auto.Click(x, y)
logger.info("x坐标_{}_，y坐标_{}_点击成功".format(x, y))
except:
logger.exception("x坐标_{}_，y坐标_{}_点击失败".format(x, y))
def return_from_person_info_page(self):
log.info("Return from person info page")
self.contact_page.TextControl(searchDepth=9, Name="用户详情").Click()
x, y = auto.GetCursorPos()
auto.Click(x - 302, y)
name = "test_" + str(num)
auto.EditControl(searchDepth=4,
AutomationId="1021").GetValuePattern().SetValue(name)
auto.PaneControl(searchDepth=4,
AutomationId="1142").GetFirstChildControl().Click()
auto.SendKey(0x25)
auto.SendKey(0x0D)
auto.EditControl(searchDepth=5,
AutomationId="1001").GetValuePattern().SetValue("候志明")
auto.ButtonControl(searchDepth=4, Name="确定").Click()
auto.ButtonControl(searchDepth=4, Name=" 请选择X光片 ").Click()
time.sleep(1.5)
auto.EditControl(searchDepth=6,
Name="文件名(N):").GetValuePattern().SetValue(c_path_name)
auto.SendKey(0x0D)
auto.Click(800, 500)
auto.Click(500, 800)
auto.ButtonControl(searchDepth=4,
Name=" 请选择X光片 ").GetNextSiblingControl().Click()
time.sleep(1.5)
auto.EditControl(searchDepth=6,
Name="文件名(N):").GetValuePattern().SetValue(z_path_name)
auto.SendKey(0x0D)
auto.SendKey(0x0D)
auto.Click(600, 400)
auto.Click(300, 600)
auto.ButtonControl(searchDepth=4, Name=" 患者列表 ").Click()
search_patient_name = auto.ButtonControl(searchDepth=4, Name="查找患者姓名")
if not search_patient_name.Exists(maxSearchSeconds=60):
print("查找失败")
auto.TextControl(searchDepth=6, Name=patient_number).DoubleClick()
