def go_to_meeting_page(self):
self.hexMeetHJTWindow.GroupControl(searchDepth=1, AutomationId="CHomeDlg.m_pWgtOperationBar").ListControl(
searchDepth=2, ClassName="ev_app::custom_controls::CHomeMenuListView").Click()
x, y = auto.GetCursorPos()
# auto.Click(x, y - 70)
# high DPI
auto.Click(x, y - 55)
def click_meetingLabel(self):
log.info("Click the meeting number to call the toolbar")
x, y = auto.GetCursorPos()
auto.SetCursorPos(x + 100, y + 100)
self.meetingLabel.Click()
# auto.MoveTo(x, y, 1, 0)
auto.SetCursorPos(x, y)
def append_password(self, password):
self.login_windows.TextControl(searchDepth=2, Name="加入会议").Click()
x, y = auto.GetCursorPos()
# Normal PC auto.Click(x + 55, y + 95)
# Hihg DPI
auto.Click(x + 45, y + 75)
auto.SendKeys(password)
def return_from_reserve_meeting(self):
"""无法取得返回的控件，只能通过像素点"""
log.info("Go back the main reserve meeting page")
self.participants_page.TextControl(searchDepth=7,
Name="会议基本信息:").Click()
x, y = auto.GetCursorPos()
auto.Click(x - 60, y - 60)
def video_restore_from_fullscreen(self):
"""恢复先前视频窗口"""
time.sleep(10)
x, y = auto.GetCursorPos()
auto.Click(x - 800, y + 300)
self.hjtMeetingWindow.WindowControl(searchDepth=1, AutomationId="CLayoutBackgroundDlg.CLayoutPeopleSettingDlg.CLayoutCoverDlg")\
.ButtonControl(searchDepth=4, AutomationId="CLayoutBackgroundDlg.CLayoutPeopleSettingDlg.CLayoutCoverDlg.CLayoutToolbarDlg.m_pWgtTitleBar.m_pRightWidget.m_pBtnFullScreen_Exit").Click()
auto.Click(x - 100, y + 50)
def fill_in_conference_id(self, conference_id):
"""暂时改变不了值"""
self.login_windows.TextControl(searchDepth=2, Name="加入会议").Click()
x, y = auto.GetCursorPos()
# Normal PC auto.Click(x + 55, y + 95)
# Hihg DPI
auto.Click(x + 45, y + 75)
auto.SendKeys("{BACK}" * 40)
auto.SendKeys(conference_id)
def show_media_statistics(self):
log.info("show media statistics")
self.meetingWindowControl.ButtonControl(
searchDepth=2,
AutomationId=
"CLayoutBackgroundDlg.m_pWgtTitleBar.m_pLeftWidget.m_pBtnNetworkQuality"
).Click()
# 点击完后，把鼠标移到视频，以便移到鼠标调出工具条
x, y = auto.GetCursorPos()
auto.Click(x, y + 40)
def go_to_setting_page(self):
self.log.info("go_to_setting_page")
self.hexMeetHJTWindow.GroupControl(
searchDepth=1,
AutomationId="CHomeDlg.m_pWgtOperationBar").ListControl(
searchDepth=2,
ClassName="ev_app::custom_controls::CHomeMenuListView").Click(
)
x, y = auto.GetCursorPos()
auto.Click(x, y + 70)
def chang_bandwidth(self, value):
"""
values are
384K,512K,768K,1M,1.5M,2M,3M,4M
"""
self.log.info("chang_bandwidth")
self.go_to_setting_page()
sleep(3)
self.setting_page.TextControl(searchDepth=9, Name="呼叫速率").Click()
x, y = auto.GetCursorPos()
auto.Click(x, y + 40)
self.setting_page.ListItemControl(searchDepth=11, Name=value).Click()
def go_to_meeting_page(self):
log.info("Go to the meeting page")
self.hexMeetWindow.GroupControl(
searchDepth=1,
AutomationId="CHomeDlg.m_pWgtOperationBar").ListControl(
searchDepth=2,
ClassName="ev_app::custom_controls::CHomeMenuListView").Click(
)
x, y = auto.GetCursorPos()
# auto.Click(x, y - 70)
# high DPI
auto.Click(x, y - 55)
sleep(4)
self.click_upgrade_notice()
def uia_info(dir, namePrefix):
if os.path.exists(dir) == False:
os.mkdir(os.getcwd() + dir)
#当前鼠标的x y绝对坐标
x, y = auto.GetCursorPos()
#当前操作控件
control = currentControl = auto.ControlFromPoint(x, y)
docInfo = xml.dom.minidom.Document()
currentControlNode = docInfo.createElement('Control')
currentControlNode.setAttribute('Name', currentControl.Name)
currentControlNode.setAttribute('Type', currentControl.ControlTypeName)
currentControlNode.setAttribute('ClassName', currentControl.ClassName)
currentControlNode.setAttribute('AutomationId',
currentControl.AutomationId)
controlRect = currentControl.BoundingRectangle
currentControlNode.setAttribute('RectLeft', str(controlRect.left))
currentControlNode.setAttribute('RectTop', str(controlRect.top))
currentControlNode.setAttribute('RectRight', str(controlRect.right))
currentControlNode.setAttribute('RectBottom', str(controlRect.bottom))
docInfo.appendChild(currentControlNode)
fpInfo = open(dir + namePrefix + "-Info.xml", 'w')
docInfo.writexml(fpInfo, encoding="utf-8")
fpInfo.close()
docXML = xml.dom.minidom.Document()
root = docXML.createElement('Controlpath')
docXML.appendChild(root)
parents = []
#找父节点
while control:
parents.insert(0, control)
control = control.GetParentControl()
i = 0
for c in parents:
node = docXML.createElement('Control')
node.setAttribute('Depth', str(i))
node.setAttribute('Name', c.Name)
node.setAttribute('Type', c.ControlTypeName)
node.setAttribute('Classname', c.ClassName)
node.setAttribute('AutomationId', c.AutomationId)
root.appendChild(node)
i += 1
fpXML = open(dir + namePrefix + "-XML.xml", 'w')
docXML.writexml(fpXML, encoding="utf-8")
fpXML.close()
def terminate_call_in_full_mode(self):
"""结束会议"""
log.info("Terminate the call in fullscreen view mode")
time.sleep(10)
x, y = auto.GetCursorPos()
# Normal PC auto.Click(x+800, y-300)
# High DPI
auto.Click(x + 800, y - 300)
self.meetingControlToolBar.ButtonControl(
searchDepth=1,
AutomationId=
"CLayoutBackgroundDlg.CLayoutPeopleSettingDlg.CLayoutCoverDlg.CLayoutToolbarDlg.m_pBtnHangup"
).Click()
time.sleep(2)
auto.WindowControl(searchDepth=1,
ClassName="ev_app::views::AlertDlg").ButtonControl(
searchDepth=2, Name="离开会议").Click()
def chang_language(self, language):
"""
languages are
English,简体中文
"""
self.log.info("chang_language")
self.go_to_setting_page()
sleep(3)
if self.setting_page.TextControl(searchDepth=9, Name="语言").Exists():
self.setting_page.TextControl(searchDepth=9, Name="语言").Click()
else:
self.setting_page.TextControl(searchDepth=9,
Name="Language").Click()
x, y = auto.GetCursorPos()
auto.Click(x, y + 40)
self.setting_page.ListItemControl(searchDepth=11,
Name=language).Click()
def join_a_meeting_from_panel_with_password_one_line(
self, password, mute="false"):
self.go_to_meeting_page()
sleep(10)
self.hexMeetHJTWindow.GroupControl(
searchDepth=1, AutomationId="CHomeDlg.m_pWgtContext").TextControl(
searchDepth=9, Name="加入会议").Click()
sleep(1)
self.hexMeetHJTWindow.WindowControl(
searchDepth=1,
ClassName="ev_app::views::CJoinConfDlg").TextControl(
searchDepth=2, Name="加入会议").Click()
self.mute_audio(mute)
self.mute_camera(mute)
# 通过移动鼠标到会议号码后面并添加密码
x, y = auto.GetCursorPos()
# auto.Click(x+50, y+50)
# hig DPI
auto.Click(x + 40, y + 40)
auto.SendKeys("*" + password + "{ENTER}")
def __start_time_pos(self):
self.hexMeetWindow.TextControl(searchDepth=10, Name="开始时间").Click()
return auto.GetCursorPos()
def main():
import getopt
auto.Logger.Write('UIAutomation {} (Python {}.{}.{}, {} bit)\n'.format(
auto.VERSION, sys.version_info.major, sys.version_info.minor,
sys.version_info.micro, 64 if sys.maxsize > 0xFFFFFFFF else 32))
options, args = getopt.getopt(sys.argv[1:], 'hrfcand:t:', [
'help', 'root', 'focus', 'cursor', 'ancestor', 'showAllName', 'depth=',
'time='
])
root = False
focus = False
cursor = False
ancestor = False
foreground = True
showAllName = False
depth = 0xFFFFFFFF
seconds = 3
for (o, v) in options:
if o in ('-h', '-help'):
usage()
exit(0)
elif o in ('-r', '-root'):
root = True
foreground = False
elif o in ('-f', '-focus'):
focus = True
foreground = False
elif o in ('-c', '-cursor'):
cursor = True
foreground = False
elif o in ('-a', '-ancestor'):
ancestor = True
foreground = False
elif o in ('-n', '-showAllName'):
showAllName = True
elif o in ('-d', '-depth'):
depth = int(v)
elif o in ('-t', '-time'):
seconds = int(v)
if seconds > 0:
auto.Logger.Write('please wait for {0} seconds\n\n'.format(seconds),
writeToFile=False)
time.sleep(seconds)
auto.Logger.Log('Starts, Current Cursor Position: {}'.format(
auto.GetCursorPos()))
control = None
if root:
control = auto.GetRootControl()
if focus:
control = auto.GetFocusedControl()
if cursor:
control = auto.ControlFromCursor()
if depth < 0:
while depth < 0 and control:
control = control.GetParentControl()
depth += 1
depth = 0xFFFFFFFF
if ancestor:
control = auto.ControlFromCursor()
if control:
auto.EnumAndLogControlAncestors(control, showAllName)
else:
auto.Logger.Write(
'IUIAutomation returns null element under cursor\n',
auto.ConsoleColor.Yellow)
else:
indent = 0
if not control:
control = auto.GetFocusedControl()
controlList = []
while control:
controlList.insert(0, control)
control = control.GetParentControl()
if len(controlList) == 1:
control = controlList[0]
else:
control = controlList[1]
if foreground:
indent = 1
auto.LogControl(controlList[0], 0, showAllName)
auto.EnumAndLogControl(control, depth, showAllName, startDepth=indent)
auto.Logger.Log('Ends\n')
def __meeting_duration_pos(self):
self.hexMeetWindow.TextControl(searchDepth=10, Name="会议时长").Click()
return auto.GetCursorPos()
def __meeting_page_click(self):
self.__contact_page_click()
x, y = auto.GetCursorPos()
auto.Click(x, y - 70)
def return_from_person_info_page(self):
log.info("Return from person info page")
self.contact_page.TextControl(searchDepth=9, Name="用户详情").Click()
x, y = auto.GetCursorPos()
auto.Click(x - 302, y)
def click_meetingLabel(self):
x, y = auto.GetCursorPos()
self.meetingLabel.Click()
auto.SetCursorPos(x, y)
def doStep(taskName, instanceCount=0, run_mode=False):
time.sleep(1)
global app
global dc
# global run_mode
# Load Action-Object Data
ACTION_OBJECT_DB_FILE = './action_object_db/action_object_db.csv'
with open(ACTION_OBJECT_DB_FILE, newline='') as csvfile:
action_object_data = list(csv.reader(csvfile, delimiter=','))
# print(action_object_data)
# while True:
# command = getCommand('What is the name of your task?')
command = taskName.lower()
doc = nlp(command)
token_text = [token.text for token in doc]
# ['This', 'is', 'a', 'text']
print(token_text)
token_pos = [token.pos_ for token in doc]
# ['VERB', 'DET', 'NOUN', 'PUNCT']
print(token_pos)
token_dep = [token.dep_ for token in doc]
# ['ROOT', 'det', 'dobj', 'punct']
print(token_dep)
# if commandReceived == "switch to voice":
# prompt('Switching to voice mode')
# manual_mode = False
# if commandReceived == "switch to manual mode":
# prompt('Switching to manual mode')
# manual_mode = True
if command == "switch to instruction mode":
prompt('Entering instruction mode...')
run_mode = False
instructionMode()
return '', '', 0
elif command == "help":
prompt(
"\n\nYou can say:\nswitch to instruction mode\nwhat is this\nhelp\nexit, shut up, shutdown, goodbye, stop it\nshow instruction list\n\n\nopen a/the/my***\nclick on/the/a\nclick on it\ngo to ***\n\n"
)
return '', '', 0
# if commandReceived == "listen":
# commandReceived = getCommand("What do you want me to do?")
# doStep(commandReceived, 0, False)
elif command == "show instruction list":
instructionList = os.listdir(INSTRUCTIONS_FOLDER)
print('\n\nInstruction List')
print('======================')
for instructionIndex, taskName in enumerate(instructionList):
santizedTaskName = taskName.replace('_', ' ').replace('.csv', '')
print(santizedTaskName)
print('\n\n')
return '', '', 0
# if commandReceived == "do this":
# commandReceived = getCommand("What is the name of the task?")
# prompt('Finding instructions from memory...')
# runInstructions(commandReceived)
elif command == "what is this":
currentMouseX, currentMouseY = auto.GetCursorPos()
control = auto.ControlFromPoint(currentMouseX, currentMouseY)
for c, d in auto.WalkControl(control, True, 1000):
print(c.ControlTypeName, c.ClassName, c.AutomationId,
c.BoundingRectangle, c.Name,
'0x{0:X}({0})'.format(c.NativeWindowHandle))
return '', '', 0
elif command == "exit" or command == "shut up" or command == "shutdown" or command == "goodbye" or command == "stop it":
prompt('Thank you for using virtual employee. Goodbye.')
sys.exit()
if len(token_text) > 0:
verbid = -1
dobjid = -1
first_word = token_text[0].lower().strip()
# override behavior during TYPE command:
# if first_word == 'type'
for textid, text in enumerate(token_text):
if (token_pos[textid] == 'VERB' and
(token_dep[textid] == 'ROOT' or token_dep[textid]
== 'amod')) or (token_pos[textid] == 'ADJ'
and token_dep[textid] == 'amod'):
verbid = textid
if token_dep[textid] == 'dobj' or token_dep[textid] == 'pobj' or (
token_pos[textid] == 'NOUN' and token_dep[textid] == 'ROOT'
) or (token_pos[textid] == 'NOUN' and token_dep[textid]
== 'advcl') or (token_pos[textid] == 'VERB'
and token_dep[textid] == 'advcl') or (
token_pos[textid] == 'VERB'
and token_dep[textid] == 'advmod') or (
token_pos[textid] == 'VERB'
and token_dep[textid] == 'xcomp'
) or (token_pos[textid] == 'VERB'
and token_dep[textid] == 'pcomp'):
dobjid = textid
if verbid == -1:
prompt(
'There is no action in your instruction step. Please start with a verb.'
)
else:
if dobjid == -1:
prompt(
'There is no object to act on. Please state a recipient of the action.'
)
else:
action_from_command = str(token_text[verbid].lower().strip())
object_from_command = str(token_text[dobjid].lower().strip())
# prompt('Instruction valid.')
objectInstanceCount = 1 # Counts how many times it should match before selecting the right control
# Look for built-in actions first
for action, dobject, subprocess_command, requirement in action_object_data:
# print('token_text[verbid]',token_text[verbid])
# print('action',action)
# print('token_text[dobjid]', token_text[dobjid])
# print('dobject', dobject)
# print('token_text[verbid] == action', token_text[verbid] == action)
# print('token_text[dobjid] == dobject', token_text[dobjid] == dobject)
action_from_db = str(action.lower().strip())
object_from_db = str(dobject.lower().strip())
if action_from_command == action_from_db and object_from_command == object_from_db:
# prompt('Instruction found. [' + subprocess_command + ']')
if action_from_command == 'open':
# print(subprocess_command)
subprocess.Popen(subprocess_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
time.sleep(2)
control = auto.GetRootControl()
# Get Target Window
detectControls = ['WindowControl', 'PaneControl']
for c, d in auto.WalkControl(control, True, 1000):
if c.ControlTypeName in detectControls:
# print(c.ControlTypeName, c.ClassName, c.AutomationId, c.BoundingRectangle, c.Name, '0x{0:X}({0})'.format(c.NativeWindowHandle))
if object_from_db.lower() in c.Name.lower(
):
dc.Clear()
dc.SetPen(wx.Pen("blue"))
dc.SetBrush(
wx.Brush("blue", wx.TRANSPARENT))
ControlBoundingRect = c.BoundingRectangle
# TODO: calculate dynamic width/height based on screensize using wx at header
xScale = 1 #1.25
yScale = 1 #1.25
width = (
ControlBoundingRect.right * xScale
) - (ControlBoundingRect.left * xScale)
height = (
ControlBoundingRect.bottom * yScale
) - (ControlBoundingRect.top * yScale)
dc.DrawRectangle(
(ControlBoundingRect.left *
xScale),
(ControlBoundingRect.top * yScale),
width, height)
break
control = getActiveWindowControls()
# Then process all other commands here
if action_from_command == 'go':
prompt(action_from_command + ' to ' + object_from_command)
control = getActiveWindowControls()
# Get controls of the target window
# detectControls = ['TextControl', 'ButtonControl', 'EditControl']
possibleControlMatch = []
if object_from_command == 'dropdown':
possibleControlMatch = ['▾']
detectControls = [
'HyperlinkControl', 'EditControl', 'ButtonControl',
'ComboBoxControl', 'ListItemControl', 'ListControl',
'TextControl'
]
for c, d in auto.WalkControl(control, True, 1000):
# print(c.ControlTypeName, c.ClassName, c.AutomationId, c.BoundingRectangle, c.Name, '0x{0:X}({0})'.format(c.NativeWindowHandle))
if object_from_command in c.Name.lower().strip(
) or object_from_command in possibleControlMatch:
if c.ControlTypeName in detectControls:
# if object_from_db.lower() in c.Name.lower():
dc.Clear()
dc.SetPen(wx.Pen("green"))
dc.SetBrush(wx.Brush("green", wx.TRANSPARENT))
ControlBoundingRect = c.BoundingRectangle
xScale = 1 #1.25
yScale = 1 #1.25
width = (ControlBoundingRect.right *
xScale) - (ControlBoundingRect.left *
xScale)
height = (ControlBoundingRect.bottom *
yScale) - (ControlBoundingRect.top *
yScale)
dc.DrawRectangle(
(ControlBoundingRect.left * xScale),
(ControlBoundingRect.top * yScale), width,
height)
controlCenterX = int(ControlBoundingRect.left +
(width / 2))
controlCenterY = int(ControlBoundingRect.top +
(height / 2))
auto.SetCursorPos(controlCenterX,
controlCenterY)
if run_mode == False:
# confirmAction = getCommand('This one?')
commandReceived = ''
commandReceived = getCommand("This one?")
while commandReceived == '':
# print('waiting...')
continue
print('commandReceived', commandReceived)
if commandReceived in yesConfirmations:
break
else:
objectInstanceCount += 1
continue
else:
if int(objectInstanceCount) == int(
instanceCount):
break
else:
objectInstanceCount += 1
continue
elif action_from_command == 'click':
# prompt(action_from_command + ' on ' + object_from_command)
if object_from_command == 'it':
currentMouseX, currentMouseY = auto.GetCursorPos()
auto.Click(currentMouseX, currentMouseY)
else:
control = getActiveWindowControls()
# Get controls of the target window
# detectControls = ['TextControl', 'ButtonControl', 'EditControl']
detectControls = [
'HyperlinkControl', 'EditControl', 'ButtonControl',
'ComboBoxControl', 'ListItemControl',
'ListControl', 'TextControl'
]
for c, d in auto.WalkControl(control, True, 1000):
if object_from_command in c.Name.lower().strip():
# print(c.ControlTypeName, c.ClassName, c.AutomationId, c.BoundingRectangle, c.Name, '0x{0:X}({0})'.format(c.NativeWindowHandle))
# if c.ControlTypeName in detectControls:
# if object_from_db.lower() in c.Name.lower():
dc.Clear()
dc.SetPen(wx.Pen("green"))
dc.SetBrush(wx.Brush("green", wx.TRANSPARENT))
ControlBoundingRect = c.BoundingRectangle
xScale = 1 #1.25
yScale = 1 #1.25
width = (ControlBoundingRect.right *
xScale) - (ControlBoundingRect.left *
xScale)
height = (ControlBoundingRect.bottom *
yScale) - (ControlBoundingRect.top *
yScale)
dc.DrawRectangle(
(ControlBoundingRect.left * xScale),
(ControlBoundingRect.top * yScale), width,
height)
controlCenterX = int(ControlBoundingRect.left +
(width / 2))
controlCenterY = int(ControlBoundingRect.top +
(height / 2))
auto.SetCursorPos(controlCenterX,
controlCenterY)
if run_mode == False:
commandReceived = getCommand('This one?')
if commandReceived in yesConfirmations:
auto.Click(controlCenterX,
controlCenterY)
break
else:
objectInstanceCount += 1
continue
else:
print('objectInstanceCount',
objectInstanceCount)
print('instanceCount', instanceCount)
if int(objectInstanceCount) == int(
instanceCount):
print('It goes here 3')
# auto.Click(controlCenterX, controlCenterY)
currentMouseX, currentMouseY = auto.GetCursorPos(
)
auto.Click(currentMouseX,
currentMouseY)
# time.sleep(2)
break
else:
print('It goes here 2')
objectInstanceCount += 1
continue
return action_from_command, object_from_command, objectInstanceCount
return '', '', 0
