def go_to_meeting_page(self): self.hexMeetHJTWindow.GroupControl(searchDepth=1, AutomationId="CHomeDlg.m_pWgtOperationBar").ListControl( searchDepth=2, ClassName="ev_app::custom_controls::CHomeMenuListView").Click() x, y = auto.GetCursorPos() # auto.Click(x, y - 70) # high DPI auto.Click(x, y - 55)
def click_meetingLabel(self): log.info("Click the meeting number to call the toolbar") x, y = auto.GetCursorPos() auto.SetCursorPos(x + 100, y + 100) self.meetingLabel.Click() # auto.MoveTo(x, y, 1, 0) auto.SetCursorPos(x, y)
def append_password(self, password): self.login_windows.TextControl(searchDepth=2, Name="加入会议").Click() x, y = auto.GetCursorPos() # Normal PC auto.Click(x + 55, y + 95) # Hihg DPI auto.Click(x + 45, y + 75) auto.SendKeys(password)
def return_from_reserve_meeting(self): """无法取得返回的控件,只能通过像素点""" log.info("Go back the main reserve meeting page") self.participants_page.TextControl(searchDepth=7, Name="会议基本信息:").Click() x, y = auto.GetCursorPos() auto.Click(x - 60, y - 60)
def video_restore_from_fullscreen(self): """恢复先前视频窗口""" time.sleep(10) x, y = auto.GetCursorPos() auto.Click(x - 800, y + 300) self.hjtMeetingWindow.WindowControl(searchDepth=1, AutomationId="CLayoutBackgroundDlg.CLayoutPeopleSettingDlg.CLayoutCoverDlg")\ .ButtonControl(searchDepth=4, AutomationId="CLayoutBackgroundDlg.CLayoutPeopleSettingDlg.CLayoutCoverDlg.CLayoutToolbarDlg.m_pWgtTitleBar.m_pRightWidget.m_pBtnFullScreen_Exit").Click() auto.Click(x - 100, y + 50)
def fill_in_conference_id(self, conference_id): """暂时改变不了值""" self.login_windows.TextControl(searchDepth=2, Name="加入会议").Click() x, y = auto.GetCursorPos() # Normal PC auto.Click(x + 55, y + 95) # Hihg DPI auto.Click(x + 45, y + 75) auto.SendKeys("{BACK}" * 40) auto.SendKeys(conference_id)
def show_media_statistics(self): log.info("show media statistics") self.meetingWindowControl.ButtonControl( searchDepth=2, AutomationId= "CLayoutBackgroundDlg.m_pWgtTitleBar.m_pLeftWidget.m_pBtnNetworkQuality" ).Click() # 点击完后,把鼠标移到视频,以便移到鼠标调出工具条 x, y = auto.GetCursorPos() auto.Click(x, y + 40)
def go_to_setting_page(self): self.log.info("go_to_setting_page") self.hexMeetHJTWindow.GroupControl( searchDepth=1, AutomationId="CHomeDlg.m_pWgtOperationBar").ListControl( searchDepth=2, ClassName="ev_app::custom_controls::CHomeMenuListView").Click( ) x, y = auto.GetCursorPos() auto.Click(x, y + 70)
def chang_bandwidth(self, value): """ values are 384K,512K,768K,1M,1.5M,2M,3M,4M """ self.log.info("chang_bandwidth") self.go_to_setting_page() sleep(3) self.setting_page.TextControl(searchDepth=9, Name="呼叫速率").Click() x, y = auto.GetCursorPos() auto.Click(x, y + 40) self.setting_page.ListItemControl(searchDepth=11, Name=value).Click()
def go_to_meeting_page(self): log.info("Go to the meeting page") self.hexMeetWindow.GroupControl( searchDepth=1, AutomationId="CHomeDlg.m_pWgtOperationBar").ListControl( searchDepth=2, ClassName="ev_app::custom_controls::CHomeMenuListView").Click( ) x, y = auto.GetCursorPos() # auto.Click(x, y - 70) # high DPI auto.Click(x, y - 55) sleep(4) self.click_upgrade_notice()
def uia_info(dir, namePrefix): if os.path.exists(dir) == False: os.mkdir(os.getcwd() + dir) #当前鼠标的x y绝对坐标 x, y = auto.GetCursorPos() #当前操作控件 control = currentControl = auto.ControlFromPoint(x, y) docInfo = xml.dom.minidom.Document() currentControlNode = docInfo.createElement('Control') currentControlNode.setAttribute('Name', currentControl.Name) currentControlNode.setAttribute('Type', currentControl.ControlTypeName) currentControlNode.setAttribute('ClassName', currentControl.ClassName) currentControlNode.setAttribute('AutomationId', currentControl.AutomationId) controlRect = currentControl.BoundingRectangle currentControlNode.setAttribute('RectLeft', str(controlRect.left)) currentControlNode.setAttribute('RectTop', str(controlRect.top)) currentControlNode.setAttribute('RectRight', str(controlRect.right)) currentControlNode.setAttribute('RectBottom', str(controlRect.bottom)) docInfo.appendChild(currentControlNode) fpInfo = open(dir + namePrefix + "-Info.xml", 'w') docInfo.writexml(fpInfo, encoding="utf-8") fpInfo.close() docXML = xml.dom.minidom.Document() root = docXML.createElement('Controlpath') docXML.appendChild(root) parents = [] #找父节点 while control: parents.insert(0, control) control = control.GetParentControl() i = 0 for c in parents: node = docXML.createElement('Control') node.setAttribute('Depth', str(i)) node.setAttribute('Name', c.Name) node.setAttribute('Type', c.ControlTypeName) node.setAttribute('Classname', c.ClassName) node.setAttribute('AutomationId', c.AutomationId) root.appendChild(node) i += 1 fpXML = open(dir + namePrefix + "-XML.xml", 'w') docXML.writexml(fpXML, encoding="utf-8") fpXML.close()
def terminate_call_in_full_mode(self): """结束会议""" log.info("Terminate the call in fullscreen view mode") time.sleep(10) x, y = auto.GetCursorPos() # Normal PC auto.Click(x+800, y-300) # High DPI auto.Click(x + 800, y - 300) self.meetingControlToolBar.ButtonControl( searchDepth=1, AutomationId= "CLayoutBackgroundDlg.CLayoutPeopleSettingDlg.CLayoutCoverDlg.CLayoutToolbarDlg.m_pBtnHangup" ).Click() time.sleep(2) auto.WindowControl(searchDepth=1, ClassName="ev_app::views::AlertDlg").ButtonControl( searchDepth=2, Name="离开会议").Click()
def chang_language(self, language): """ languages are English,简体中文 """ self.log.info("chang_language") self.go_to_setting_page() sleep(3) if self.setting_page.TextControl(searchDepth=9, Name="语言").Exists(): self.setting_page.TextControl(searchDepth=9, Name="语言").Click() else: self.setting_page.TextControl(searchDepth=9, Name="Language").Click() x, y = auto.GetCursorPos() auto.Click(x, y + 40) self.setting_page.ListItemControl(searchDepth=11, Name=language).Click()
def join_a_meeting_from_panel_with_password_one_line( self, password, mute="false"): self.go_to_meeting_page() sleep(10) self.hexMeetHJTWindow.GroupControl( searchDepth=1, AutomationId="CHomeDlg.m_pWgtContext").TextControl( searchDepth=9, Name="加入会议").Click() sleep(1) self.hexMeetHJTWindow.WindowControl( searchDepth=1, ClassName="ev_app::views::CJoinConfDlg").TextControl( searchDepth=2, Name="加入会议").Click() self.mute_audio(mute) self.mute_camera(mute) # 通过移动鼠标到会议号码后面并添加密码 x, y = auto.GetCursorPos() # auto.Click(x+50, y+50) # hig DPI auto.Click(x + 40, y + 40) auto.SendKeys("*" + password + "{ENTER}")
def __start_time_pos(self): self.hexMeetWindow.TextControl(searchDepth=10, Name="开始时间").Click() return auto.GetCursorPos()
def main(): import getopt auto.Logger.Write('UIAutomation {} (Python {}.{}.{}, {} bit)\n'.format( auto.VERSION, sys.version_info.major, sys.version_info.minor, sys.version_info.micro, 64 if sys.maxsize > 0xFFFFFFFF else 32)) options, args = getopt.getopt(sys.argv[1:], 'hrfcand:t:', [ 'help', 'root', 'focus', 'cursor', 'ancestor', 'showAllName', 'depth=', 'time=' ]) root = False focus = False cursor = False ancestor = False foreground = True showAllName = False depth = 0xFFFFFFFF seconds = 3 for (o, v) in options: if o in ('-h', '-help'): usage() exit(0) elif o in ('-r', '-root'): root = True foreground = False elif o in ('-f', '-focus'): focus = True foreground = False elif o in ('-c', '-cursor'): cursor = True foreground = False elif o in ('-a', '-ancestor'): ancestor = True foreground = False elif o in ('-n', '-showAllName'): showAllName = True elif o in ('-d', '-depth'): depth = int(v) elif o in ('-t', '-time'): seconds = int(v) if seconds > 0: auto.Logger.Write('please wait for {0} seconds\n\n'.format(seconds), writeToFile=False) time.sleep(seconds) auto.Logger.Log('Starts, Current Cursor Position: {}'.format( auto.GetCursorPos())) control = None if root: control = auto.GetRootControl() if focus: control = auto.GetFocusedControl() if cursor: control = auto.ControlFromCursor() if depth < 0: while depth < 0 and control: control = control.GetParentControl() depth += 1 depth = 0xFFFFFFFF if ancestor: control = auto.ControlFromCursor() if control: auto.EnumAndLogControlAncestors(control, showAllName) else: auto.Logger.Write( 'IUIAutomation returns null element under cursor\n', auto.ConsoleColor.Yellow) else: indent = 0 if not control: control = auto.GetFocusedControl() controlList = [] while control: controlList.insert(0, control) control = control.GetParentControl() if len(controlList) == 1: control = controlList[0] else: control = controlList[1] if foreground: indent = 1 auto.LogControl(controlList[0], 0, showAllName) auto.EnumAndLogControl(control, depth, showAllName, startDepth=indent) auto.Logger.Log('Ends\n')
def __meeting_duration_pos(self): self.hexMeetWindow.TextControl(searchDepth=10, Name="会议时长").Click() return auto.GetCursorPos()
def __meeting_page_click(self): self.__contact_page_click() x, y = auto.GetCursorPos() auto.Click(x, y - 70)
def return_from_person_info_page(self): log.info("Return from person info page") self.contact_page.TextControl(searchDepth=9, Name="用户详情").Click() x, y = auto.GetCursorPos() auto.Click(x - 302, y)
def click_meetingLabel(self): x, y = auto.GetCursorPos() self.meetingLabel.Click() auto.SetCursorPos(x, y)
def doStep(taskName, instanceCount=0, run_mode=False): time.sleep(1) global app global dc # global run_mode # Load Action-Object Data ACTION_OBJECT_DB_FILE = './action_object_db/action_object_db.csv' with open(ACTION_OBJECT_DB_FILE, newline='') as csvfile: action_object_data = list(csv.reader(csvfile, delimiter=',')) # print(action_object_data) # while True: # command = getCommand('What is the name of your task?') command = taskName.lower() doc = nlp(command) token_text = [token.text for token in doc] # ['This', 'is', 'a', 'text'] print(token_text) token_pos = [token.pos_ for token in doc] # ['VERB', 'DET', 'NOUN', 'PUNCT'] print(token_pos) token_dep = [token.dep_ for token in doc] # ['ROOT', 'det', 'dobj', 'punct'] print(token_dep) # if commandReceived == "switch to voice": # prompt('Switching to voice mode') # manual_mode = False # if commandReceived == "switch to manual mode": # prompt('Switching to manual mode') # manual_mode = True if command == "switch to instruction mode": prompt('Entering instruction mode...') run_mode = False instructionMode() return '', '', 0 elif command == "help": prompt( "\n\nYou can say:\nswitch to instruction mode\nwhat is this\nhelp\nexit, shut up, shutdown, goodbye, stop it\nshow instruction list\n\n\nopen a/the/my***\nclick on/the/a\nclick on it\ngo to ***\n\n" ) return '', '', 0 # if commandReceived == "listen": # commandReceived = getCommand("What do you want me to do?") # doStep(commandReceived, 0, False) elif command == "show instruction list": instructionList = os.listdir(INSTRUCTIONS_FOLDER) print('\n\nInstruction List') print('======================') for instructionIndex, taskName in enumerate(instructionList): santizedTaskName = taskName.replace('_', ' ').replace('.csv', '') print(santizedTaskName) print('\n\n') return '', '', 0 # if commandReceived == "do this": # commandReceived = getCommand("What is the name of the task?") # prompt('Finding instructions from memory...') # runInstructions(commandReceived) elif command == "what is this": currentMouseX, currentMouseY = auto.GetCursorPos() control = auto.ControlFromPoint(currentMouseX, currentMouseY) for c, d in auto.WalkControl(control, True, 1000): print(c.ControlTypeName, c.ClassName, c.AutomationId, c.BoundingRectangle, c.Name, '0x{0:X}({0})'.format(c.NativeWindowHandle)) return '', '', 0 elif command == "exit" or command == "shut up" or command == "shutdown" or command == "goodbye" or command == "stop it": prompt('Thank you for using virtual employee. Goodbye.') sys.exit() if len(token_text) > 0: verbid = -1 dobjid = -1 first_word = token_text[0].lower().strip() # override behavior during TYPE command: # if first_word == 'type' for textid, text in enumerate(token_text): if (token_pos[textid] == 'VERB' and (token_dep[textid] == 'ROOT' or token_dep[textid] == 'amod')) or (token_pos[textid] == 'ADJ' and token_dep[textid] == 'amod'): verbid = textid if token_dep[textid] == 'dobj' or token_dep[textid] == 'pobj' or ( token_pos[textid] == 'NOUN' and token_dep[textid] == 'ROOT' ) or (token_pos[textid] == 'NOUN' and token_dep[textid] == 'advcl') or (token_pos[textid] == 'VERB' and token_dep[textid] == 'advcl') or ( token_pos[textid] == 'VERB' and token_dep[textid] == 'advmod') or ( token_pos[textid] == 'VERB' and token_dep[textid] == 'xcomp' ) or (token_pos[textid] == 'VERB' and token_dep[textid] == 'pcomp'): dobjid = textid if verbid == -1: prompt( 'There is no action in your instruction step. Please start with a verb.' ) else: if dobjid == -1: prompt( 'There is no object to act on. Please state a recipient of the action.' ) else: action_from_command = str(token_text[verbid].lower().strip()) object_from_command = str(token_text[dobjid].lower().strip()) # prompt('Instruction valid.') objectInstanceCount = 1 # Counts how many times it should match before selecting the right control # Look for built-in actions first for action, dobject, subprocess_command, requirement in action_object_data: # print('token_text[verbid]',token_text[verbid]) # print('action',action) # print('token_text[dobjid]', token_text[dobjid]) # print('dobject', dobject) # print('token_text[verbid] == action', token_text[verbid] == action) # print('token_text[dobjid] == dobject', token_text[dobjid] == dobject) action_from_db = str(action.lower().strip()) object_from_db = str(dobject.lower().strip()) if action_from_command == action_from_db and object_from_command == object_from_db: # prompt('Instruction found. [' + subprocess_command + ']') if action_from_command == 'open': # print(subprocess_command) subprocess.Popen(subprocess_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) time.sleep(2) control = auto.GetRootControl() # Get Target Window detectControls = ['WindowControl', 'PaneControl'] for c, d in auto.WalkControl(control, True, 1000): if c.ControlTypeName in detectControls: # print(c.ControlTypeName, c.ClassName, c.AutomationId, c.BoundingRectangle, c.Name, '0x{0:X}({0})'.format(c.NativeWindowHandle)) if object_from_db.lower() in c.Name.lower( ): dc.Clear() dc.SetPen(wx.Pen("blue")) dc.SetBrush( wx.Brush("blue", wx.TRANSPARENT)) ControlBoundingRect = c.BoundingRectangle # TODO: calculate dynamic width/height based on screensize using wx at header xScale = 1 #1.25 yScale = 1 #1.25 width = ( ControlBoundingRect.right * xScale ) - (ControlBoundingRect.left * xScale) height = ( ControlBoundingRect.bottom * yScale ) - (ControlBoundingRect.top * yScale) dc.DrawRectangle( (ControlBoundingRect.left * xScale), (ControlBoundingRect.top * yScale), width, height) break control = getActiveWindowControls() # Then process all other commands here if action_from_command == 'go': prompt(action_from_command + ' to ' + object_from_command) control = getActiveWindowControls() # Get controls of the target window # detectControls = ['TextControl', 'ButtonControl', 'EditControl'] possibleControlMatch = [] if object_from_command == 'dropdown': possibleControlMatch = ['▾'] detectControls = [ 'HyperlinkControl', 'EditControl', 'ButtonControl', 'ComboBoxControl', 'ListItemControl', 'ListControl', 'TextControl' ] for c, d in auto.WalkControl(control, True, 1000): # print(c.ControlTypeName, c.ClassName, c.AutomationId, c.BoundingRectangle, c.Name, '0x{0:X}({0})'.format(c.NativeWindowHandle)) if object_from_command in c.Name.lower().strip( ) or object_from_command in possibleControlMatch: if c.ControlTypeName in detectControls: # if object_from_db.lower() in c.Name.lower(): dc.Clear() dc.SetPen(wx.Pen("green")) dc.SetBrush(wx.Brush("green", wx.TRANSPARENT)) ControlBoundingRect = c.BoundingRectangle xScale = 1 #1.25 yScale = 1 #1.25 width = (ControlBoundingRect.right * xScale) - (ControlBoundingRect.left * xScale) height = (ControlBoundingRect.bottom * yScale) - (ControlBoundingRect.top * yScale) dc.DrawRectangle( (ControlBoundingRect.left * xScale), (ControlBoundingRect.top * yScale), width, height) controlCenterX = int(ControlBoundingRect.left + (width / 2)) controlCenterY = int(ControlBoundingRect.top + (height / 2)) auto.SetCursorPos(controlCenterX, controlCenterY) if run_mode == False: # confirmAction = getCommand('This one?') commandReceived = '' commandReceived = getCommand("This one?") while commandReceived == '': # print('waiting...') continue print('commandReceived', commandReceived) if commandReceived in yesConfirmations: break else: objectInstanceCount += 1 continue else: if int(objectInstanceCount) == int( instanceCount): break else: objectInstanceCount += 1 continue elif action_from_command == 'click': # prompt(action_from_command + ' on ' + object_from_command) if object_from_command == 'it': currentMouseX, currentMouseY = auto.GetCursorPos() auto.Click(currentMouseX, currentMouseY) else: control = getActiveWindowControls() # Get controls of the target window # detectControls = ['TextControl', 'ButtonControl', 'EditControl'] detectControls = [ 'HyperlinkControl', 'EditControl', 'ButtonControl', 'ComboBoxControl', 'ListItemControl', 'ListControl', 'TextControl' ] for c, d in auto.WalkControl(control, True, 1000): if object_from_command in c.Name.lower().strip(): # print(c.ControlTypeName, c.ClassName, c.AutomationId, c.BoundingRectangle, c.Name, '0x{0:X}({0})'.format(c.NativeWindowHandle)) # if c.ControlTypeName in detectControls: # if object_from_db.lower() in c.Name.lower(): dc.Clear() dc.SetPen(wx.Pen("green")) dc.SetBrush(wx.Brush("green", wx.TRANSPARENT)) ControlBoundingRect = c.BoundingRectangle xScale = 1 #1.25 yScale = 1 #1.25 width = (ControlBoundingRect.right * xScale) - (ControlBoundingRect.left * xScale) height = (ControlBoundingRect.bottom * yScale) - (ControlBoundingRect.top * yScale) dc.DrawRectangle( (ControlBoundingRect.left * xScale), (ControlBoundingRect.top * yScale), width, height) controlCenterX = int(ControlBoundingRect.left + (width / 2)) controlCenterY = int(ControlBoundingRect.top + (height / 2)) auto.SetCursorPos(controlCenterX, controlCenterY) if run_mode == False: commandReceived = getCommand('This one?') if commandReceived in yesConfirmations: auto.Click(controlCenterX, controlCenterY) break else: objectInstanceCount += 1 continue else: print('objectInstanceCount', objectInstanceCount) print('instanceCount', instanceCount) if int(objectInstanceCount) == int( instanceCount): print('It goes here 3') # auto.Click(controlCenterX, controlCenterY) currentMouseX, currentMouseY = auto.GetCursorPos( ) auto.Click(currentMouseX, currentMouseY) # time.sleep(2) break else: print('It goes here 2') objectInstanceCount += 1 continue return action_from_command, object_from_command, objectInstanceCount return '', '', 0