def __init__(self, mdFilename, templateFilename, output): logger.verbose("Project initialization : %s, %s, %s" % (mdFilename, templateFilename, output)) self._mdFilename = mdFilename with open(mdFilename, 'r') as f: self._rawMd = f.read() self.md = Document(self._rawMd) self._templateFilename = templateFilename self.template = DocxTemplate(templateFilename) self.parsedTree = Parser(self.md, self.template, mdFilename) logger.debug("Parsed Tree:\n" + str(self.parsedTree)) context = self.parsedTree.generateDocx() logger.spam("context: %s" % context) logger.info('First rendering start.') self.template.render(context) self.template.save(output + self.TMP_FILENAME) logger.info('Second rendering start.') self.template = DocxTemplate(output + self.TMP_FILENAME) self.parsedTree.finalize(self.template) self.template.render(context) self.template.save(output) logger.info('Full rendering finished.') os.remove(output + self.TMP_FILENAME)
def run(self): factory = VoteCollectorFactory(self) port = reactor.listenTCP(app.config.getInt('collector-port'), factory) logger.verbose("Collector: Awaiting connections on %s" % port.getHost()) stdio.StandardIO(Console(self)) reactor.run()
def _setIndex(self, message): try: encPair = message['pair'] cert = message['certificate'] try: sessionKey = self.factory.authNode.sessionKeys[cert] except KeyError: return self.factory.fail("Requested to set index for an unregistered certificate") pairString = sessionKey.decrypt(encPair) try: # logger.special(pairString) #pairString = pairString[0:pairString.index('}')+1] pair = json.loads(pairString) except ValueError as e: logger.warning("Invalid JSON as cert/index pair, discarding message...") logger.warning(e.message, False) return if pair['certificate'] != cert: logger.warning("Encrypted certificate doesn't match client's certificte, discarding message...") return logger.info("Index set for client") logger.verbose("Index: %s" % pair['index'], False) logger.verbose("Session Key: %s" % sessionKey.hex(), False) self.factory.authNode.indices[cert] = pair['index'] logger.split() except KeyError as e: return self.factory.fail("Get-Session-Key no: '%s' field found in message" % e)
def messageReceived(self, message): encTable = message['encrypted-table'] tableString = self.factory.collectorNode.authKey.decrypt(encTable) print(tableString) table = json.loads(tableString) logger.verbose("Table received from Authority") self.factory.collectorNode.countVotes(table)
def run(self): from twisted.internet import reactor factory = CAFactory() port = reactor.listenTCP(app.config.getInt('ca-port'), factory) logger.verbose("Awaiting connections on %s" % port.getHost()) reactor.run()
def endElections(self): host = app.config.get("authority-host") port = app.config.get("authority-port") factory = AuthorityFactory(self) logger.split() logger.verbose("Connecting to Authority...") reactor.connectTCP(host, port, factory)
def getCertificateFromCA(self): host = app.config.get("ca-host") port = app.config.get("ca-port") d = Deferred() factory = CertFactory(d) logger.verbose("Connecting to CA...") reactor.connectTCP(host, port, factory) return d
def _sendKey(self): myKey = app.keyManager.getMyKey() myPublicKey = myKey.publickey() message = SecureMessage() message['public-key'] = myPublicKey.pem() logger.verbose("Sending Public Key to CA...") logger.debug(message) self.sendMessage(message)
def messageReceived(self, message): logger.verbose("Message with action '%s' received from %s" % (message.action, message.sender)) if message.action == 'get-session-key': self._getSessionKey(message) elif message.action == 'set-index': self._setIndex(message) elif message.action == 'request-table': self._requestTable(message) else: return self.factory.fail("Unrecognized action in received message: '%s'" % message.action)
def sendCertToAuth(self, cert): host = app.config.get("authority-host") port = app.config.get("authority-port") d = Deferred() factory = Authority2Factory(cert, d) logger.split() logger.verbose("Connecting to Authority...") reactor.connectTCP(host, port, factory) return d
def _dfs(self, depth): logger.debug('Into DFS depth {}'.format(depth)) # Found destination path = [e.info for e in self.visiting_elements] if self._dest_keywords and self.wait_for_destination(self._dest_keywords, timeout=10): logger.info(u'Destination keywords matched') return {'status': 'success', 'path': path, 'package': self.package} if self._dest_activities and self.wait_for_activities(self._dest_activities, timeout=10): logger.info(u'Destination activities matched.') return {'status': 'success', 'path': path, 'package': self.package} # Reach depth limit if depth == self.DFS_DEPTH_LIMIT: logger.debug('Reached DFS depth limit: {}'.format(self.DFS_DEPTH_LIMIT)) self.visited_elements.append(self.visiting_elements.pop()) self.return_to_home() return self._dfs(0) # Find next unvisited element in current page for ue in self._elements_generator(by_xml=True, score_threshold=0.15): status = 'untouched' # detect if exploration of ue is on going for ve in self.visiting_elements: if ue.diff(ve) < 0.05: logger.verbose(u'{} is still under visit'.format(ue.info)) status = 'revisit' break # detect if if status == 'untouched': for ve in self.visited_elements: if ue.diff(ve) < 0.05: logger.verbose(u'{} has been visited'.format(ue.info)) status = 'visited' break if not status == 'visited': logger.debug(u'DFS: try visiting {}'.format(ue.info)) if self.tap_test(ue.get_web_element(driver=self.driver)): if not status == 'revisit': self.visiting_elements.append(ue) return self._dfs(depth+1) else: self.visited_elements.append(ue) # All elements in current page have been tried # If in root page, exploration is done if depth == 0: logger.info('All explored but found nothing') return {'status': 'failed', 'path': None, 'package': self.package} # TODO: jump back to parent page # If in child page, return to root page and continue exploration else: self.visited_elements.append(self.visiting_elements.pop()) self.return_to_home() return self._dfs(0)
def __init_subclass__(cls, tokenClass=None, **kwargs): if tokenClass: super().__init_subclass__(**kwargs) logger.verbose("Register %s for %s" % (cls, tokenClass)) if cls.styles: AbstractNode.styles |= cls.styles cls.styles = AbstractNode.styles if tokenClass not in cls.classByToken: AbstractNode.classByToken[tokenClass] = [cls] else: AbstractNode.classByToken[tokenClass].append(cls)
def _sendVote(self): encVote = self.factory.sessionKey.encrypt(self.factory.vote) logger.debug("Encrypting vote with session key:") logger.debug(encVote, False) logger.verbose("Sending encrypted vote to collector...") message = SecureMessage() message.action = "vote" message['vote'] = encVote message.sign() self.sendMessage(message)
def messageReceived(self, message): if message['status'] == 'ok': if 'certificate' not in message: logger.warning("No certificate in received message, discarding...") else: cert = message['certificate'] logger.verbose("Received certificate from CA") self.factory.certficateReceived(cert) else: logger.verbose("Public Key was not authorized by CA") self.transport.loseConnection()
def sendVoteToCollector(self): logger.split() self.vote = vote = raw_input("Please Enter Vote Number: ") host = app.config.get("collector-host") port = app.config.get("collector-port") d = Deferred() factory = CollectorFactory(self.sessionKey, self.vote, d) logger.split() logger.verbose("Connecting to Collector...") reactor.connectTCP(host, port, factory) return d
def tap_test(self, element=None, x=None, y=None, diff=0.98): if self.__class__.__name__ == 'Navigator': diff = 1 act_before = self.driver.current_activity source_before = self.page_source.lower() self.action.tap(element, x, y).perform() diff_ratio = difflib.SequenceMatcher(None, self.page_source.lower(), source_before).ratio() act_after = self.driver.current_activity logger.verbose(u'Before tap: {}. After tap: {}. Diff: {}'.format(act_before, act_after, diff_ratio)) if act_after == act_before \ and diff_ratio > diff: return False else: return True
def __init__(self, block, parent=None, children=[]): super().__init__(block, parent, children) self.props = {} try: for item in block.children: # Todo support other token type. Only rawtext cut_index = item.children[0].children[0].content.index(':') idProp = item.children[0].children[0].content[:cut_index].strip() prop = item.children[0].children[0].content[cut_index+1:].strip() self.props[idProp] = prop logger.verbose("Parsed properties: %s" % self.props) except Exception as e: logger.error("Unable to parse properties. Do you follow the syntax ? (`- key : value ` in one line without styling token )") logger.error(e)
def sendIndex(self, index, sessionKey): message = SecureMessage() message.action = "set-index" pair = { 'certificate': self.factory.cert, 'index': index } pairString = json.dumps(pair) encPair = sessionKey.encrypt(pairString) message['certificate'] = self.factory.cert message['pair'] = encPair logger.verbose("Sending certificate/index pair to Authority") self.sendMessage(message.sign()) self.transport.loseConnection()
def messageReceived(self, message): logger.debug("Message received from Collector") try: if message.action == 'set-index': index = message['index'] logger.info("Index received from Collector:") logger.verbose(index, False) self.factory.receivedIndex(index) elif message.action == 'announce-results': voteCount = message['vote-count'] myVote = message['your-vote'] mVote = None logger.split().split() logger.info("Election Results:") for vote in voteCount: if not mVote or voteCount[vote] > mVote: mVote = voteCount[vote] for vote in voteCount: if voteCount[vote] == mVote: logger.extraspecial("%5s: %s (%d)" % (vote, '#'*voteCount[vote], voteCount[vote]), False) else: logger.special("%5s: %s (%d)" % (vote, '#'*voteCount[vote], voteCount[vote]), False) logger.split() logger.info("Your Vote:") logger.special(" Index: %s" % myVote['index']) logger.special(" Vote: %s" % myVote['vote']) reactor.stop() except KeyError as e: return self.factory.fail("Malformed message received from Collector. No '%s' field" % e)
def messageReceived(self, message): peer = self.transport.getPeer() #logger.debug("<%s:%s> Message Received:" % (peer.host, peer.port)) #logger.debug(message, False) myKey = app.keyManager.getMyKey() if 'public-key' not in message: logger.warning("Faulty message received: no public-key found. Discarding...") return publicKey = message['public-key'] cert = self.factory.cert.createCertificate(publicKey) logger.info("Certificate created for client <%s:%s>" % (peer.host, peer.port)) logger.verbose(cert.as_text()) reply = SecureMessage() reply['status'] = 'ok' reply['certificate'] = cert.as_pem() reply.sign() self.sendMessage(reply)
def countVotes(self, indexTable): logger.split() logger.verbose("Counting votes") logger.split() logger.info("Election Results:", False) logger.split() voteCount = {} for index in indexTable: # index = int(ind) key = SessionKey(int(indexTable[index], 16)) vote = key.decrypt(self.votes[index]) logger.special("%7s: %s" % (index, vote), False) if vote in voteCount: voteCount[vote] += 1 else: voteCount[vote] = 1 logger.split().split() mVote = None for vote in voteCount: if not mVote or voteCount[vote] > mVote: mVote = voteCount[vote] for vote in voteCount: if voteCount[vote] == mVote: logger.extraspecial("%5s: %s (%d)" % (vote, '#'*voteCount[vote], voteCount[vote]), False) else: logger.special("%5s: %s (%d)" % (vote, '#'*voteCount[vote], voteCount[vote]), False) logger.split() # Send results to clients for client in self.clients: client['connection'].sendResults(voteCount=voteCount, index=client['index'], vote=client['vote'])
def receivedVote(self, connection, vote): index = self.collector.createIndex() logger.verbose("Vote received from client") logger.verbose("Index: %x" % index, False) logger.verbose("EncryptedVote: %s" % vote, False) self.collector.storeVote(connection, index, vote) return index
def _getSessionKey(self, message): try: certificate = message['certificate'] # Create Cert # This is not working #cert = m2c.X509.load_cert_string(certificate) # Stupid hack to get it working tmp = open('.tmp.cert', 'w') tmp.write(certificate) tmp.close() cert = m2c.X509.load_cert('.tmp.cert') if not self.factory.authNode.validateCert(cert): logger.info("Invalid Certificate received from %s" % self.transport.getPeer()) reply = SecureMessage() reply['status'] = 'invalid' else: logger.verbose("Certificate validated") reply = SecureMessage() reply['status'] = 'ok' sessionKey = self.factory.authNode.generateSessionKey(cert) logger.verbose("Generated session key '%s'" % sessionKey.hex()) logger.verbose("Sending session key to client..") publicKeyPem = cert.get_pubkey().get_rsa().as_pem() keyParser = KeyParser() publicKey = keyParser.parsePemPublic(publicKeyPem) encSessionKey = publicKey.publicEncrypt(sessionKey.hex()) reply['encrypted-session-key'] = encSessionKey logger.split() self.sendMessage(reply.sign()) except KeyError as e: return self.factory.fail("Get-Session-Key no: '%s' field found in message" % e)
def run(self): factory = AuthorityFactory(self) port = reactor.listenTCP(app.config.getInt('authority-port'), factory) logger.verbose("Authority: Awaiting connections on %s" % port.getHost()) reactor.run()
def find_elements_by_keyword(self, keyword, clickable_only=False, exact=False, scroll=False, text_max_len=0, sort_elements=False, use_uiautomator=True): """ Find elements where keyword matches one of {text, resource id, content description} :param keyword: The keyword to search for :param clickable_only: Only return clickable elements if set to True :param exact: When exact is True, return only the whole word match. Otherwise return partial match as well. :param scroll: Scroll and search or only search current screen :param text_max_len: Limit maximum text length of returned elements :param sort_elements: Sort returned elements by text length. The shorter, the higher priority. :param use_uiautomator: Whether use UIAutomator or use pure xpath for element searching. UIAutomator mode support regex, but cannot match other attributes like bounds. Pure XPath mode doesn't support regex, but will match all attributes. Also Pure XPath mode is supposed to be faster as it sends 1 request per search while UIAutomator mode sends 3. :return: list of element objects """ # TODO: make sure no path config use these keywords anymore and remove this deprecated part # ========================================================= xmax = self.window_size['width'] ymax = self.window_size['height'] corner_words = { 'TOP_LEFT_CORNER': [0, 200, 0, 200], 'TOP_RIGHT_CORNER': [xmax - 200, xmax, 0, 200], 'BOTTOM_RIGHT_CORNER': [xmax - 200, xmax, ymax - 200, ymax], 'BOTTOM_LEFT_CORNER': [0, 200, ymax - 200, ymax] } if keyword in corner_words: return self.find_clickable_elements_in_area(*corner_words[keyword]) # ========================================================== # =================== Use UIAutomator ====================== # ========================================================== if use_uiautomator: if exact: regex = u"(?i)^\\s*(?:{0})\\s*$".format(keyword) else: regex = u"(?i).*(?:{0}).*".format(keyword) queries = [ u'new UiSelector().textMatches("{0}")'.format(regex), u'new UiSelector().resourceIdMatches("{0}")'.format(regex), u'new UiSelector().descriptionMatches("{0}")'.format(regex) ] if clickable_only: queries = [q + u'.clickable(true)' for q in queries] # TODO: find a way to search all three attributes in one scroll # https://android.googlesource.com/platform/frameworks/testing/+/master/uiautomator/library/core-src/com/android/uiautomator/core/UiScrollable.java if scroll: queries = [u'new UiScrollable(new UiSelector().scrollable(true))' u'.setMaxSearchSwipes(3).scrollIntoView({0})'.format(q) for q in queries] elements = [] for q in queries: elements = self.driver.find_elements_by_android_uiautomator(q) # short cut in scroll mode (once found, return) to prevent too many times of scrolling. if elements: break else: q_combined = ';'.join(queries) elements = self.driver.find_elements_by_android_uiautomator(q_combined) # ========================================================== # ======================= Use XPath ======================== # ========================================================== # We can use one query to search for text and all attributes: # //*[text()[contains(,'keyword')] or @*[contains(.,'keyword')]] # Use translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz') for case-insensitive in XPath1.0 # # !!! Appium doesnt support XPath2.0, thus doesnt support matches() and lower-case() # Case-insensitive: # //*[text()[matches(.,'keyword','i')] or @*[matches(.,'keyword','i')]] # Clickable only: # //*[text()[matches(.,'keyword','i')] or @*[matches(.,'keyword','i')] and @clickable='true'] else: keyword = keyword.lower() if exact: condition = u"{1}='{0}'" \ .format(keyword, "translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')") else: condition = u"contains({1},'{0}')" \ .format(keyword, "translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')") if clickable_only: query = u"//*[text()[{0}] or @*[{0}] and @clickable='true']".format(condition) else: query = u"//*[text()[{0}] or @*[{0}]]".format(condition) elements = self.driver.find_elements_by_xpath(query) if scroll: scrollables = self.driver.find_elements_by_xpath(u"//*[@scrollable='true']") if scrollables: scroll_cnt = 0 while not elements and scroll_cnt < 3: areas = [s.size['width'] * s.size['height'] for s in scrollables] i_max = areas.index(max(areas)) to_scroll = scrollables[i_max] x = to_scroll.location['x'] + to_scroll.size['width'] / 2 self.scroll_down(x=x) scroll_cnt += 1 elements = self.driver.find_elements_by_xpath(query) # ========================================================== # ===================== sort elements ====================== # ========================================================== if text_max_len or sort_elements: len_elements = [(len(layout.ElementInfo(e).text) if layout.ElementInfo(e).text else len(layout.ElementInfo(e).desc), e) for e in elements] # if elements have no texts, try content_desc if text_max_len: len_elements = [pair for pair in len_elements if pair[0] <= text_max_len] if sort_elements: len_elements.sort() elements = [e for (l, e) in len_elements] if elements: logger.verbose(u"max: {0}, sort: {1} - {2}".format(text_max_len, sort_elements, [p[0] for p in len_elements])) return elements
def skip_irrelevant(self, initial=True, limit=None, detect_login=None): """ Skip irrelevant activities and dialogs like update and permission notifications Will call itself recursively until no irrelevant things found This function may be invoked frequently to prevent stuck """ if initial: self.rec_count = 0 if detect_login is None: detect_login = self.app_style == 'international' if limit is None: if self.SKIP_IRR_LIMIT: limit = self.SKIP_IRR_LIMIT else: limit = 20 elif self.rec_count >= self.SKIP_IRR_LIMIT: raise myexceptions.SkipIrrelevantExceedLimit else: self.rec_count += 1 try: cur_act = self.driver.current_activity if not self.is_in_app(): logger.error('!!! APP not running, raise exception ...') raise myexceptions.AppNotRunningException logger.debug(u"Try to check and skip irrelevant activities {1}, current activity: {0}" .format(cur_act, "(%d)" % self.rec_count if self.rec_count else "")) clickable = self.driver.find_elements_by_android_uiautomator('new UiSelector().clickable(true)') textedit = self.driver.find_elements_by_class_name('android.widget.EditText') logger.verbose(u'''Found: {0} clickable elements {1} TextEdit elements'''.format(len(clickable), len(textedit))) # Nothing can be clicked, wait or swipe if len(clickable) == 0 \ and len(textedit) == 0: if not self.waited: logger.debug(u'Seems to be in loading page, wait then try again ...') timeout = 6 source_before = self.page_source while timeout: sleep(1) source_after = self.page_source if difflib.SequenceMatcher(None, source_before, source_after).ratio() < 0.8: break source_before = source_after timeout -= 1 self.waited = 1 elif self.waited == 1: logger.debug(u'\ttap then wait ...') self.driver.tap([(self.window_size['width'] / 2, self.window_size['height'] / 2)]) sleep(1) self.waited = 2 elif self.waited > 1: logger.debug(u'\tswipe then wait ...') self.swipe_left() sleep(1) self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login) return # Welcome page skippable = self.contain_skip_text() get_sinks = lambda: self.driver.find_elements_by_xpath('//*[not(*)]') if not self.is_in_dialog() \ and ((skippable or (len(clickable) in range(1, 5) and len(textedit) == 0)) and len(get_sinks()) < 20): logger.debug(u'Seems to be in welcome page, try bypassing it') if detect_login and self.contain_login_text(): logger.debug(u'Find login keywords in welcome page, break skip_irrelevant') return if skippable: for kw in self.skipkw: for e in self.find_elements_by_keyword(kw): if self.tap_test(e, diff=0.98): self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login) return safe_clickable = self.find_safe_clickable_elements() if safe_clickable: to_tap = safe_clickable[-1] ele_info = layout.ElementInfo(to_tap) logger.debug(u'Tapped {0}'.format(ele_info)) if not self.tap_test(to_tap, diff=0.98): logger.warning(u'Tap failed: {0}, try swiping'.format(ele_info)) self.swipe_left() self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login) return # Dialog # TODO: decide cancel/ok by context if self.is_in_dialog() \ and len(clickable) in range(1, 5) \ and len(textedit) == 0: logger.debug(u'Seems to be a dialog, try bypassing it') if detect_login and self.contain_login_text(): logger.debug(u'Find login keywords in welcome page, break skip_irrelevant') return safe_clickable = self.find_safe_clickable_elements() if not safe_clickable: raise myexceptions.NoSafeClickableElement("Seems like a dialog that requiring update") source_before_tap = self.page_source to_tap = safe_clickable[-1] ele_info = layout.ElementInfo(to_tap) logger.debug(u'Tapped {0}'.format(ele_info)) self.tap(to_tap) if self.driver.current_activity == cur_act \ and difflib.SequenceMatcher(None, self.page_source.lower(), source_before_tap).ratio() > 0.95: logger.warning(u'Tap failed: {0}'.format(ele_info)) self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login) return # City list cities_pattern = u'(?:鞍山|安庆|安阳|安顺|北京|天津|上海|深圳|广州|成都|南京|重庆|杭州)市?' filtered = self.find_elements_by_keyword(cities_pattern, clickable_only=False, exact=True, scroll=False) if len(filtered) > 3: logger.debug(u'Seems to be a city select page, try bypassing it') to_tap = filtered[0] ele_info = layout.ElementInfo(to_tap) logger.debug(u'Tapped {0}'.format(ele_info)) self.tap(to_tap) if self.driver.current_activity == cur_act: logger.warning(u'Tap failed: {0}'.format(ele_info)) self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login) return except NoSuchElementException: self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)