def __init__(self, mdFilename, templateFilename, output):
logger.verbose("Project initialization : %s, %s, %s" %
(mdFilename, templateFilename, output))
self._mdFilename = mdFilename
with open(mdFilename, 'r') as f:
self._rawMd = f.read()
self.md = Document(self._rawMd)
self._templateFilename = templateFilename
self.template = DocxTemplate(templateFilename)
self.parsedTree = Parser(self.md, self.template, mdFilename)
logger.debug("Parsed Tree:\n" + str(self.parsedTree))
context = self.parsedTree.generateDocx()
logger.spam("context: %s" % context)
logger.info('First rendering start.')
self.template.render(context)
self.template.save(output + self.TMP_FILENAME)
logger.info('Second rendering start.')
self.template = DocxTemplate(output + self.TMP_FILENAME)
self.parsedTree.finalize(self.template)
self.template.render(context)
self.template.save(output)
logger.info('Full rendering finished.')
os.remove(output + self.TMP_FILENAME)
def run(self):
factory = VoteCollectorFactory(self)
port = reactor.listenTCP(app.config.getInt('collector-port'), factory)
logger.verbose("Collector: Awaiting connections on %s" % port.getHost())
stdio.StandardIO(Console(self))
reactor.run()
def _setIndex(self, message):
try:
encPair = message['pair']
cert = message['certificate']
try:
sessionKey = self.factory.authNode.sessionKeys[cert]
except KeyError:
return self.factory.fail("Requested to set index for an unregistered certificate")
pairString = sessionKey.decrypt(encPair)
try:
# logger.special(pairString)
#pairString = pairString[0:pairString.index('}')+1]
pair = json.loads(pairString)
except ValueError as e:
logger.warning("Invalid JSON as cert/index pair, discarding message...")
logger.warning(e.message, False)
return
if pair['certificate'] != cert:
logger.warning("Encrypted certificate doesn't match client's certificte, discarding message...")
return
logger.info("Index set for client")
logger.verbose("Index: %s" % pair['index'], False)
logger.verbose("Session Key: %s" % sessionKey.hex(), False)
self.factory.authNode.indices[cert] = pair['index']
logger.split()
except KeyError as e:
return self.factory.fail("Get-Session-Key no: '%s' field found in message" % e)
def messageReceived(self, message):
encTable = message['encrypted-table']
tableString = self.factory.collectorNode.authKey.decrypt(encTable)
print(tableString)
table = json.loads(tableString)
logger.verbose("Table received from Authority")
self.factory.collectorNode.countVotes(table)
def run(self):
from twisted.internet import reactor
factory = CAFactory()
port = reactor.listenTCP(app.config.getInt('ca-port'), factory)
logger.verbose("Awaiting connections on %s" % port.getHost())
reactor.run()
def endElections(self):
host = app.config.get("authority-host")
port = app.config.get("authority-port")
factory = AuthorityFactory(self)
logger.split()
logger.verbose("Connecting to Authority...")
reactor.connectTCP(host, port, factory)
def getCertificateFromCA(self):
host = app.config.get("ca-host")
port = app.config.get("ca-port")
d = Deferred()
factory = CertFactory(d)
logger.verbose("Connecting to CA...")
reactor.connectTCP(host, port, factory)
return d
def _sendKey(self):
myKey = app.keyManager.getMyKey()
myPublicKey = myKey.publickey()
message = SecureMessage()
message['public-key'] = myPublicKey.pem()
logger.verbose("Sending Public Key to CA...")
logger.debug(message)
self.sendMessage(message)
def messageReceived(self, message):
logger.verbose("Message with action '%s' received from %s" % (message.action, message.sender))
if message.action == 'get-session-key':
self._getSessionKey(message)
elif message.action == 'set-index':
self._setIndex(message)
elif message.action == 'request-table':
self._requestTable(message)
else:
return self.factory.fail("Unrecognized action in received message: '%s'" % message.action)
def sendCertToAuth(self, cert):
host = app.config.get("authority-host")
port = app.config.get("authority-port")
d = Deferred()
factory = Authority2Factory(cert, d)
logger.split()
logger.verbose("Connecting to Authority...")
reactor.connectTCP(host, port, factory)
return d
def _dfs(self, depth):
logger.debug('Into DFS depth {}'.format(depth))
# Found destination
path = [e.info for e in self.visiting_elements]
if self._dest_keywords and self.wait_for_destination(self._dest_keywords, timeout=10):
logger.info(u'Destination keywords matched')
return {'status': 'success', 'path': path, 'package': self.package}
if self._dest_activities and self.wait_for_activities(self._dest_activities, timeout=10):
logger.info(u'Destination activities matched.')
return {'status': 'success', 'path': path, 'package': self.package}
# Reach depth limit
if depth == self.DFS_DEPTH_LIMIT:
logger.debug('Reached DFS depth limit: {}'.format(self.DFS_DEPTH_LIMIT))
self.visited_elements.append(self.visiting_elements.pop())
self.return_to_home()
return self._dfs(0)
# Find next unvisited element in current page
for ue in self._elements_generator(by_xml=True, score_threshold=0.15):
status = 'untouched'
# detect if exploration of ue is on going
for ve in self.visiting_elements:
if ue.diff(ve) < 0.05:
logger.verbose(u'{} is still under visit'.format(ue.info))
status = 'revisit'
break
# detect if
if status == 'untouched':
for ve in self.visited_elements:
if ue.diff(ve) < 0.05:
logger.verbose(u'{} has been visited'.format(ue.info))
status = 'visited'
break
if not status == 'visited':
logger.debug(u'DFS: try visiting {}'.format(ue.info))
if self.tap_test(ue.get_web_element(driver=self.driver)):
if not status == 'revisit':
self.visiting_elements.append(ue)
return self._dfs(depth+1)
else:
self.visited_elements.append(ue)
# All elements in current page have been tried
# If in root page, exploration is done
if depth == 0:
logger.info('All explored but found nothing')
return {'status': 'failed', 'path': None, 'package': self.package}
# TODO: jump back to parent page
# If in child page, return to root page and continue exploration
else:
self.visited_elements.append(self.visiting_elements.pop())
self.return_to_home()
return self._dfs(0)
def __init_subclass__(cls, tokenClass=None, **kwargs):
if tokenClass:
super().__init_subclass__(**kwargs)
logger.verbose("Register %s for %s" % (cls, tokenClass))
if cls.styles:
AbstractNode.styles |= cls.styles
cls.styles = AbstractNode.styles
if tokenClass not in cls.classByToken:
AbstractNode.classByToken[tokenClass] = [cls]
else:
AbstractNode.classByToken[tokenClass].append(cls)
def _sendVote(self):
encVote = self.factory.sessionKey.encrypt(self.factory.vote)
logger.debug("Encrypting vote with session key:")
logger.debug(encVote, False)
logger.verbose("Sending encrypted vote to collector...")
message = SecureMessage()
message.action = "vote"
message['vote'] = encVote
message.sign()
self.sendMessage(message)
def messageReceived(self, message):
if message['status'] == 'ok':
if 'certificate' not in message:
logger.warning("No certificate in received message, discarding...")
else:
cert = message['certificate']
logger.verbose("Received certificate from CA")
self.factory.certficateReceived(cert)
else:
logger.verbose("Public Key was not authorized by CA")
self.transport.loseConnection()
def sendVoteToCollector(self):
logger.split()
self.vote = vote = raw_input("Please Enter Vote Number: ")
host = app.config.get("collector-host")
port = app.config.get("collector-port")
d = Deferred()
factory = CollectorFactory(self.sessionKey, self.vote, d)
logger.split()
logger.verbose("Connecting to Collector...")
reactor.connectTCP(host, port, factory)
return d
def tap_test(self, element=None, x=None, y=None, diff=0.98):
if self.__class__.__name__ == 'Navigator':
diff = 1
act_before = self.driver.current_activity
source_before = self.page_source.lower()
self.action.tap(element, x, y).perform()
diff_ratio = difflib.SequenceMatcher(None, self.page_source.lower(), source_before).ratio()
act_after = self.driver.current_activity
logger.verbose(u'Before tap: {}. After tap: {}. Diff: {}'.format(act_before, act_after, diff_ratio))
if act_after == act_before \
and diff_ratio > diff:
return False
else:
return True
def __init__(self, block, parent=None, children=[]):
super().__init__(block, parent, children)
self.props = {}
try:
for item in block.children:
# Todo support other token type. Only rawtext
cut_index = item.children[0].children[0].content.index(':')
idProp = item.children[0].children[0].content[:cut_index].strip()
prop = item.children[0].children[0].content[cut_index+1:].strip()
self.props[idProp] = prop
logger.verbose("Parsed properties: %s" % self.props)
except Exception as e:
logger.error("Unable to parse properties. Do you follow the syntax ? (`- key : value ` in one line without styling token )")
logger.error(e)
def sendIndex(self, index, sessionKey):
message = SecureMessage()
message.action = "set-index"
pair = {
'certificate': self.factory.cert,
'index': index
}
pairString = json.dumps(pair)
encPair = sessionKey.encrypt(pairString)
message['certificate'] = self.factory.cert
message['pair'] = encPair
logger.verbose("Sending certificate/index pair to Authority")
self.sendMessage(message.sign())
self.transport.loseConnection()
def messageReceived(self, message):
logger.debug("Message received from Collector")
try:
if message.action == 'set-index':
index = message['index']
logger.info("Index received from Collector:")
logger.verbose(index, False)
self.factory.receivedIndex(index)
elif message.action == 'announce-results':
voteCount = message['vote-count']
myVote = message['your-vote']
mVote = None
logger.split().split()
logger.info("Election Results:")
for vote in voteCount:
if not mVote or voteCount[vote] > mVote:
mVote = voteCount[vote]
for vote in voteCount:
if voteCount[vote] == mVote:
logger.extraspecial("%5s: %s (%d)" % (vote, '#'*voteCount[vote], voteCount[vote]), False)
else:
logger.special("%5s: %s (%d)" % (vote, '#'*voteCount[vote], voteCount[vote]), False)
logger.split()
logger.info("Your Vote:")
logger.special(" Index: %s" % myVote['index'])
logger.special(" Vote: %s" % myVote['vote'])
reactor.stop()
except KeyError as e:
return self.factory.fail("Malformed message received from Collector. No '%s' field" % e)
def messageReceived(self, message):
peer = self.transport.getPeer()
#logger.debug("<%s:%s> Message Received:" % (peer.host, peer.port))
#logger.debug(message, False)
myKey = app.keyManager.getMyKey()
if 'public-key' not in message:
logger.warning("Faulty message received: no public-key found. Discarding...")
return
publicKey = message['public-key']
cert = self.factory.cert.createCertificate(publicKey)
logger.info("Certificate created for client <%s:%s>" % (peer.host, peer.port))
logger.verbose(cert.as_text())
reply = SecureMessage()
reply['status'] = 'ok'
reply['certificate'] = cert.as_pem()
reply.sign()
self.sendMessage(reply)
def countVotes(self, indexTable):
logger.split()
logger.verbose("Counting votes")
logger.split()
logger.info("Election Results:", False)
logger.split()
voteCount = {}
for index in indexTable:
# index = int(ind)
key = SessionKey(int(indexTable[index], 16))
vote = key.decrypt(self.votes[index])
logger.special("%7s: %s" % (index, vote), False)
if vote in voteCount:
voteCount[vote] += 1
else:
voteCount[vote] = 1
logger.split().split()
mVote = None
for vote in voteCount:
if not mVote or voteCount[vote] > mVote:
mVote = voteCount[vote]
for vote in voteCount:
if voteCount[vote] == mVote:
logger.extraspecial("%5s: %s (%d)" % (vote, '#'*voteCount[vote], voteCount[vote]), False)
else:
logger.special("%5s: %s (%d)" % (vote, '#'*voteCount[vote], voteCount[vote]), False)
logger.split()
# Send results to clients
for client in self.clients:
client['connection'].sendResults(voteCount=voteCount, index=client['index'], vote=client['vote'])
def receivedVote(self, connection, vote):
index = self.collector.createIndex()
logger.verbose("Vote received from client")
logger.verbose("Index: %x" % index, False)
logger.verbose("EncryptedVote: %s" % vote, False)
self.collector.storeVote(connection, index, vote)
return index
def _getSessionKey(self, message):
try:
certificate = message['certificate']
# Create Cert
# This is not working
#cert = m2c.X509.load_cert_string(certificate)
# Stupid hack to get it working
tmp = open('.tmp.cert', 'w')
tmp.write(certificate)
tmp.close()
cert = m2c.X509.load_cert('.tmp.cert')
if not self.factory.authNode.validateCert(cert):
logger.info("Invalid Certificate received from %s" % self.transport.getPeer())
reply = SecureMessage()
reply['status'] = 'invalid'
else:
logger.verbose("Certificate validated")
reply = SecureMessage()
reply['status'] = 'ok'
sessionKey = self.factory.authNode.generateSessionKey(cert)
logger.verbose("Generated session key '%s'" % sessionKey.hex())
logger.verbose("Sending session key to client..")
publicKeyPem = cert.get_pubkey().get_rsa().as_pem()
keyParser = KeyParser()
publicKey = keyParser.parsePemPublic(publicKeyPem)
encSessionKey = publicKey.publicEncrypt(sessionKey.hex())
reply['encrypted-session-key'] = encSessionKey
logger.split()
self.sendMessage(reply.sign())
except KeyError as e:
return self.factory.fail("Get-Session-Key no: '%s' field found in message" % e)
def run(self):
factory = AuthorityFactory(self)
port = reactor.listenTCP(app.config.getInt('authority-port'), factory)
logger.verbose("Authority: Awaiting connections on %s" % port.getHost())
reactor.run()
def find_elements_by_keyword(self, keyword, clickable_only=False, exact=False, scroll=False,
text_max_len=0, sort_elements=False, use_uiautomator=True):
"""
Find elements where keyword matches one of {text, resource id, content description}
:param keyword: The keyword to search for
:param clickable_only: Only return clickable elements if set to True
:param exact: When exact is True, return only the whole word match. Otherwise return partial match as well.
:param scroll: Scroll and search or only search current screen
:param text_max_len: Limit maximum text length of returned elements
:param sort_elements: Sort returned elements by text length. The shorter, the higher priority.
:param use_uiautomator: Whether use UIAutomator or use pure xpath for element searching.
UIAutomator mode support regex, but cannot match other attributes like bounds.
Pure XPath mode doesn't support regex, but will match all attributes.
Also Pure XPath mode is supposed to be faster
as it sends 1 request per search while UIAutomator mode sends 3.
:return: list of element objects
"""
# TODO: make sure no path config use these keywords anymore and remove this deprecated part
# =========================================================
xmax = self.window_size['width']
ymax = self.window_size['height']
corner_words = {
'TOP_LEFT_CORNER': [0, 200, 0, 200],
'TOP_RIGHT_CORNER': [xmax - 200, xmax, 0, 200],
'BOTTOM_RIGHT_CORNER': [xmax - 200, xmax, ymax - 200, ymax],
'BOTTOM_LEFT_CORNER': [0, 200, ymax - 200, ymax]
}
if keyword in corner_words:
return self.find_clickable_elements_in_area(*corner_words[keyword])
# ==========================================================
# =================== Use UIAutomator ======================
# ==========================================================
if use_uiautomator:
if exact:
regex = u"(?i)^\\s*(?:{0})\\s*$".format(keyword)
else:
regex = u"(?i).*(?:{0}).*".format(keyword)
queries = [
u'new UiSelector().textMatches("{0}")'.format(regex),
u'new UiSelector().resourceIdMatches("{0}")'.format(regex),
u'new UiSelector().descriptionMatches("{0}")'.format(regex)
]
if clickable_only:
queries = [q + u'.clickable(true)' for q in queries]
# TODO: find a way to search all three attributes in one scroll
# https://android.googlesource.com/platform/frameworks/testing/+/master/uiautomator/library/core-src/com/android/uiautomator/core/UiScrollable.java
if scroll:
queries = [u'new UiScrollable(new UiSelector().scrollable(true))'
u'.setMaxSearchSwipes(3).scrollIntoView({0})'.format(q) for q in queries]
elements = []
for q in queries:
elements = self.driver.find_elements_by_android_uiautomator(q)
# short cut in scroll mode (once found, return) to prevent too many times of scrolling.
if elements:
break
else:
q_combined = ';'.join(queries)
elements = self.driver.find_elements_by_android_uiautomator(q_combined)
# ==========================================================
# ======================= Use XPath ========================
# ==========================================================
# We can use one query to search for text and all attributes:
# //*[text()[contains(,'keyword')] or @*[contains(.,'keyword')]]
# Use translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz') for case-insensitive in XPath1.0
#
# !!! Appium doesnt support XPath2.0, thus doesnt support matches() and lower-case()
# Case-insensitive:
# //*[text()[matches(.,'keyword','i')] or @*[matches(.,'keyword','i')]]
# Clickable only:
# //*[text()[matches(.,'keyword','i')] or @*[matches(.,'keyword','i')] and @clickable='true']
else:
keyword = keyword.lower()
if exact:
condition = u"{1}='{0}'" \
.format(keyword, "translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')")
else:
condition = u"contains({1},'{0}')" \
.format(keyword, "translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')")
if clickable_only:
query = u"//*[text()[{0}] or @*[{0}] and @clickable='true']".format(condition)
else:
query = u"//*[text()[{0}] or @*[{0}]]".format(condition)
elements = self.driver.find_elements_by_xpath(query)
if scroll:
scrollables = self.driver.find_elements_by_xpath(u"//*[@scrollable='true']")
if scrollables:
scroll_cnt = 0
while not elements and scroll_cnt < 3:
areas = [s.size['width'] * s.size['height'] for s in scrollables]
i_max = areas.index(max(areas))
to_scroll = scrollables[i_max]
x = to_scroll.location['x'] + to_scroll.size['width'] / 2
self.scroll_down(x=x)
scroll_cnt += 1
elements = self.driver.find_elements_by_xpath(query)
# ==========================================================
# ===================== sort elements ======================
# ==========================================================
if text_max_len or sort_elements:
len_elements = [(len(layout.ElementInfo(e).text)
if layout.ElementInfo(e).text
else len(layout.ElementInfo(e).desc), e)
for e in elements] # if elements have no texts, try content_desc
if text_max_len:
len_elements = [pair for pair in len_elements if pair[0] <= text_max_len]
if sort_elements:
len_elements.sort()
elements = [e for (l, e) in len_elements]
if elements:
logger.verbose(u"max: {0}, sort: {1} - {2}".format(text_max_len, sort_elements,
[p[0] for p in len_elements]))
return elements
def skip_irrelevant(self, initial=True, limit=None, detect_login=None):
""" Skip irrelevant activities and dialogs like update and permission notifications
Will call itself recursively until no irrelevant things found
This function may be invoked frequently to prevent stuck
"""
if initial:
self.rec_count = 0
if detect_login is None:
detect_login = self.app_style == 'international'
if limit is None:
if self.SKIP_IRR_LIMIT:
limit = self.SKIP_IRR_LIMIT
else:
limit = 20
elif self.rec_count >= self.SKIP_IRR_LIMIT:
raise myexceptions.SkipIrrelevantExceedLimit
else:
self.rec_count += 1
try:
cur_act = self.driver.current_activity
if not self.is_in_app():
logger.error('!!! APP not running, raise exception ...')
raise myexceptions.AppNotRunningException
logger.debug(u"Try to check and skip irrelevant activities {1}, current activity: {0}"
.format(cur_act, "(%d)" % self.rec_count if self.rec_count else ""))
clickable = self.driver.find_elements_by_android_uiautomator('new UiSelector().clickable(true)')
textedit = self.driver.find_elements_by_class_name('android.widget.EditText')
logger.verbose(u'''Found:
{0} clickable elements
{1} TextEdit elements'''.format(len(clickable), len(textedit)))
# Nothing can be clicked, wait or swipe
if len(clickable) == 0 \
and len(textedit) == 0:
if not self.waited:
logger.debug(u'Seems to be in loading page, wait then try again ...')
timeout = 6
source_before = self.page_source
while timeout:
sleep(1)
source_after = self.page_source
if difflib.SequenceMatcher(None, source_before, source_after).ratio() < 0.8:
break
source_before = source_after
timeout -= 1
self.waited = 1
elif self.waited == 1:
logger.debug(u'\ttap then wait ...')
self.driver.tap([(self.window_size['width'] / 2, self.window_size['height'] / 2)])
sleep(1)
self.waited = 2
elif self.waited > 1:
logger.debug(u'\tswipe then wait ...')
self.swipe_left()
sleep(1)
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
return
# Welcome page
skippable = self.contain_skip_text()
get_sinks = lambda: self.driver.find_elements_by_xpath('//*[not(*)]')
if not self.is_in_dialog() \
and ((skippable or (len(clickable) in range(1, 5) and len(textedit) == 0))
and len(get_sinks()) < 20):
logger.debug(u'Seems to be in welcome page, try bypassing it')
if detect_login and self.contain_login_text():
logger.debug(u'Find login keywords in welcome page, break skip_irrelevant')
return
if skippable:
for kw in self.skipkw:
for e in self.find_elements_by_keyword(kw):
if self.tap_test(e, diff=0.98):
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
return
safe_clickable = self.find_safe_clickable_elements()
if safe_clickable:
to_tap = safe_clickable[-1]
ele_info = layout.ElementInfo(to_tap)
logger.debug(u'Tapped {0}'.format(ele_info))
if not self.tap_test(to_tap, diff=0.98):
logger.warning(u'Tap failed: {0}, try swiping'.format(ele_info))
self.swipe_left()
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
return
# Dialog
# TODO: decide cancel/ok by context
if self.is_in_dialog() \
and len(clickable) in range(1, 5) \
and len(textedit) == 0:
logger.debug(u'Seems to be a dialog, try bypassing it')
if detect_login and self.contain_login_text():
logger.debug(u'Find login keywords in welcome page, break skip_irrelevant')
return
safe_clickable = self.find_safe_clickable_elements()
if not safe_clickable:
raise myexceptions.NoSafeClickableElement("Seems like a dialog that requiring update")
source_before_tap = self.page_source
to_tap = safe_clickable[-1]
ele_info = layout.ElementInfo(to_tap)
logger.debug(u'Tapped {0}'.format(ele_info))
self.tap(to_tap)
if self.driver.current_activity == cur_act \
and difflib.SequenceMatcher(None, self.page_source.lower(),
source_before_tap).ratio() > 0.95:
logger.warning(u'Tap failed: {0}'.format(ele_info))
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
return
# City list
cities_pattern = u'(?:鞍山|安庆|安阳|安顺|北京|天津|上海|深圳|广州|成都|南京|重庆|杭州)市?'
filtered = self.find_elements_by_keyword(cities_pattern, clickable_only=False, exact=True, scroll=False)
if len(filtered) > 3:
logger.debug(u'Seems to be a city select page, try bypassing it')
to_tap = filtered[0]
ele_info = layout.ElementInfo(to_tap)
logger.debug(u'Tapped {0}'.format(ele_info))
self.tap(to_tap)
if self.driver.current_activity == cur_act:
logger.warning(u'Tap failed: {0}'.format(ele_info))
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
return
except NoSuchElementException:
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
