def test_Diff_Basic(self):
size_info1 = self._CloneSizeInfo(use_elf=False, use_pak=True)
size_info2 = self._CloneSizeInfo(use_elf=False, use_pak=True)
size_info1.metadata = {"foo": 1, "bar": [1, 2, 3], "baz": "yes"}
size_info2.metadata = {"foo": 1, "bar": [1, 3], "baz": "yes"}
size_info1.raw_symbols -= size_info1.raw_symbols[:2]
size_info2.raw_symbols -= size_info2.raw_symbols[-3:]
changed_sym = size_info1.raw_symbols.WhereNameMatches(
'Patcher::Name_')[0]
changed_sym.size -= 10
padding_sym = size_info2.raw_symbols.WhereNameMatches(
'symbol gap 0')[0]
padding_sym.padding += 20
padding_sym.size += 20
pak_sym = size_info2.raw_symbols.WhereInSection(
models.SECTION_PAK_NONTRANSLATED)[0]
pak_sym.full_name = 'foo: ' + pak_sym.full_name.split()[-1]
# Serialize & de-serialize so that name normalization runs again for the pak
# symbol.
stringio = cStringIO.StringIO()
file_format.SaveSizeInfo(size_info2, 'path', file_obj=stringio)
stringio.seek(0)
size_info2 = archive.LoadAndPostProcessSizeInfo('path',
file_obj=stringio)
d = diff.Diff(size_info1, size_info2)
d.raw_symbols = d.raw_symbols.Sorted()
self.assertEquals(d.raw_symbols.CountsByDiffStatus()[1:], (2, 2, 3))
changed_sym = d.raw_symbols.WhereNameMatches('Patcher::Name_')[0]
padding_sym = d.raw_symbols.WhereNameMatches('symbol gap 0')[0]
bss_sym = d.raw_symbols.WhereInSection(models.SECTION_BSS)[0]
# Padding-only deltas should sort after all non-padding changes.
padding_idx = d.raw_symbols.index(padding_sym)
changed_idx = d.raw_symbols.index(changed_sym)
bss_idx = d.raw_symbols.index(bss_sym)
self.assertLess(changed_idx, padding_idx)
# And before bss.
self.assertLess(padding_idx, bss_idx)
return describe.GenerateLines(d, verbose=True)
def testNumberNormalization(self):
TEXT = models.SECTION_TEXT
size_info1 = _CreateSizeInfo()
size_info1.raw_symbols += [
_MakeSym(TEXT, 11, 'a', name='.L__unnamed_1193'),
_MakeSym(TEXT, 22, 'a', name='.L__unnamed_1194'),
_MakeSym(TEXT, 33, 'a', name='SingleCategoryPreferences$3#this$0'),
_MakeSym(TEXT, 44, 'a', name='.L.ref.tmp.2'),
]
size_info2 = _CreateSizeInfo()
size_info2.raw_symbols += [
_MakeSym(TEXT, 11, 'a', name='.L__unnamed_2194'),
_MakeSym(TEXT, 22, 'a', name='.L__unnamed_2195'),
_MakeSym(TEXT, 33, 'a', name='SingleCategoryPreferences$9#this$009'),
_MakeSym(TEXT, 44, 'a', name='.L.ref.tmp.137'),
]
# For simplicity, not associating |symbols| with |containers|.
d = diff.Diff(size_info1, size_info2)
self.assertEqual((0, 0, 0), d.raw_symbols.CountsByDiffStatus()[1:])
self.assertEqual(0, d.raw_symbols.size)
def _DiffFunc(self, before=None, after=None, sort=True):
"""Diffs two SizeInfo objects. Returns a DeltaSizeInfo.
Args:
before: Defaults to first size_infos[0].
after: Defaults to second size_infos[1].
sort: When True (default), calls SymbolGroup.Sorted() after diffing.
"""
before = before if before is not None else self._size_infos[0]
after = after if after is not None else self._size_infos[1]
ret = diff.Diff(before, after)
if sort:
syms = ret.symbols # Triggers clustering.
logging.debug('Grouping')
# Group path aliases so that functions defined in headers will be sorted
# by their actual size rather than shown as many small symbols.
syms = syms.GroupedByAliases(same_name_only=True)
logging.debug('Sorting')
ret.symbols = syms.Sorted()
logging.debug('Diff complete')
return ret
def test_Diff_Clustering(self):
size_info1 = self._CloneSizeInfo()
size_info2 = self._CloneSizeInfo()
S = '.text'
size_info1.symbols += [
models.Symbol(S, 11, name='.L__unnamed_1193', object_path='a'), # 1
models.Symbol(S, 22, name='.L__unnamed_1194', object_path='a'), # 2
models.Symbol(S, 33, name='.L__unnamed_1195', object_path='b'), # 3
models.Symbol(S, 44, name='.L__bar_195', object_path='b'), # 4
models.Symbol(S, 55, name='.L__bar_1195', object_path='b'), # 5
]
size_info2.symbols += [
models.Symbol(S, 33, name='.L__unnamed_2195', object_path='b'), # 3
models.Symbol(S, 11, name='.L__unnamed_2194', object_path='a'), # 1
models.Symbol(S, 22, name='.L__unnamed_2193', object_path='a'), # 2
models.Symbol(S, 44, name='.L__bar_2195', object_path='b'), # 4
models.Symbol(S, 55, name='.L__bar_295', object_path='b'), # 5
]
d = diff.Diff(size_info1, size_info2)
self.assertEquals(d.symbols.added_count, 0)
self.assertEquals(d.symbols.size, 0)
def main(options, arguments):
if options.input != None:
a = androguard.Androguard(options.input)
a.ianalyze()
vm1 = a.get_bc()[0][1].get_vm()
vmx1 = a.get_bc()[0][1].get_analysis()
vm2 = a.get_bc()[1][1].get_vm()
vmx2 = a.get_bc()[1][1].get_analysis()
d = diff.Diff([vm1, vmx1], [vm2, vmx2], diff.FILTERS_DIFF)
details = False
if options.display != None:
details = True
print "DIFF METHODS :"
diff_methods = d.get_diff_methods()
for i in diff_methods:
for elem in diff_methods[i]:
elem.show(i, details)
print
print "NEW METHODS :"
new_methods = d.get_new_methods()
for i in new_methods:
for elem in new_methods[i]:
elem.show2(details)
print
print "DELETE METHODS :"
del_methods = d.get_delete_methods()
for i in del_methods:
for elem in del_methods[i]:
elem.show2(details)
print
elif options.version != None:
print "Androdiff version %s" % misc.ANDRODIFF_VERSION
def test_Diff_Clustering(self):
size_info1 = self._CloneSizeInfo()
size_info2 = self._CloneSizeInfo()
S = models.SECTION_TEXT
size_info1.symbols += [
models.Symbol(S, 11, name='.L__unnamed_1193', object_path='a'), # 1
models.Symbol(S, 22, name='.L__unnamed_1194', object_path='a'), # 2
models.Symbol(S, 33, name='.L__unnamed_1195', object_path='b'), # 3
models.Symbol(S, 44, name='.L__bar_195', object_path='b'), # 4
models.Symbol(S, 55, name='.L__bar_1195', object_path='b'), # 5
]
size_info2.symbols += [
models.Symbol(S, 33, name='.L__unnamed_2195', object_path='b'), # 3
models.Symbol(S, 11, name='.L__unnamed_2194', object_path='a'), # 1
models.Symbol(S, 22, name='.L__unnamed_2193', object_path='a'), # 2
models.Symbol(S, 44, name='.L__bar_2195', object_path='b'), # 4
models.Symbol(S, 55, name='.L__bar_295', object_path='b'), # 5
]
d = diff.Diff(size_info1, size_info2)
d.symbols = d.symbols.Sorted()
self.assertEquals(d.symbols.CountsByDiffStatus()[models.DIFF_STATUS_ADDED],
0)
self.assertEquals(d.symbols.size, 0)
def Run(args, on_config_error):
if not args.input_size_file.endswith('.size'):
on_config_error('Input must end with ".size"')
if args.diff_with and not args.diff_with.endswith('.size'):
on_config_error('Diff input must end with ".size"')
if not args.output_report_file.endswith('.ndjson'):
on_config_error('Output must end with ".ndjson"')
size_info = archive.LoadAndPostProcessSizeInfo(args.input_size_file)
if args.diff_with:
before_size_info = archive.LoadAndPostProcessSizeInfo(args.diff_with)
size_info = diff.Diff(before_size_info, size_info)
BuildReportFromSizeInfo(args.output_report_file,
size_info,
all_symbols=args.all_symbols)
logging.warning('Done!')
msg = [
'View using a local server via: ',
' {0}/upload_html_viewer.py --local',
'or run:',
' gsutil.py cp -a public-read {1} gs://chrome-supersize/oneoffs/'
'{2}.ndjson',
' to view at:',
' https://chrome-supersize.firebaseapp.com/viewer.html?load_url='
'https://storage.googleapis.com/chrome-supersize/oneoffs/{2}.ndjson',
]
libsupersize_path = os.path.relpath(
os.path.join(path_util.TOOLS_SRC_ROOT, 'tools', 'binary_size',
'libsupersize'))
# Use a random UUID as the filename so user can copy-and-paste command
# directly without a name collision.
upload_id = uuid.uuid4()
print('\n'.join(msg).format(libsupersize_path, args.output_report_file,
upload_id))
def get(self):
login_name=str(users.get_current_user())
self.response.out.write("<html><head><link rel=\"shortcut\" href=\"/favicon.ico\" /></head><body>")
if users.get_current_user():
url = users.create_logout_url(self.request.uri)
self.response.out.write("<a href='"+url+"'>%s Google Logout</a><br /><br />\n"%(login_name))
else:
self.redirect('/')
auth=getauth(users.get_current_user())
if auth:
user_name=tweepy.API(auth).me().screen_name
showtmp=diff.getemail(login_name)
self.response.out.write("<a href='/oauth_logout'>%s OAuth Logout</a><br /><br />\n"%user_name)
booltmp=""
if showtmp.fo:
booltmp="checked=\"true\""
delta=0
if showtmp.timezone:
delta=showtmp.timezone
if tweepy.API(auth).test():
timeout=""
for i in range(25):
if i-12!=delta:
timeout+="<option value =\"%d\">%d</option>"%(i-12,i-12)
else:timeout+="<option value =\"%d\" selected=\"selected\">%d</option>"%(i-12,i-12)
self.response.out.write("""
<font color="#FF0000">if you put your email below,when you get new unfo i will send email to you</font>
<table>
<tr>
<form action="/set" method="post"><table>
<div><input type="text" name="email" size="20" value="%s"></div>
<div><input name="fo" type="checkbox" %s> new follower mail notify</div>
<div><select name="timezone">"""%(showtmp.email,booltmp)+
timeout+"""
</select> timezone</div>
<div><input type="submit" value="save change"></div>
</table></form>
</tr><br /><tr>
<form action="/clear" method="post">
<div><input type="submit" value="clear all the data"></div>
</form>
</tr>
</table>""")
diff.Diff(auth,user_name,1,0,login_name);
tmp=db.GqlQuery('SELECT * FROM Show WHERE name=:1',user_name)
datatmp=tmp.get()
self.response.out.write('<p><font color="#FF0000">new unfo:</font></p>')
self.response.out.write("<table>")
datatmp.unfo.reverse()
for i in datatmp.unfo:
s=i.split("@")
self.response.out.write("<tr><td><a href=\"http://twitter.com/"+s[0]+"\">"+s[0]+"</a></td><td>"+s[1]+"</td></tr>")
self.response.out.write("</table>")
self.response.out.write('<p><font color="#FF0000">new fo:</font></p>')
self.response.out.write("<table>")
datatmp.fo.reverse()
for i in datatmp.fo:
s=i.split("@")
self.response.out.write("<tr><td><a href=\"http://twitter.com/"+s[0]+"\">"+s[0]+"</a></td><td>"+s[1]+"</td></tr>")
self.response.out.write("</table>")
else:
self.response.out.write('OAuth Error.<br />\n')
else:
self.response.out.write("<a href='/auth'>OAuth Login</a>\n")
self.response.out.write("</body></html>")
def post(self):
name=users.get_current_user()
diff.Diff(getauth(name),tweepy.API(getauth(name)).me().screen_name,1,1,name);
self.redirect('/')
def Run(args, parser):
if not args.input_file.endswith('.size'):
parser.error('Input must end with ".size"')
if args.diff_with and not args.diff_with.endswith('.size'):
parser.error('Diff input must end with ".size"')
elif args.diff_with and not args.tree_view_ui:
parser.error('Diffs only supported in --tree-view-ui mode')
if args.tree_view_ui and args.method_count:
parser.error('--method-count is no longer supported as a command line '
'flag, use the client-side options instead.')
logging.info('Reading .size file')
size_info = archive.LoadAndPostProcessSizeInfo(args.input_file)
if args.diff_with:
before_size_info = archive.LoadAndPostProcessSizeInfo(args.diff_with)
after_size_info = size_info
size_info = diff.Diff(before_size_info, after_size_info)
symbols = size_info.raw_symbols
if args.method_count:
symbols = symbols.WhereInSection('m')
elif not args.include_bss:
symbols = symbols.WhereInSection('b').Inverted()
if args.tree_view_ui:
size_header = 'Delta size' if args.diff_with else 'Size'
template_src = os.path.join(os.path.dirname(__file__),
'template_tree_view')
_CopyTreeViewTemplateFiles(template_src, args.report_dir, size_header)
logging.info('Creating JSON objects')
tree_root = _MakeTreeViewList(symbols, args.min_symbol_size)
logging.info('Serializing JSON')
with open(os.path.join(args.report_dir, 'data.js'), 'w') as out_file:
out_file.write('var tree_data=`')
# Use separators without whitespace to get a smaller file.
json.dump(tree_root,
out_file,
ensure_ascii=False,
check_circular=False,
separators=(',', ':'))
out_file.write('`')
else:
# Copy report boilerplate into output directory. This also proves that the
# output directory is safe for writing, so there should be no problems
# writing the nm.out file later.
template_src = os.path.join(os.path.dirname(__file__), 'template')
_CopyTemplateFiles(template_src, args.report_dir)
logging.info('Creating JSON objects')
tree_root = _MakeCompactTree(symbols, args.min_symbol_size,
args.method_count)
logging.info('Serializing JSON')
with open(os.path.join(args.report_dir, 'data.js'), 'w') as out_file:
out_file.write('var tree_data=')
# Use separators without whitespace to get a smaller file.
json.dump(tree_root,
out_file,
ensure_ascii=False,
check_circular=False,
separators=(',', ':'))
logging.warning('Report saved to %s/index.html', args.report_dir)
def identify(config, target, cms, ratio=0.95, diff404=None, summary=False):
base_url = target.rstrip('/')
if not base_url.startswith('http'): base_url = 'http://{}'.format(base_url)
dbh = dbhandler.Handler(db=config.db)
sess = http.Session()
tags = dbh.tags(cms=cms)
total = len(tags)
requests = 0
prev_response_tags = []
rand_str = lambda: ''.join([
random.choice(string.letters) for _ in xrange(random.randrange(8, 17))
])
not_found = sess.get(base_url + '/' + (diff404 or rand_str()),
verify=False)
not_found_code = not_found.status_code
not_found_data = not_found.content
not_found_ratio = lambda _: diff.Diff(not_found_data, _).ratio()
response_tags = None
offset = 0
while True:
id = dbh.query_next(cms=cms, tags=tags, offset=offset)
logging.debug('requesting {}'.format(id.url))
response = sess.get(base_url + id.url)
requests += 1
code = response.status_code
data = response.content
url = response.url
file = id.url
whitespace = string.whitespace.replace(' ', '')
printable = lambda data: ''.join(
[_ for _ in data if _ not in whitespace])
logging.debug('response: {} {}'.format(code, printable(data)[:66]))
if code in [500, 502, 503]:
logging.critical('server down')
break
elif code == 404 or (code == not_found_code
and not_found_ratio(data) >= ratio):
logging.warning('{} not found {}'.format(code, url))
response_tags = dbh.tags_via_url(cms=cms, url=file)
tags = list(set(tags).difference(response_tags))
elif code == 200:
response_hash = hash.hash(response.content)
logging.debug('{} {}'.format(url, response_hash))
response_tags = dbh.tags_via_hash(cms=cms, hash=response_hash)
tags = list(set(tags).intersection(response_tags))
elif code == 403:
logging.warning('{} access denied {}'.format(code, url))
offset += 1
else:
logging.critical('unrecognized http status code {}'.format(code))
break
if code != 403: offset = 0
if prev_response_tags == response_tags:
logging.info('duplicates detected')
for tag in tags:
logging.info('{}'.format(tag))
break
prev_response_tags = response_tags
if len(tags) == 1:
logging.info('match {}'.format(tags[0]))
break
if summary:
logging.info('{} cms versions exist'.format(total))
logging.info('{} cms versions detected'.format(len(tags)))
logging.info('{} requests sent'.format(requests))
