def setUp(self):
'''
Don't run Moses server here (just incremental tests)
- Take all the .orig test files and create the IndexBuilders and ScopeAnalysts here
'''
self.testDir = Folder("./testing/test_files/")
self.clearTextFiles = self.fileSort(
self.testDir.baseFileNames("*.orig.js"))
print("Files: " + str(self.clearTextFiles))
self.obsfuscatedTextFiles = self.fileSort(
self.testDir.baseFileNames("*.obs.js"))
self.postTextFiles = self.fileSort(
self.testDir.baseFileNames("*.post_input.js"))
self.clearTextFiles = [
os.path.join(self.testDir.path, file)
for file in self.clearTextFiles
]
self.obsfuscatedTextFiles = [
os.path.join(self.testDir.path, file)
for file in self.obsfuscatedTextFiles
]
self.postTextFiles = [
os.path.join(self.testDir.path, file)
for file in self.postTextFiles
]
#print(self.testDir.path)
#print(self.clearTextFiles)
self.clearLexed = [Lexer(file) for file in self.clearTextFiles]
self.obsLexed = [Lexer(file) for file in self.obsfuscatedTextFiles]
self.postText = [
"".join(open(file, "r").readlines()) for file in self.postTextFiles
]
def setUp(self):
self.testDir = Folder("./testing//test_files/")
self.fileList = self.fileSort(self.testDir.baseFileNames("*.orig.js"))
self.fileList = [
os.path.join(self.testDir.path, nextFile)
for nextFile in self.fileList
]
print("FileList:")
print(self.fileList)
class testAST(unittest.TestCase):
def fileSort(self, fileList):
'''
Ensure that the list of files is sorted - e.g. test_file1 test_file2, etc.
'''
return sorted(fileList, key=lambda (x): int(x[9:10]))
def setUp(self):
self.testDir = Folder("./testing//test_files/")
self.fileList = self.fileSort(self.testDir.baseFileNames("*.orig.js"))
self.fileList = [
os.path.join(self.testDir.path, nextFile)
for nextFile in self.fileList
]
print("FileList:")
print(self.fileList)
def testFiles(self):
#Known bugs: The definitions of sum and numberEquals in test_file1 seem to be pointing to the wrong instance...
i = 1
lexed = Lexer(self.fileList[0])
ib = IndexBuilder(lexed.tokenList)
sa = ScopeAnalyst(self.fileList[0])
for variable in sa.nameDefScope2pos.keys():
print(
str(variable[0]) + " : " + str(sa.nameDefScope2pos[variable]) +
" -> " + str(ib.revFlatMat[sa.nameDefScope2pos[variable]]))
def setUp(self):
#self.pr = cProfile.Profile()
#self.pr.enable()
#self.testDir = Folder("/data/bogdanv/js_files/")
self.testDir = Folder("./experiments/samples/stress_sample/")
self.id_list = []
with open("./experiments/samples/stress.csv", 'r') as f:
for next_id in f:
self.id_list.append(next_id.strip())
print(self.id_list)
self.clearTextFiles = self.fileSort(self.id_list)
print("Files: " + str(self.clearTextFiles))
self.clearTextFiles = [
os.path.join(self.testDir.path, file)
for file in self.clearTextFiles
]
self.client = MosesClient("./testing/performance_output/")
class testLmQuery(unittest.TestCase):
def setUp(self):
self.testDir = Folder("./testing/test_files/")
self.fileList = self.testDir.baseFileNames("*.querytest")
self.fileList = [os.path.join(self.testDir.path, nextFile) for nextFile in self.fileList]
print("FileList:")
print(self.fileList)
def testFiles(self):
q = LMQuery("/data/bogdanv/deobfuscator/experiments/corpora/corpus.lm.500k/js.blm.lm")
start = time.time()
for file in self.fileList:
with open(file, 'r') as f:
for line in f:
(stat, prob1, msg) = q.run(line)
(stat, prob2, msg) = q.queryServer(line)
self.assertAlmostEqual(prob1, prob2, places = 2)
end = time.time()
print(end-start)
def getFileId(filename):
'''
Given a file path of the form [/.../.../]<num>.<exts>
Return <num>
Assumes there are no "/" in <exts>, but may be "." in the directory path if its included.
'''
base = ntpath.basename(filename)
return int(base[:base.find(".")])
inputFile = sys.argv[1]
fileIds = set()
testDir = Folder("/home/ccasal/jsnaughty/testing/consistencyFailureFiles/")
#print(testDir.baseFileNames("*.js"))
#fileIds = set([getFileId(nextFile) for nextFile in testDir.baseFileNames("*.js")])
with open(inputFile, 'r') as f:
for line in f:
#files.append(line.split(")")[0].replace("(", "."))
fileIds.add(int(line.split("(")[0]))
print(fileIds)
renamings = ["hash_def_one_renaming", "no_renaming"]
consistency = ["", "freqlen", "lmdrop", "lm"]
with open("/home/ccasal/jsnaughty/testing/copyFailures.sh", 'w') as f:
for fileId in fileIds:
def setUp(self):
self.testDir = Folder("./testing//test_files/")
self.fileList = self.fileSort(self.testDir.baseFileNames("*.orig.js"))
self.fileList = [
os.path.join(self.testDir.path, nextFile)
for nextFile in self.fileList
]
print("FileList:")
print(self.fileList)
#Manual solutions:
self.file_definitions = {}
self.file_definitions[1] = {}
self.file_definitions[1]["geom2d"] = [0]
self.file_definitions[1]["sum"] = [1]
self.file_definitions[1]["numeric"] = [1]
self.file_definitions[1]["numberEquals"] = [1]
self.file_definitions[1]["a"] = [2]
self.file_definitions[1]["Vector2d"] = [3]
self.file_definitions[1]["dotProduct"] = [8]
self.file_definitions[1]["equals"] = [11]
self.file_definitions[1]["mix"] = [15]
self.file_definitions[1]["k"] = [16]
self.file_definitions[1]["vector"] = [8, 11]
self.file_definitions[1]["x"] = [3]
self.file_definitions[1]["y"] = [3]
self.file_definitions[1]["epsilon"] = [11]
self.file_definitions[1]["dest"] = [15]
self.file_definitions[1]["src"] = [15]
self.file_definitions[5] = {}
self.file_definitions[5]["button"] = [5, 19]
self.file_definitions[5]["e"] = [5, 19]
self.file_definitions[5]["options"] = [5, 19]
self.file_definitions[5]["win"] = [6]
self.file_definitions[5]["frm"] = [7]
self.file_definitions[5]["store"] = [8]
self.file_definitions[5]["customer"] = [22] # the 10 is global, skip
self.file_definitions[5]["grid"] = [22, 29]
self.file_definitions[5]["cp"] = [23, 30]
self.file_definitions[5]["Ext"] = [0]
self.file_definitions[5]["mode"] = [9]
self.file_definitions[6] = {}
self.file_definitions[6]["feather"] = [0]
self.file_definitions[6]["cardmagic"] = [3]
self.file_definitions[6]["newCard"] = [9]
self.file_definitions[6]["nPlayerId"] = [9]
self.file_definitions[7] = {}
self.file_definitions[7]["require"] = [0]
self.file_definitions[7]["fs"] = [0]
self.file_definitions[7]["path"] = [2]
self.file_definitions[7]["__dirname"] = [2]
self.file_definitions[7]["ignoreList"] = [2]
self.file_definitions[7]["describe"] = [4]
self.file_definitions[7]["done"] = [5]
self.file_definitions[7]["it"] = [5]
self.file_definitions[7]["err"] = [6]
self.file_definitions[7]["file"] = [6]
self.file_definitions[8] = {}
self.file_definitions[8]["module"] = [0]
self.file_definitions[8]["testCompile"] = [2, 8]
self.file_definitions[8]["promise"] = [5, 11]
self.file_definitions[8]["anonymous"] = [
2, 4, 8, 10
] # I don't know if I agree with this?
self.file_definitions[9] = {}
self.file_definitions[9]["document"] = [0]
self.file_definitions[9]["studentSelector"] = [1]
self.file_definitions[9]["courseSelector"] = [1]
self.file_definitions[9]["$"] = [0] #The other ones aren't?
self.file_definitions[9]["data"] = [4, 12]
self.file_definitions[9]["i"] = [5, 13]
self.file_definitions[9]["s"] = [5]
self.file_definitions[9]["student"] = [6, 14]
self.file_definitions[9]["t"] = [13]
#self.file_definitions[9]["tablesorter"] = [20]
#self.file_definitions[9]["sortList"] = [21]
self.file_definitions[9]["studentRows"] = [35]
self.file_definitions[9]["filterButtons"] = [35]
self.file_definitions[9]["e"] = [36]
#self.file_definitions[9]["on"] = [36]
self.file_definitions[9]["activeRows"] = [37]
self.file_definitions[9]["inactiveRows"] = [37]
self.file_definitions[9]["grade"] = [37]
#self.file_definitions[9]["filter"] = [42]
self.file_definitions[10] = {}
self.file_definitions[10]["BaseView"] = [8]
self.file_definitions[10]["setup"] = [6]
self.file_definitions[10]["teardown"] = [13]
self.file_definitions[10]["test"] = [41]
self.file_definitions[10]["document"] = [7]
self.file_definitions[10]["assert"] = [11]
self.file_definitions[10]["stubAddListener"] = [7]
self.file_definitions[10]["window"] = [39]
self.file_definitions[10]["stubReqWakeLock"] = [17]
self.file_definitions[10]["overflowCall"] = [42]
self.file_definitions[10]["suite"] = [4]
self.file_definitions[10]["require"] = [2]
self.file_definitions[10]["options"] = [5]
self.file_definitions[11] = {}
self.file_definitions[11]["func"] = [4]
self.file_definitions[11]["elem"] = [8]
self.file_definitions[11]["layer"] = [11]
self.file_definitions[11]["data"] = [12]
#Apparently older versions of javascript don't support block level scoping.
#This also should be 39 probably, but 46 has the orig tag for some reason
#I don't see any way to fix this under the constraints of the current ast
self.file_definitions[11]["name"] = [13, 46] #,39,52]
self.file_definitions[11]["tmp"] = [14]
self.file_definitions[11]["parent"] = [18]
self.file_definitions[11]["node"] = [27]
self.file_definitions[11]["stack"] = [28]
self.file_definitions[11]["args"] = [36]
self.file_definitions[11]["b"] = [37]
#This has the same deal as name does
self.file_definitions[11]["i"] = [38] #,45,50]
self.file_definitions[11]["a"] = [44]
self.file_definitions[11]["arg"] = [51]
class testAST(unittest.TestCase):
def fileSort(self, fileList):
'''
Ensure that the list of files is sorted - e.g. test_file1 test_file2, etc.
'''
return sorted(fileList, key=lambda (x): int(x[9:x.find(".")]))
def setUp(self):
self.testDir = Folder("./testing//test_files/")
self.fileList = self.fileSort(self.testDir.baseFileNames("*.orig.js"))
self.fileList = [
os.path.join(self.testDir.path, nextFile)
for nextFile in self.fileList
]
print("FileList:")
print(self.fileList)
#Manual solutions:
self.file_definitions = {}
self.file_definitions[1] = {}
self.file_definitions[1]["geom2d"] = [0]
self.file_definitions[1]["sum"] = [1]
self.file_definitions[1]["numeric"] = [1]
self.file_definitions[1]["numberEquals"] = [1]
self.file_definitions[1]["a"] = [2]
self.file_definitions[1]["Vector2d"] = [3]
self.file_definitions[1]["dotProduct"] = [8]
self.file_definitions[1]["equals"] = [11]
self.file_definitions[1]["mix"] = [15]
self.file_definitions[1]["k"] = [16]
self.file_definitions[1]["vector"] = [8, 11]
self.file_definitions[1]["x"] = [3]
self.file_definitions[1]["y"] = [3]
self.file_definitions[1]["epsilon"] = [11]
self.file_definitions[1]["dest"] = [15]
self.file_definitions[1]["src"] = [15]
self.file_definitions[5] = {}
self.file_definitions[5]["button"] = [5, 19]
self.file_definitions[5]["e"] = [5, 19]
self.file_definitions[5]["options"] = [5, 19]
self.file_definitions[5]["win"] = [6]
self.file_definitions[5]["frm"] = [7]
self.file_definitions[5]["store"] = [8]
self.file_definitions[5]["customer"] = [22] # the 10 is global, skip
self.file_definitions[5]["grid"] = [22, 29]
self.file_definitions[5]["cp"] = [23, 30]
self.file_definitions[5]["Ext"] = [0]
self.file_definitions[5]["mode"] = [9]
self.file_definitions[6] = {}
self.file_definitions[6]["feather"] = [0]
self.file_definitions[6]["cardmagic"] = [3]
self.file_definitions[6]["newCard"] = [9]
self.file_definitions[6]["nPlayerId"] = [9]
self.file_definitions[7] = {}
self.file_definitions[7]["require"] = [0]
self.file_definitions[7]["fs"] = [0]
self.file_definitions[7]["path"] = [2]
self.file_definitions[7]["__dirname"] = [2]
self.file_definitions[7]["ignoreList"] = [2]
self.file_definitions[7]["describe"] = [4]
self.file_definitions[7]["done"] = [5]
self.file_definitions[7]["it"] = [5]
self.file_definitions[7]["err"] = [6]
self.file_definitions[7]["file"] = [6]
self.file_definitions[8] = {}
self.file_definitions[8]["module"] = [0]
self.file_definitions[8]["testCompile"] = [2, 8]
self.file_definitions[8]["promise"] = [5, 11]
self.file_definitions[8]["anonymous"] = [
2, 4, 8, 10
] # I don't know if I agree with this?
self.file_definitions[9] = {}
self.file_definitions[9]["document"] = [0]
self.file_definitions[9]["studentSelector"] = [1]
self.file_definitions[9]["courseSelector"] = [1]
self.file_definitions[9]["$"] = [0] #The other ones aren't?
self.file_definitions[9]["data"] = [4, 12]
self.file_definitions[9]["i"] = [5, 13]
self.file_definitions[9]["s"] = [5]
self.file_definitions[9]["student"] = [6, 14]
self.file_definitions[9]["t"] = [13]
#self.file_definitions[9]["tablesorter"] = [20]
#self.file_definitions[9]["sortList"] = [21]
self.file_definitions[9]["studentRows"] = [35]
self.file_definitions[9]["filterButtons"] = [35]
self.file_definitions[9]["e"] = [36]
#self.file_definitions[9]["on"] = [36]
self.file_definitions[9]["activeRows"] = [37]
self.file_definitions[9]["inactiveRows"] = [37]
self.file_definitions[9]["grade"] = [37]
#self.file_definitions[9]["filter"] = [42]
self.file_definitions[10] = {}
self.file_definitions[10]["BaseView"] = [8]
self.file_definitions[10]["setup"] = [6]
self.file_definitions[10]["teardown"] = [13]
self.file_definitions[10]["test"] = [41]
self.file_definitions[10]["document"] = [7]
self.file_definitions[10]["assert"] = [11]
self.file_definitions[10]["stubAddListener"] = [7]
self.file_definitions[10]["window"] = [39]
self.file_definitions[10]["stubReqWakeLock"] = [17]
self.file_definitions[10]["overflowCall"] = [42]
self.file_definitions[10]["suite"] = [4]
self.file_definitions[10]["require"] = [2]
self.file_definitions[10]["options"] = [5]
self.file_definitions[11] = {}
self.file_definitions[11]["func"] = [4]
self.file_definitions[11]["elem"] = [8]
self.file_definitions[11]["layer"] = [11]
self.file_definitions[11]["data"] = [12]
#Apparently older versions of javascript don't support block level scoping.
#This also should be 39 probably, but 46 has the orig tag for some reason
#I don't see any way to fix this under the constraints of the current ast
self.file_definitions[11]["name"] = [13, 46] #,39,52]
self.file_definitions[11]["tmp"] = [14]
self.file_definitions[11]["parent"] = [18]
self.file_definitions[11]["node"] = [27]
self.file_definitions[11]["stack"] = [28]
self.file_definitions[11]["args"] = [36]
self.file_definitions[11]["b"] = [37]
#This has the same deal as name does
self.file_definitions[11]["i"] = [38] #,45,50]
self.file_definitions[11]["a"] = [44]
self.file_definitions[11]["arg"] = [51]
def testFiles(self):
tf = [1, 5, 6, 7, 8, 9, 10, 11]
#tf = [11]
for i in tf:
print("-----------------------------------------------------")
lexed = Lexer(self.fileList[i - 1])
ib = IndexBuilder(lexed.tokenList)
#print(ib)
sa = ScopeAnalyst(self.fileList[i - 1])
print(sa)
nameCount = {}
#TODO: Grab only the non-globals to look at (get the start key and look it up)
for variable in sa.nameDefScope2pos.keys():
start = sa.nameDefScope2pos[variable]
name = variable[0]
if (not sa.isGlobal[(name, start)]):
if (name in nameCount):
nameCount[name] += 1
else:
nameCount[name] = 1
print(
str(name) + " : " +
str(sa.nameDefScope2pos[variable]) + " -> " +
str(ib.revFlatMat[sa.nameDefScope2pos[variable]]) +
" Manual: " + str(self.file_definitions[i][name]))
assert (ib.revFlatMat[sa.nameDefScope2pos[variable]][0]
in self.file_definitions[i][name])
#Finally make sure that the count of definitions matches our manual check.
for name, count in nameCount.iteritems():
print(name + " : " + str(count) + " =?= " +
str(len(self.file_definitions[i][name])))
assert (len(self.file_definitions[i][name]) == count)
class variableMetricsTest(unittest.TestCase):
def asBool(self, string):
return string in ["TRUE", "true", "True", "t", "T", "1"]
def fileSort(self, fileList):
'''
Ensure that the list of files is sorted - e.g. test_file1 test_file2, etc.
'''
return sorted(fileList, key=lambda (x): int(x[9:x.find(".")]))
def setUp(self):
self.testDir = Folder("./testing//test_files/")
self.fileList = self.fileSort(self.testDir.baseFileNames("*.orig.js"))
self.fileList = [
os.path.join(self.testDir.path, nextFile)
for nextFile in self.fileList
]
print("FileList:")
print(self.fileList)
def testFiles(self):
#TODO: Automated checks against the files.
#Known bugs: The definitions of sum and numberEquals in test_file1 seem to be pointing to the wrong instance...
i = 1
for nextFile in self.fileList:
print(nextFile)
lexed = Lexer(nextFile)
ib = IndexBuilder(lexed.tokenList)
sa = ScopeAnalyst(nextFile)
s_min = ScopeAnalyst(
os.path.join(self.testDir.path, "test_file1.obs.js"))
#print(s_min.name2defScope)
#print("TokenList----------------------------------------------------------------")
#print(lexed.tokenList)
#print("Index Builder----------------------------------------------------------------")
#print(ib)
#print("Scope Analyst----------------------------------------------------------------")
#print(sa)
vm = VariableMetrics(sa, ib, lexed.tokenList)
#print("VM----------------------------------------------------------------")
#print(vm)
#print("VM----------------------------------------------------------------")
for var in vm.getVariables():
print(var)
print(
"Num Lines,Max Lines,Global Def,Global Usage,For,While,Literal Def,Literal Usage,Max Length Line,Ave Line Length"
)
print vm.getNameMetrics(var)
#Automated tests:
csv_file = os.path.join(self.testDir.path,
"test_file" + str(i) + ".csv")
print(csv_file)
if (os.path.exists(csv_file)):
with open(csv_file, 'r') as f:
csv_reader = csv.reader(f, delimiter=",")
#Skip header
next(csv_reader, None)
for row in csv_reader:
key = (row[0], row[1])
print(key)
(num_lines, max_lines, external_def, external_use,
in_for, in_while, literal_def, literal_use,
max_length_line,
ave_line_length) = vm.getNameMetrics(key)
self.assertTrue(num_lines == int(row[2]))
self.assertTrue(max_lines == int(row[3]))
self.assertTrue(external_def == self.asBool(row[4]))
self.assertTrue(external_use == int(row[5]))
self.assertTrue(in_for == int(row[6]))
self.assertTrue(in_while == int(row[7]))
self.assertTrue(literal_def == self.asBool(row[8]))
self.assertTrue(literal_use == int(row[9]))
self.assertTrue(max_length_line == int(row[10]))
self.assertAlmostEqual(ave_line_length,
float(row[11]),
places=3)
else:
print("no manually annotated csv file for: " + nextFile)
break
#Get all known names in the file.
known_names = set([name for name, value in sa.isGlobal.iteritems()])
for name, loc in ib.name2CharPositions.iteritems():
if (name not in known_names): #if never seen, its a global
base_global.add(name)
return [base_name, len(base_global)]
except:
return [base_name, None]
input_dir = sys.argv[1]
output_csv = sys.argv[2]
num_threads = int(sys.argv[3])
base_dir = Folder(input_dir)
fileList = base_dir.baseFileNames("*.js")
origList = [next for next in fileList if next.count(".") == 1]
toProcess = [(nextFile, os.path.join(base_dir.path, nextFile))
for nextFile in origList]
#print(fileList)
with open(output_csv, 'w') as g:
pass
pool = multiprocessing.Pool(processes=num_threads)
for result in pool.imap_unordered(processFile, toProcess):
#for file_pair in toProcess:
# result = processFile(file_pair)
print(result[0])
class defobfuscate_tests(unittest.TestCase):
def fileSort(self, fileList):
'''
Ensure that the list of files is sorted - e.g. test_file1 test_file2, etc.
'''
return sorted(fileList, key=lambda (x): int(x[9:x.find(".")]))
def setUp(self):
'''
Don't run Moses server here (just incremental tests)
- Take all the .orig test files and create the IndexBuilders and ScopeAnalysts here
'''
self.testDir = Folder("./testing/test_files/")
self.clearTextFiles = self.fileSort(
self.testDir.baseFileNames("*.orig.js"))
print("Files: " + str(self.clearTextFiles))
self.obsfuscatedTextFiles = self.fileSort(
self.testDir.baseFileNames("*.obs.js"))
self.postTextFiles = self.fileSort(
self.testDir.baseFileNames("*.post_input.js"))
self.clearTextFiles = [
os.path.join(self.testDir.path, file)
for file in self.clearTextFiles
]
self.obsfuscatedTextFiles = [
os.path.join(self.testDir.path, file)
for file in self.obsfuscatedTextFiles
]
self.postTextFiles = [
os.path.join(self.testDir.path, file)
for file in self.postTextFiles
]
#print(self.testDir.path)
#print(self.clearTextFiles)
self.clearLexed = [Lexer(file) for file in self.clearTextFiles]
self.obsLexed = [Lexer(file) for file in self.obsfuscatedTextFiles]
self.postText = [
"".join(open(file, "r").readlines()) for file in self.postTextFiles
]
#print(self.postText[0])
#print(self.clearLexed)
#quit()
def testIndexBuilder(self):
'''
Check that the index builder has correct values for the test files.
'''
ib1 = IndexBuilder(self.clearLexed[0].tokenList)
'''
# - map from (line,col) position to name
self.charPosition2Name = {}
# - map from name to list of (line,col) positions
self.name2CharPositions = {}
# - map from (line,col) position to flat position
self.flatMap = {}
# - map from flat position to (line,col)
self.revFlatMat = {}
# - map from (token_line, token_column) position in the
# bidimensional list of tokens to (line,col) text position
self.tokMap = {}
# - map from (line,col) position to (token_line, token_column)
# position in the bidimensional list of tokens
self.revTokMap = {}
'''
print([item[1] for item in self.clearLexed[0].tokenList])
print(ib1.charPosition2Name)
print(len(ib1.charPosition2Name))
#print(len(ib1.charPosition2Name) == 53)
#for i in range(0,22):
# linecount = 0
# for j in range(0, 110):
# if((i,j) in ib1.charPosition2Name):
# linecount += 1
# print("Line " + str(i+1) + " has " + str(linecount) + " variables.")
#Test charPosition2Name
self.assertTrue(len(ib1.charPosition2Name) == 53)
self.assertTrue(ib1.charPosition2Name[(0, 4)] == u'geom2d')
self.assertTrue(ib1.charPosition2Name[(2, 8)] == u'a')
self.assertTrue(ib1.charPosition2Name[(15, 13)] == u'mix')
self.assertTrue(ib1.charPosition2Name[(16, 17)] == u'k')
#Test name2charPositions
self.assertTrue(len(ib1.name2CharPositions) == 16)
self.assertTrue(
sum([
len(value)
for key, value in ib1.name2CharPositions.iteritems()
]) == 53)
self.assertTrue(len(ib1.name2CharPositions[u'x']) == 7)
self.assertTrue(len(ib1.name2CharPositions[u'Vector2d']) == 4)
#Test flatMap
self.assertTrue(ib1.flatMap[(1, 8)] == 34)
self.assertTrue(ib1.flatMap[(3, 22)] == 128)
#Test revFlatMap
#Typo Bug: revFlatMat or revFlatMap?
self.assertTrue(len(ib1.flatMap) == len(ib1.revFlatMat))
for key, value in ib1.flatMap.iteritems():
self.assertTrue(ib1.revFlatMat[value] == key)
#Test tokMap and revTokMap
#These are supposed to be different? Yes, includes maps to whitespace. (so leading whitespace also maps to identifiers)
print(len(ib1.tokMap))
print(len(ib1.revTokMap))
#self.assertTrue(len(ib1.tokMap) == len(ib1.revTokMap))
#i = 0
for key, value in ib1.tokMap.iteritems():
if (value in ib1.revTokMap.keys()):
self.assertTrue(ib1.revTokMap[value] == key)
#print("RevTokMap " + str(key) + " : " + str(value))
#print("TokMap " + str(value) + " : " + str(ib1.tokMap[value]))
#if(ib1.tokMap[value] != key):
# i += 1
# print("Error Case!!!")
#self.assertTrue(ib1.tokMap[value] == key)
#print("Error Cases " + str(i))
def testScopeAnalyst(self):
'''
TODO: Check that the scope analyst works properly
'''
#__main__.py in tools is a useful tool for examining these.
#print(self.obsfuscatedTextFiles[0])
#This doesn't work when run inside pyDev for some weird reason.
sa1 = ScopeAnalyst(self.obsfuscatedTextFiles[0])
#print(sa1)
#Not really sure how to test this effectively.
#Check (using minified file) if identifier name maps to different variables if
#they are in different scopes. Can look at __main__.py
#Variables: geom2d,t,i,r,x,y,n,e,o,u
#Why do x and y not appear in the variables? (Is it b/c they are not defined anywhere in this snippet?)
self.assertTrue(len(sa1.nameScopes[(u'geom2d')]) == 1)
self.assertTrue(len(sa1.nameScopes[(u'numeric')]) == 1)
self.assertTrue(len(sa1.nameScopes[(u't')]) == 3)
self.assertTrue(len(sa1.nameScopes[(u'i')]) == 1)
self.assertTrue(len(sa1.nameScopes[(u'r')]) == 4)
self.assertTrue(len(sa1.nameScopes[(u'n')]) == 4)
#self.assertTrue(len(sa1.nameScopes[(u'x')]) == 2)
#self.assertTrue(len(sa1.nameScopes[(u'y')]) == 2)
self.assertTrue(len(sa1.nameScopes[(u'u')]) == 1)
self.assertTrue(len(sa1.nameScopes[(u'e')]) == 1)
self.assertTrue(len(sa1.nameScopes[(u'o')]) == 1)
#isGlobal:
#print("IsGlobal-----------------------------------------------")
#print(sa1.isGlobal)
#print("IsGlobal-----------------------------------------------")
self.assertTrue(sa1.isGlobal[(u'geom2d', 4)] == True)
self.assertTrue(sa1.isGlobal[(u'i', 85)] == False)
self.assertTrue(True)
def testHashDefRenaming(self):
'''
TODO: Test the hashing functions are using the context correctly for both one and two line
options. Goals are to confirm a) correct line summarization b) consistency of naming
of the same variable. However, two different variables may map to the same name with
insufficient context.
'''
#print(self.obsfuscatedTextFiles[0])
ib1 = IndexBuilder(self.obsLexed[0].tokenList)
sa1 = ScopeAnalyst(self.obsfuscatedTextFiles[0])
RS = RenamingStrategies()
preRen = PreRenamer()
oneLine1 = preRen.rename(RS.HASH_ONE, ib1, sa1, True)
twoLine1 = preRen.rename(RS.HASH_TWO, ib1, sa1, True)
# oneLine1 = renameUsingHashDefLine(sa1, ib1, False, True)
# twoLine1 = renameUsingHashDefLine(sa1, ib1, True, True)
#print("OneLine1------------------------------------------------")
#print(oneLine1)
#print("TwoLine1------------------------------------------------")
#print(twoLine1)
#One line tests
lines = oneLine1.split("\n")
self.assertTrue(lines[0] == "var geom2d = function ( ) {")
#var <<var#=numeric.sum,=numeric.numberEquals;>> = numeric . sum , <<var=numeric.sum,#=numeric.numberEquals;>> = numeric . numberEquals ;
self.assertTrue(
lines[1] ==
"var <<var#=numeric.sum,=numeric.numberEquals;>> = numeric . sum , <<var=numeric.sum,#=numeric.numberEquals;>> = numeric . numberEquals ;"
)
self.assertTrue(
lines[3] ==
"function <<function#(,){>> ( <<function(#,){>> , <<function(,#){>> ) {"
)
self.assertTrue(lines[4] == "this . x = <<function(#,){>> ;"
) #Why is x not transformed? Global, can't change...
#print(lines[7])
self.assertTrue(
lines[7] == "u ( <<function#(,){>> , {"
) #Why is u not transformed? -> Because u's hash <<function#(,){>> is ALREADY IN USE IN THE SAME SCOPE!! (This is why u can be translated in 2-lines)
self.assertTrue(
lines[16] ==
"for ( var <<for(var#in)[]=[];>> in <<function(,#){>> ) <<function(#,){>> [ <<for(var#in)[]=[];>> ] = <<function(,#){>> [ <<for(var#in)[]=[];>> ] ;"
)
self.assertTrue(lines[20] == "Vector2d : <<function#(,){>>")
#Two line tests (TODO)
lines = twoLine1.split("\n")
self.assertTrue(lines[0] == "var geom2d = function ( ) {")
self.assertTrue(
lines[1] ==
"var <<var#=numeric.sum,=numeric.numberEquals;return#([this.x*.x,this.y*.y]);>> = numeric . sum , <<var=numeric.sum,#=numeric.numberEquals;return#(this.x,.x,)&&(this.y,.y,);>> = numeric . numberEquals ;"
)
# function <<function#(,){(#,{>> ( <<function(#,){this.x=#;>> , <<function(,#){this.y=#;>> ) {
self.assertTrue(
lines[3] ==
"function <<function#(,){(#,{>> ( <<function(#,){this.x=#;>> , <<function(,#){this.y=#;>> ) {"
)
self.assertTrue(lines[4] == "this . x = <<function(#,){this.x=#;>> ;"
) #Why is x not transformed? Global, can't change...
#u(r, {
# #<<function#(,){#(,{>> ( <<function#(,){(#,{>> , {
self.assertTrue(
lines[7] == "<<function#(,){#(,{>> ( <<function#(,){(#,{>> , {"
) # is transformed, but order seems backwards.
self.assertTrue(
lines[16] ==
"for ( var <<for(var#in)[]=[];for(varin)[#]=[];>> in <<function(,#){for(varin#)[]=[];>> ) <<function(#,){for(varin)#[]=[];>> [ <<for(var#in)[]=[];for(varin)[#]=[];>> ] = <<function(,#){for(varin#)[]=[];>> [ <<for(var#in)[]=[];for(varin)[#]=[];>> ] ;"
) #Not really two lines, but two references?
self.assertTrue(lines[20] == "Vector2d : <<function#(,){(#,{>>")
self.assertTrue(True)
def testMinifiableLines(self):
expected = {}
expected[0] = set([1, 2, 3, 4, 5, 7, 8, 9, 11, 12, 15, 16, 17, 20])
expected[5] = set([8, 9])
for i in [0, 5]:
ib = IndexBuilder(self.clearLexed[i].tokenList)
sa = ScopeAnalyst(self.clearTextFiles[i])
lines = sa.getMinifiableLines(ib)
print("i:" + str(i))
print(lines)
print(expected[i])
self.assertTrue(lines == expected[i])
text = ib.get_text_on_lines_wo_literals(lines)
print(text)
print(len(text.split("\n")))
print(len(expected[i]))
self.assertTrue(len(text.split("\n")) == len(expected[i]))