def inconsistent_lib():
inconsistent_data = read_json("../action-8-6.json")
result = {}
for proj_id in inconsistent_data:
proj_data = inconsistent_data[proj_id]
for jar in proj_data:
jar = "__fdse__".join(jar.split("__fdse__")[0:2])
if jar not in result:
result[jar] = []
if proj_id not in result[jar]:
result[jar].append(proj_id)
print(len(result))
write_json_format(output_dir + "inconsistent.json", result)
datas = read_json(output_dir + "inconsistent.json")
new_data = {}
for lib in datas:
new_data[lib] = len(datas[lib])
sorted_usage = sorted(new_data.items(), key=lambda d: d[1], reverse=True)
sorted_usage = sorted_usage[:20]
print(sorted_usage)
sorted_usage = sorted_usage[::-1]
print(sorted_usage)
values = [value for key, value in sorted_usage]
keys = [key for key, value in sorted_usage]
draw_barh(keys, values, "The Number of Projects (#)")
def json_to_format():
dir = "call_graph_preprocessed"
files = os.listdir(dir)
for file in files:
# if file != "1383.txt":
# continue
data = read_json(os.path.join(dir, file))
write_json_format("format/" + file.replace(".txt", ".json"), data)
def project_id2name():
proj_dict = get_proj_dict()
data = read_json("datas/large_images.json")
new_data = {}
for id in data:
proj_name = proj_dict[id].replace("/", " ")
print(proj_name)
new_data[proj_name] = data[id]
write_json_format("datas/large_images.json", new_data)
def maven_count():
projects = read_json("E:/data/200_plus_with_type.txt")
print(len(projects))
count = 0
maven_projs = []
for proj in projects:
_type = proj["type"]
if _type == "maven":
count += 1
maven_projs.append(proj)
print(count)
write_json_format("E:/data/200_plus_maven.txt", maven_projs)
def no_module_maven_projs():
modules = read_json("C:/Users/yw/Desktop/modules_maven.json")
print(len(modules))
count = 0
new_modules = {}
for proj in modules:
if len(modules[proj]) == 0:
count += 1
else:
new_modules[proj] = modules[proj]
print(count)
print(len(new_modules))
write_json_format("E:/data/multiversion/modules_maven.json", new_modules)
def get_new_version_index():
lib_jar_pair = read_json("datas/lib_jar_pair.txt")
json_data = read_json("../tongjiresult-8-5.json")
# print(len(json_data))
# return
for project in json_data:
# if project != "508":
# continue
# if project != "751":
# continue
if not os.path.exists("call_graph_preprocessed/" + project + ".txt"):
# sys.stderr.write(project + "\n")
continue
# continue
print(project)
call_graph = read_json("call_graph_preprocessed/" + project + ".txt")
proj_call_count = read_json(
"E:/data/RQ1/api_call/total_with_count_preprocessed/" + project +
".txt")
project_obj = json_data[project]
for lib in project_obj:
# print(lib)
trees = project_obj[lib]
# jar_result = {}
for tree_id in trees:
subtree = trees[tree_id]
for entry in subtree:
usePostion = entry["usePostion"]
version = entry["resolved_version"]
if not usePostion.endswith("/pom.xml"):
_module = usePostion.replace("pom.xml", "")
else:
_module = usePostion.replace("/pom.xml", "")
lib_key = version + "__fdse__" + lib
# if lib_key in lib_jar_pair:
# lib转换成jar名
jar_name = lib_jar_pair[lib_key]
index_obj = None
# 从call graph中寻找
find_in_call_graph = False
for java_file in call_graph:
# print(java_file)
java_file_obj = call_graph[java_file]
# module对应，jar包对应
if _module == java_file_obj[
"module"] and jar_name in java_file_obj:
jar_obj = java_file_obj[jar_name]
# todo: jar包下面的api为空，没有候选可推荐的jar包
if len(jar_obj) == 0:
continue
if not find_in_call_graph:
index_obj = {} # 初始化
find_in_call_graph = True
# # todo: jar包下面的api为空，没有候选可推荐的jar包
# if len(jar_obj) == 0:
# if len(index_obj) == 0:
# index_obj["type"] = 'no_new_jar'
# new_dict = {"api_count": 0, "index_value": 0, "methods_in_modify_api": [],
# "delete_api_count": 0, "modify_api_count": 0}
# index_obj[jar_name] = new_dict
for api in jar_obj:
api_obj = jar_obj[api]
# todo: 推荐的new jar为空，说明没有新版本可推
if len(api_obj) == 0:
if len(index_obj) == 0:
index_obj["type"] = 'no_new_jar'
new_dict = {
"api_count": 0,
"index_value": 0,
"methods_in_modify_api": [],
"delete_api_count": 0,
"modify_api_count": 0
}
index_obj[jar_name] = new_dict
# index_obj[jar_name] = {"api_count": 0,"index_value": 0,"methods_in_modify_api": [],"delete_api_count": 0,"modify_api_count": 0},
elif index_obj["type"] != 'no_new_jar':
sys.stderr.write(
'no_new_jar type conflict : ' +
java_file + "(" + jar_name + ")")
sys.exit(0)
for new_jar in api_obj:
if new_jar not in index_obj:
index_obj[new_jar] = {}
# index_obj[new_jar]["api_count"] = 1
# index_obj[new_jar]["api_count"] = proj_call_count[java_file][api]
index_obj[new_jar]["api_count"] = [
proj_call_count[java_file][api]
]
index_obj[new_jar][
"delete_api_count"] = []
index_obj[new_jar][
"modify_api_count"] = []
index_obj[new_jar][
"methods_in_modify_api"] = []
index_obj[new_jar]["index_value"] = 0
else:
# index_obj[new_jar]["api_count"] += 1
# index_obj[new_jar]["api_count"] += proj_call_count[java_file][api]
index_obj[new_jar]["api_count"].append(
proj_call_count[java_file][api])
new_jar_obj = api_obj[new_jar]
# todo: api can't find
if api not in new_jar_obj:
if len(new_jar_obj) == 0:
# index_obj[new_jar]["delete_api_count"] += 1
# index_obj[new_jar]["index_value"] += 1
# index_obj[new_jar]["delete_api_count"] += proj_call_count[java_file][api]
index_obj[new_jar][
"delete_api_count"].append(
proj_call_count[java_file]
[api])
index_obj[new_jar][
"index_value"] += proj_call_count[
java_file][api]
else:
sys.stderr.write('KeyError : ' +
java_file + "(" +
jar_name + ":" +
api + ")")
sys.exit(0)
elif new_jar_obj[api] == "jar not found" or new_jar_obj[api] == "class not found" or \
new_jar_obj[api] == "method not found" or new_jar_obj[api] == "jdk method":
# index_obj[new_jar]["delete_api_count"] += 1
# index_obj[new_jar]["index_value"] += 1
# index_obj[new_jar]["delete_api_count"] += proj_call_count[java_file][api]
index_obj[new_jar][
"delete_api_count"].append(
proj_call_count[java_file]
[api])
index_obj[new_jar][
"index_value"] += proj_call_count[
java_file][api]
else:
total = 0
delete = 0
modify = 0
add = 0
for method in new_jar_obj:
# if method != api:
total += 1
if new_jar_obj[method] == "jar not found" or new_jar_obj[
method] == "class not found" or new_jar_obj[
method] == "method not found" or new_jar_obj[
method] == "jdk method":
delete += 1
elif new_jar_obj[
method] == "modify":
modify += 1
elif new_jar_obj[method] == "add":
add += 1
if delete != 0 or modify != 0 or add != 0:
# index_obj[new_jar]["modify_api_count"] += 1
# index_obj[new_jar]["methods_in_modify_api"].append([total, delete, modify, add])
# index_obj[new_jar]["modify_api_count"] += proj_call_count[java_file][api]
index_obj[new_jar][
"modify_api_count"].append(
proj_call_count[java_file]
[api])
for i in range(
0,
proj_call_count[java_file]
[api]):
index_obj[new_jar][
"methods_in_modify_api"].append(
[
total, delete,
modify, add
])
if total > 0:
# index_obj[new_jar]["index_value"] += (delete + modify + add) / total
index_obj[new_jar][
"index_value"] += (
delete + modify + add
) / total * proj_call_count[
java_file][api]
# todo : module 和 jar包对应不上，说明call graph中没有用到该第三方库的API，则index={}
if not find_in_call_graph:
index_obj = {}
index_obj["type"] = 'no_api_use'
entry["index"] = index_obj
write_json_format("datas/tongji_with_index.json", json_data)
def data2pdf():
proj_dict = get_proj_dict()
table_style = [('GRID', (0, 0), (-1, -1), 0.3, colors.grey),
('BACKGROUND', (0, 0), (-1, 0), colors.lightgrey),
('VALIGN', (0, 0), (-1, -1), 'MIDDLE')]
jar_box_style = [('BOX', (0, 0), (-1, -1), 0.8, colors.darksalmon)]
dir = "datas/pdf_data_unify"
output_dir = "datas/pdf_unify/"
large_images = read_json("datas/large_images.json")
# dir = "datas/pdf_data"
# output_dir = "datas/pdf/"
files = os.listdir(dir)
# error_proj = ['1342', '1238', '1107', '1344', '1356', '136', '1383', '1419', '1468', '148', '1520', '1582', '1716', '1758', '1770', '1784', '1866', '1878', '2004', '2075', '217', '2209', '236', '2635', '2834', '2886', '2973', '2975', '3105', '3186', '32', '3214', '3220', '3277', '3568', '3606', '3648', '3822', '383', '4049', '407', '4288', '5033', '508', '5538', '572', '60', '692', '709', '751', '83']
# print(len(error_proj))
error_proj = []
for file in files:
proj_name = proj_dict[file.replace(".json", "")].replace("/", " ")
print(proj_name)
# if os.path.exists(output_dir + file.replace(".json", ".pdf")):
# continue
if os.path.exists(output_dir + proj_name + ".pdf"):
continue
print(file)
proj_data = read_json(os.path.join(dir, file))
story = []
# title
rpt_title = title("Project : " + proj_dict[file.replace(".json", "")],
1)
story.append(Paragraph(rpt_title, normalStyle))
# definition
add_definition(story)
# overview
inconsistent_lib_cnt = 0
if "inconsistent_lib" in proj_data:
inconsistent_lib_cnt = proj_data["inconsistent_lib"]
add_overview(story, normalStyle, inconsistent_lib_cnt,
len(proj_data["ununified_output"]))
if "multilib_output" in proj_data:
story.append(
Paragraph(title('II. Inconsistent Libraries', 2),
normalStyle))
multi_data = proj_data["multilib_output"]
multi_jar_cnt = 0
for jar in multi_data:
multi_jar_cnt += 1
# if jar != "org.easymock__fdse__easymock__fdse__jar":
# continue
one_box = []
# jar box
one_box.append([
Paragraph(
title(
str(multi_jar_cnt) + '. ' +
":".join(jar.split("__fdse__")[:2]), 3),
normalStyle)
])
# Summery
desc_dict = multi_data[jar]["desc_dict"]
# add_summary(one_box,desc_dict,None,None,":".join(desc_dict["libraryname"].split("__fdse__")[:2]),str(len(set(multi_data[jar]["module_versions"].keys()))),True)
add_summary(
one_box, desc_dict, None, None,
":".join(desc_dict["libraryname"].split("__fdse__")[:2]),
str(
len(
set(
dict(multi_data[jar]
["module_versions"]).values()))), True)
# Multiple versions
multi_versions(one_box, table_style, multi_data[jar])
# Unify Recommendation
# title2 = '<para autoLeading="off" fontSize=14 align=left leading=16 textColor="black"><b><font>Unify Recommendation</font></b></para>'
# one_box.append([Paragraph(title2, normalStyle)])
# one_box.append([Paragraph(title("Harmonization Recommendation", 4), normalStyle)])
one_box.append([
Paragraph(
'<para fontSize=13 face="Times" leading=17><b> Harmonization Recommendation</b></para>',
normalStyle)
])
# subtrees
for tree_id in multi_data[jar]["tree_actions"]:
action_version = None if multi_data[jar]["tree_actions"][
tree_id] == "no action" else multi_data[jar][
"tree_actions"][tree_id]["action_version"]
action_pos = None if multi_data[jar]["tree_actions"][
tree_id] == "no action" else multi_data[jar][
"tree_actions"][tree_id]["action_pos"]
# one tree
tree_indices = multi_data[jar]["tree_indices"][tree_id]
subtree_numbers = multi_data[jar]["tree_numbers"][tree_id]
# modules_list = list(OrderedDict(tree_indices).keys())
# modules_str = ', '.join(modules_list[:50]) + '<br/> ···<br/>' + modules_list[-1] + "<br/>" if len(modules_list) > 50 else ', '.join(modules_list)
modules_str = "The inconsistent library versions in Modules " + str(
subtree_numbers[0]) + "-" + str(subtree_numbers[1])
# content = modules_str if multi_data[jar]["tree_actions"][tree_id] == "no action" else modules_str + ' → ' + action_version
content = modules_str + ' can not be harmonized into a suitable version.' if multi_data[
jar]["tree_actions"][
tree_id] == "no action" else modules_str + ' are harmonized into version ' + action_version + '.'
text = '<para fontSize=12 align=left textColor = "darkblue" leading=13>' + content + '</para>'
one_box.append([Paragraph(text, normalStyle)])
one_box.append([
Paragraph('<para leading=6> <br/></para>',
normalStyle)
])
if multi_data[jar]["tree_actions"][tree_id] == "no action":
# text = '<para autoLeading="off" fontSize=9 align=left>There is no new version to recommend.</para>'
# one_box.append([Paragraph(text, normalStyle)])
continue
# add_image()
add_image(one_box, file, jar, tree_id)
# efforts
add_efforts(one_box, table_style, tree_indices,
subtree_numbers)
# # files related
# add_files_related(one_box, table_style, multi_data[jar]["tree_files"][tree_id])
story = story + list(np.transpose(one_box)[0])
# one_box.append([Paragraph('<para align=center leading=2><br/><br/></para>', normalStyle)])
# border_table = Table(one_box)
# border_table.setStyle(border_style)
# story.append(border_table)
story.append(Paragraph('<para><br/></para>', normalStyle))
ununified_data = proj_data["ununified_output"]
if len(ununified_data) > 0:
if "multilib_output" in proj_data:
story.append(
Paragraph(title('III. False Consistent Libraries', 2),
normalStyle))
else:
story.append(
Paragraph(title('II. False Consistent Libraries', 2),
normalStyle))
ununified_jar_cnt = 0
for jar in ununified_data:
ununified_jar_cnt += 1
# border_box
one_box = []
# jar box
one_box.append([
Paragraph(
title(
str(ununified_jar_cnt) + '. ' +
":".join(jar.split("__fdse__")[:2]), 3), normalStyle)
])
# Summery
unify_version = ununified_data[jar]["module_versions"][0][1]
# unify_version = None
# for _module in ununified_data[jar]["module_versions"]:
# unify_version = ununified_data[jar]["module_versions"][_module]
# break
desc_dict = ununified_data[jar]["desc_dict"]
# add_summary(one_box,desc_dict, unify_version, ":".join(desc_dict["libraryname"].split("__fdse__")[:2]),str(len(set(ununified_data[jar]["module_versions"].keys()))),None,False)
add_summary(
one_box, desc_dict,
str(len(ununified_data[jar]["module_versions"])),
unify_version,
":".join(desc_dict["libraryname"].split("__fdse__")[:2]), None,
False)
# Module versions
add_ununified_module_version(one_box, table_style,
ununified_data[jar])
# trees
tree_numbers = ununified_data[jar]["tree_numbers"]
for tree_id in tree_numbers:
if tree_id in ununified_data[jar][
"tree_actions"] and ununified_data[jar][
"tree_actions"][tree_id] == "no action":
text = '<para fontSize=12 align=left textColor = "darkblue" leading=12>The false consistent library versions in Modules ' + str(
tree_numbers[tree_id][0]
) + "-" + str(
tree_numbers[tree_id][1]
) + ' can not reference a common property on a local POM file.</para>'
one_box.append([Paragraph(text, normalStyle)])
one_box.append([
Paragraph('<para leading=6> <br/></para>',
normalStyle)
])
continue
content = "The false consistent library versions in Modules " + str(
tree_numbers[tree_id][0]) + "-" + str(
tree_numbers[tree_id][1])
text = '<para fontSize=12 align=left textColor = "darkblue" leading=10>' + content + '</para>'
one_box.append([Paragraph(text, normalStyle)])
one_box.append([
Paragraph('<para leading=6> <br/></para>',
normalStyle)
])
# image
add_image(one_box, file, jar, tree_id)
story = story + list(np.transpose(one_box)[0])
# border_table = Table(one_box)
# border_table.setStyle(border_style)
# story.append(border_table)
story.append(Paragraph('<para><br/><br/></para>', normalStyle))
doc = SimpleDocTemplate(output_dir + proj_name + ".pdf")
# doc = SimpleDocTemplate(output_dir + file.replace(".json", ".pdf"),pagesize=A0)
# try:
doc.build(story)
# except:
# error_proj.append(file.replace(".json", ""))
# sys.stderr.write(file +" error\n")
print(error_proj)
write_json_format("datas/large_images.json", large_images)