def _generate_precondition(self, run, entry):
callee_location = entry["callee_location"]
titos = [
SourceLocation(t["line"], t["start"], t["end"])
for t in entry.get("titos", [])
]
if len(titos) > 200:
pre_key: Tuple[str, str, int] = (
entry["filename"],
entry["caller"],
len(titos),
)
if pre_key not in self.summary["bad_preconditions"]:
log.info("Bad Precondition: %s", str(pre_key))
self.summary["bad_preconditions"].add(pre_key)
titos = titos[:200]
return self._generate_raw_precondition(
run,
filename=entry["filename"],
caller=entry["caller"],
caller_port=entry["caller_port"],
callee=entry["callee"],
callee_port=entry["callee_port"],
callee_location=callee_location,
titos=titos,
sinks=entry["sinks"],
type_interval=entry["type_interval"],
features=entry["features"],
)
def _generate_trace_annotations(self, pre_id, frame_id, features) -> None:
for f in features:
if "extra_trace" in f:
annotation = f["extra_trace"]
location = annotation["position"]
self.graph.add_trace_annotation(
TraceFrameAnnotation.Record(
id=DBID(),
trace_frame_id=pre_id,
trace_frame_id2=frame_id,
location=SourceLocation(location["line"],
location["start"],
location["end"]),
message=annotation["msg"],
link=annotation.get("link", None),
trace_key=annotation.get("trace", None),
))
def _generate_issue(self, run, entry, callablesCount):
"""Insert the issue instance into a run. This includes creating (for
new issues) or finding (for existing issues) Issue objects to associate
with the instances.
Also create sink entries and associate related issues"""
trace_frames = []
preconditions = []
for p in entry["preconditions"]:
pre, tf = self._generate_issue_precondition(run, entry, p)
preconditions.append(pre)
trace_frames.append(tf)
postconditions = []
for p in entry["postconditions"]:
post, tf = self._generate_issue_postcondition(run, entry, p)
postconditions.append(post)
trace_frames.append(tf)
features = set()
for f in entry["features"]:
features.update(self._generate_issue_feature_contents(entry, f))
callable = entry["callable"]
handle = self._get_issue_handle(entry)
# TODO: record depth in issue_sink and issue_source assoc, but this can
# be different per instance, so should be stored with the instance.
initial_sources = {
self._get_shared_text(SharedTextKind.SOURCE, s)
for (s, _) in entry["initial_sources"]
}
final_sinks = {
self._get_shared_text(SharedTextKind.SINK, s)
for (s, _) in entry["final_sinks"]
}
issue = Issue.Record(
id=IssueDBID(),
code=entry["code"],
handle=handle,
callable=callable,
filename=entry["filename"],
status=IssueStatus.UNCATEGORIZED,
first_seen=run.date,
last_seen=run.date,
run_id=run.id,
visible=True,
json=self.summary["compress"](json.dumps(entry.get(
"json", {})).encode("utf-8")),
)
self.graph.add_issue(issue)
fix_info = None
fix_info_id = None
if entry.get("fix_info") is not None:
fix_info = IssueInstanceFixInfo.Record(id=DBID(),
fix_info=json.dumps(
entry["fix_info"]))
fix_info_id = fix_info.id
message = self._get_shared_text(SharedTextKind.MESSAGE,
entry["message"])
instance = IssueInstance.Record(
id=DBID(),
issue_id=issue.id,
location=self.get_location(entry),
filename=entry["filename"],
run_id=run.id,
fix_info_id=fix_info_id,
message_id=message.id,
rank=0,
min_trace_length_to_sources=self._get_minimum_trace_length(
entry["postconditions"]),
min_trace_length_to_sinks=self._get_minimum_trace_length(
entry["preconditions"]),
callable_count=callablesCount[issue.callable],
)
for sink in final_sinks:
self.graph.add_issue_instance_sink_assoc(instance, sink)
for source in initial_sources:
self.graph.add_issue_instance_source_assoc(instance, source)
if fix_info is not None:
self.graph.add_issue_instance_fix_info(instance, fix_info)
taint_locations = [
SourceLocation(t["line"], t["start"], t["end"])
for t in entry["taint_locations"]
]
if len(taint_locations) > 0:
instance.taint_locations = taint_locations
for precondition in preconditions:
self.graph.add_issue_instance_precondition_assoc(
instance, precondition)
for postcondition in postconditions:
self.graph.add_issue_instance_postcondition_assoc(
instance, postcondition)
for trace_frame in trace_frames:
self.graph.add_issue_instance_trace_frame_assoc(
instance, trace_frame)
for feature in features:
feature = self._get_shared_text(SharedTextKind.FEATURE, feature)
self.graph.add_issue_instance_shared_text_assoc(instance, feature)
self.graph.add_issue_instance(instance)
if "extracted_features" in entry:
self.summary["extracted_features"][
instance.id.local_id] = entry["extracted_features"]
def get_callable_location(entry):
line = entry["callable_line"]
return SourceLocation(line, entry["start"], entry["end"])
def get_location(entry, is_relative=False):
line = entry["line"]
if is_relative:
line -= entry["callable_line"]
return SourceLocation(line, entry["start"], entry["end"])
def _generate_raw_precondition(
self,
run,
filename,
caller,
caller_port,
callee,
callee_port,
callee_location,
titos,
sinks,
type_interval,
features,
):
lb, ub, preserves_type_context = self._get_interval(type_interval)
precondition = Precondition.Record(
id=DBID(),
caller=caller,
caller_condition=caller_port,
callee=callee,
callee_condition=callee_port,
callee_location=SourceLocation(
callee_location["line"],
callee_location["start"],
callee_location["end"],
),
filename=filename,
titos=titos,
message="",
run_id=run.id,
preserves_type_context=preserves_type_context,
type_interval_lower=lb,
type_interval_upper=ub,
)
trace_frame = TraceFrame.Record(
id=DBID(),
kind=TraceKind.PRECONDITION,
caller=caller,
caller_port=caller_port,
callee=callee,
callee_port=callee_port,
callee_location=SourceLocation(
callee_location["line"],
callee_location["start"],
callee_location["end"],
),
filename=filename,
titos=titos,
run_id=run.id,
preserves_type_context=preserves_type_context,
type_interval_lower=lb,
type_interval_upper=ub,
)
for (sink, depth) in sinks:
sink_record = self._get_shared_text(SharedTextKind.SINK, sink)
self.graph.add_precondition_sink_assoc(precondition, sink_record,
depth)
self.graph.add_trace_frame_leaf_assoc(trace_frame, sink_record,
depth)
self.graph.add_precondition(precondition)
self.graph.add_trace_frame(trace_frame)
self.graph.add_trace_frame_to_precondition(trace_frame, precondition)
self._generate_trace_annotations(precondition.id, trace_frame.id,
features)
return precondition, trace_frame
def _generate_raw_postcondition(
self,
run,
filename,
caller,
caller_port,
callee,
callee_port,
callee_location,
sources,
type_interval,
):
lb, ub, preserves_type_context = self._get_interval(type_interval)
postcondition = Postcondition.Record(
id=DBID(),
caller=caller,
callee=callee,
callee_location=SourceLocation(
callee_location["line"],
callee_location["start"],
callee_location["end"],
),
filename=filename,
run_id=run.id,
caller_condition=caller_port,
callee_condition=callee_port,
preserves_type_context=preserves_type_context,
type_interval_lower=lb,
type_interval_upper=ub,
)
trace_frame = TraceFrame.Record(
id=DBID(),
kind=TraceKind.POSTCONDITION,
caller=caller,
callee=callee,
callee_location=SourceLocation(
callee_location["line"],
callee_location["start"],
callee_location["end"],
),
filename=filename,
run_id=run.id,
caller_port=caller_port,
callee_port=callee_port,
preserves_type_context=preserves_type_context,
type_interval_lower=lb,
type_interval_upper=ub,
titos=[],
)
for (source, depth) in sources:
source_record = self._get_shared_text(SharedTextKind.SOURCE,
source)
self.graph.add_postcondition_source_assoc(postcondition,
source_record, depth)
self.graph.add_trace_frame_leaf_assoc(trace_frame, source_record,
depth)
self.graph.add_postcondition(postcondition)
self.graph.add_trace_frame(trace_frame)
self.graph.add_trace_frame_to_postcondition(trace_frame, postcondition)
return postcondition, trace_frame