def get_taint_result(self,
pm_proxy,
pkg_name,
outdir,
configpath=None,
pkg_version=None,
cache_only=False):
taint_fname = pm_proxy.get_taint_fname(pkg_name=pkg_name,
pkg_version=pkg_version)
taint_file = join(outdir, taint_fname)
taint_result = None
if exists(taint_file):
logging.warning("get_taint_result: using cached taint_file %s!",
taint_file)
taint_result = ModuleStatic()
read_proto_from_file(taint_result, taint_file, binary=False)
else:
if cache_only:
logging.warning(
"skipping unprocessed pkg %s ver %s due to cache_only!",
pkg_name, pkg_version)
return taint_result
# download current package and analyze it
tempdir = tempfile.mkdtemp(prefix='taint-')
pm_proxy.download(pkg_name=pkg_name,
pkg_version=pkg_version,
outdir=tempdir)
tempdir_files = os.listdir(tempdir)
if len(tempdir_files) == 0:
logging.error("fail to download pkg %s ver %s", pkg_name,
pkg_version)
else:
pkg_file = join(tempdir, tempdir_files[0])
self.taint(inpath=pkg_file,
outfile=taint_file,
configpath=configpath,
pkg_name=pkg_name,
pkg_version=pkg_version)
if exists(taint_file):
taint_result = ModuleStatic()
read_proto_from_file(taint_result,
taint_file,
binary=False)
else:
logging.error("fail to run taint on downloaded package %s",
pkg_file)
shutil.rmtree(tempdir)
return taint_result
def reformat(apis_file, all_sources, all_sinks, json_result_file, outfile):
try:
results = json.load(open(json_result_file, 'r'))
except Exception as e:
logging.error("failed to load progpilot results in json: %s",
json_result_file)
return None
logging.warning("there are %d sources and %d sinks checked!",
len(all_sources), len(all_sinks))
# load the astgen config from file
config = AstLookupConfig()
read_proto_from_file(config, apis_file, binary=False)
logging.warning("loaded config with %d apis to check!", len(config.apis))
result = ModuleResult()
set_result(result=result,
apis=config.apis,
all_sources=all_sources,
all_sinks=all_sinks,
flows=results)
summary = ModuleSummary()
set_summary(summary=summary,
apis=config.apis,
all_sources=all_sources,
all_sinks=all_sinks,
new_sources=None,
new_sinks=None)
static = ModuleStatic()
static.flows.MergeFrom(result.flows)
static.dangers.MergeFrom(result.dangers)
static.sources.MergeFrom(summary.sources)
static.sinks.MergeFrom(summary.sinks)
static.taint_wrappers.MergeFrom(summary.taint_wrappers)
write_proto_to_file(proto=static, filename=outfile, binary=False)
def taint(self,
inpath,
outfile,
configpath=None,
pkg_name=None,
pkg_version=None):
analyze_path, is_decompress_path, outfile, _, configpath = self._sanitize_astgen_args(
inpath=inpath,
outfile=outfile,
root=None,
configpath=configpath,
language=self.language)
# convert the config to binary
configpb = AstLookupConfig()
configpath_bin = configpath + '.bin'
# create binary config from text format
self._pb_text_to_bin(proto=configpb,
infile=configpath,
outfile=configpath_bin)
# perform static taint analysis
taint_cmd = [
'node', 'jsprime_wrapper.js', pkg_name, analyze_path,
configpath_bin, outfile
]
exec_command("javascript taint", taint_cmd, cwd="static_proxy/jsprime")
pkg_static = ModuleStatic()
read_proto_from_file(pkg_static, outfile, binary=True)
logging.warning("taint analysis results: %s", pkg_static)
# save resultpb
write_proto_to_file(pkg_static, filename=outfile, binary=False)
# clean up residues
os.remove(configpath_bin)
self._cleanup_astgen(analyze_path=analyze_path,
is_decompress_path=is_decompress_path)
reachable_old_sink = sink.reachable_sinks.add()
reachable_old_sink.CopyFrom(get_mock_sink_node())
reachable_old_sink.sink_type = ast_pb.SINK_NETWORK
# load the astgen config from file
config = AstLookupConfig()
read_proto_from_file(config,
'../../../config/astgen_python_smt.config',
binary=False)
print("loaded config with %d apis to check!" % len(config.apis))
# initialize result and summary
result = ModuleResult()
summary = ModuleSummary()
static = ModuleStatic()
# compute and fill the results into protobuf
set_result(result)
set_summary(summary)
static.flows.MergeFrom(result.flows)
static.dangers.MergeFrom(result.dangers)
static.sources.MergeFrom(summary.sources)
static.sinks.MergeFrom(summary.sinks)
module_result_pb_fname = "module_result_py3.pb"
module_result_txt_fname = "module_result_py3.txt"
module_summary_pb_fname = "module_summary_py3.pb"
module_summary_txt_fname = "module_summary_py3.txt"
module_static_pb_fname = "module_static_py3.pb"
module_static_txt_fname = "module_static_py3.txt"
def danger(self, pkg_name, outdir, cache_dir=None, configpath=None, pkg_version=None):
pkg_static = ModuleStatic()
pass
def reformat(apis_file, json_result_file, outfile):
try:
results = json.load(open(json_result_file, 'r'))
except Exception as e:
logging.error("failed to load pyt results in json: %s",
json_result_file)
return None
# load the astgen config from file
config = AstLookupConfig()
read_proto_from_file(config, apis_file, binary=False)
logging.warning("loaded config with %d apis to check!", len(config.apis))
# convert list of apis into dictionary with key=id, value=full_name for easier identification
source_dict = {}
sink_dict = {}
for entry in config.apis:
# FIXME: should we support func_only mode
if entry.functionality == ast_pb2.SOURCE:
source_dict[entry.id] = entry.full_name
elif entry.functionality in (ast_pb2.SINK, ast_pb2.DANGER):
sink_dict[entry.id] = entry.full_name
nodes = []
# dictionary with key=name of file within package found to contain vulnerabilities and value=tuple of (tree, asttok, visitor) for that file
vuln_files_ASTs = {}
for entry in results['vulnerabilities']:
source = entry['source']
# source['label'], source['line_number'], source['path']
source_trigger_word = entry['source_trigger_word']
sink = entry['sink']
# sink['label'], sink['line_number'], sink['path']
sink_trigger_word = entry['sink_trigger_word']
api_type = entry['type']
reassignment_nodes = entry['reassignment_nodes']
# of type dict
vuln_files_ASTs[source['path']] = ()
vuln_files_ASTs[sink['path']] = ()
nodes.append(
Vulnerability(source, source_trigger_word, sink, sink_trigger_word,
api_type, reassignment_nodes))
# initiate AST visitors (one tree per vulnerable file within package)
for file in vuln_files_ASTs:
src_ast = open(file, 'r').read()
tree = ast.parse(src_ast, filename=file)
asttok = asttokens.ASTTokens(source_text=src_ast,
tree=tree,
filename=file)
# visitor = PythonVisitor(asttok=asttok)
visit_info = (tree, asttok)
vuln_files_ASTs[file] = visit_info
# initialize result and summary
result = ModuleResult()
set_result(result, config.apis, source_dict, sink_dict, nodes,
vuln_files_ASTs)
summary = ModuleSummary()
set_summary(summary, config.apis, source_dict, sink_dict, nodes,
vuln_files_ASTs)
static = ModuleStatic()
static.flows.MergeFrom(result.flows)
static.dangers.MergeFrom(result.dangers)
static.sources.MergeFrom(summary.sources)
static.sinks.MergeFrom(summary.sinks)
static.taint_wrappers.MergeFrom(summary.taint_wrappers)
write_proto_to_file(proto=static, filename=outfile, binary=False)