def store_binary_in_databse(binary_path: str, package: str, afl_dir_path,
configuration_dir: str):
p = pathlib.Path(afl_dir_path)
p = p.relative_to(*p.parts[:2])
package, created = Package.objects.get_or_create(
name=package, version=ArchCrawler.get_package_version(package))
binary, created = Binary.objects.get_or_create(path=binary_path,
package=package)
binary.afl_dir = os.path.abspath(os.path.join(configuration_dir, str(p)))
binary.save()
def query_packages(packages: List):
result_list = []
for p in packages:
pacman_query = "name={0}&repo=Core&repo=Extra&repo=Community".format(p)
ac = ArchCrawler(query=pacman_query)
package_result = list(ac)
if not package_result:
print("No results for package {0}".format(p))
result_list += list(ac)
return result_list
def main(package):
df = pd.DataFrame(columns=[
"package", "version", "update_date", "depends", "makedepends",
"checkdepends", "opt_depends"
])
pacman_query = "repo=Core&repo=Extra&repo=Community"
ac = ArchCrawler(query=pacman_query)
result_list = list(ac)
for i, package in enumerate(result_list):
df.loc[i] = [
package["pkgname"], package["pkgver"], package["last_update"],
" ".join(package["depends"]), " ".join(package["makedepends"]),
" ".join(package["checkdepends"]), " ".join(package["optdepends"])
]
df.to_csv("packages.csv")
def get_package_list_from_file(file_path: str):
if not os.path.exists(file_path):
raise FileNotFoundError("'{}' could not be found.".format(
os.path.abspath(file_path)))
else:
with open(file_path, "r") as file_pointer:
result_list = []
for line in file_pointer.readlines():
if not line.strip():
continue
pacman_query = "name={0}&repo=Core&repo=Extra&repo=Community".format(
line.strip())
print(pacman_query)
ac = ArchCrawler(query=pacman_query)
result_list += list(ac)
return result_list
def main(pacman_fuzzer_image: str, query: str, configurations_dir: str,
max_build_treshold: int):
pacman_query = "q={0}&repo=Core&repo=Extra&repo=Community".format(query)
ac = ArchCrawler(query=pacman_query)
result_list = list(ac)
# try:
# dr = docker.run("--rm","--cap-add=SYS_PTRACE", "-v", os.getcwd() + "/"+configurations_dir+":/results","--entrypoint","/bin/bash",pacman_fuzzer_image,"-c","pacman -Ssq {0}".format(query),_ok_code=[0,1]) # type: sh.RunningCommand
# except sh.ErrorReturnCode as e:
# print("Error for query {0}".format(query))
# print("STDOUT:\n", e.stdout.decode("utf-8"))
# print(str(e.stderr))
# raise e
# if dr.exit_code==1:
# print("No results for query {0}. Skipping".format(query.strip()))
# return
# package_list = list(filter(lambda x[""]: x,package_list))
query_package_list(result_list, pacman_fuzzer_image, configurations_dir,
max_build_treshold)
def install_opt_depends_for_pacman(self):
from repo_crawlers.archcrawler import ArchCrawler
ac = ArchCrawler(query="name={0}".format(self.package))
packagedict = list(ac)[0]
opt_dependencies = packagedict.get("optdepends")
opt_dependencies = [
dep.split(":")[0] for dep in opt_dependencies
if len(dep.split(":")) >= 1
] # Dependencies often have format package: description
for dep in opt_dependencies:
print("Trying to install dependency {0} via pacman".format(dep))
try:
install_command = pacman("pacman")("-Sy", "--needed",
"--noconfirm",
dep) # _out=sys.stdout)
self.qemu = True
except sh.ErrorReturnCode as e:
print("Could not install dependency {0}".format(dep))
print("Successfully installed package {0}".format(dep))
def get_package_list_from_query(query: str):
pacman_query = "q={0}&repo=Core&repo=Extra&repo=Community".format(query)
ac = ArchCrawler(query=pacman_query)
result_list = list(ac)
return result_list
configurations_dir=arguments.configuration_dir,
max_build_treshold=max_build_treshold)
elif arguments.packagesfile:
if not os.path.exists(arguments.packagesfile):
print("File with packages must exist!")
exit(0)
else:
with open(arguments.packagesfile, "r") as filepointer:
result_list = []
for line in filepointer.readlines():
if not line.strip():
continue
pacman_query = "name={0}&repo=Core&repo=Extra&repo=Community".format(
line.strip())
print(pacman_query)
ac = ArchCrawler(query=pacman_query)
result_list += list(ac)
query_package_list(
result_list,
arguments.docker_image,
configurations_dir=arguments.configuration_dir,
max_build_treshold=max_build_treshold)
elif arguments.package:
pacman_query = "name={0}&repo=Core&repo=Extra&repo=Community".format(
arguments.package)
ac = ArchCrawler(query=pacman_query)
result_list = list(ac)
query_package_list(result_list,
arguments.docker_image,
configurations_dir=arguments.configuration_dir,
max_build_treshold=max_build_treshold,
def store_in_database(basepath: str, package_info_filepath: str,
binary_path: str, parameter: str, crash_dir_path: str,
crash_db_path: str, afl_dir_path: str,
configuration_dir: str):
print("Package info for package {0}".format(package_info_filepath))
package_info_dict = {}
with open(basepath + "/" +
package_info_filepath) as package_info_filepointer:
text = package_info_filepointer.read()
tmp_list = [item.strip().split(":", 1) for item in text.split("\n")]
for item in tmp_list:
if len(item) == 2:
package_info_dict[item[0].strip()] = item[1].strip()
package, created = Package.objects.get_or_create(
name=package_info_dict["Name"],
version=ArchCrawler.get_package_version(package_info_dict["Name"]))
binary, created = Binary.objects.get_or_create(path=binary_path,
package=package)
p = pathlib.Path(afl_dir_path)
p = p.relative_to(*p.parts[:2]) # type: pathlib.PosixPath
binary.afl_dir = os.path.abspath(configuration_dir + "/" +
str(p)) # First path is the volume path
# binary.afl_dir= configuration_dir+"/"+afl_dir_path
binary.save()
crash_db_full_path = basepath + "/" + crash_db_path
if not os.path.exists(crash_db_full_path):
print("Error: The database {0} does not exist".format(
crash_db_full_path))
print(crash_db_path)
return
print("Opening database {0}".format(crash_db_full_path))
connect = sqlite3.connect(basepath + "/" + crash_db_path)
c = connect.cursor()
c.execute(
"select count(*) from sqlite_master where type='table' and name='Data';"
)
if c.fetchone()[0] != 1:
print("Error: The table Data does not exist")
return
c.execute("select count(*) from Data;")
conv = Ansi2HTMLConverter()
if c.fetchone()[0] > 0:
print("Crashes for package {0} binary {1}".format(
package, binary_path))
for row in c.execute('SELECT * FROM Data'):
print(basepath + "/" + crash_dir_path + "/" + row[0])
if not os.path.exists(basepath + "/" + crash_dir_path + "/" +
row[0]):
continue
with open(basepath + "/" + crash_dir_path + "/" + row[0],
"rb") as crash_filepointer:
crash_data = crash_filepointer.read()
try:
rendered_text = conv.convert(row[5].decode("utf-8"))
except Exception as e:
rendered_text = ""
crash, created = Crash.objects.get_or_create(
binary=binary,
parameter=parameter,
exploitability=row[1],
file_blob=crash_data,
description=row[2],
execution_output=rendered_text)
crash.name = row[0]
else:
print("No crashes for package {0} binary {1}".format(
package, binary_path))