def __init__(self, input_type, target, compile_commands, os_name,
log_level):
self.input_type = input_type
self.compile_commands = compile_commands
self.os = os_name
self.os_type = None
self.log_level = log_level
if input_type == "ioctl":
self.target = os.path.realpath(target)
self.out_dir = os.path.join(os.getcwd(), "out/preprocessed/",
basename(self.target), "out")
self.macro_details = ""
self.ioctls = []
self.bear = Bear(self)
self.c2xml = C2xml(self)
# initialize the sub classes
self.extractor = Extractor(self)
self.descriptions = Descriptions(self)
self.header_files = self.extractor.header_files
logging.debug("[+] Sys2syz init completed")
if input_type == "syscall":
self.target = target
self.out_dir = os.path.join(os.getcwd(), "out/preprocessed/",
basename(self.target), "out")
self.bear = Bear(self)
self.c2xml = C2xml(self)
self.syscall = Syscall(self)
self.descriptions = Descriptions(self)
if not self._sanity_check():
logging.critical("[+] Sys2syz failed to init")
sys.exit(-1)
def extract():
url = request.json.get("url")
if not url:
raise Error("A value for 'url' is required.", 422)
extractor = Extractor()
extractor.fetch(url)
extractor.parse()
return jsonify({
"city": extractor.city,
"locations": list(extractor.locations)
})
def main():
if len(argv) <= 1:
forhelp()
exit(1)
else:
if len(argv) == 2:
if argv[1] in ['-c', '--clear']:
Connection().drop()
Extractor().cls()
elif argv[1] in ['-p', '--compare']:
com = Compare().result()
else:
forhelp()
exit(1)
elif len(argv) == 4:
target = argv[3]
if argv[1] in ['-f', '--first-shot'] and \
argv[2] in ['-t', '--target']:
ex = Extractor(target, 1)
ex.walk()
elif argv[1] in ['-s', '--second-shot'] and \
argv[2] in ['-t', '--target']:
ex = Extractor(target, 2)
ex.walk()
else:
forhelp()
exit(1)
else:
forhelp()
exit(1)
nlp = NLP()
num = 1 # 知识三元组
with open(input_path, 'r', encoding='utf-8') as f_in:
# 分句，获得句子列表
origin_sentences = re.split('[。？！；]|\n', f_in.read())
# 遍历每一篇文档中的句子
for origin_sentence in origin_sentences:
# 原始句子长度小于6，跳过
if (len(origin_sentence) < 6):
continue
print('*****')
# print(origin_sentence)
# 分词处理
lemmas = nlp.segment(origin_sentence)
# print("lemmas",lemmas)
# 词性标注
words_postag = nlp.postag(lemmas)
# print("words_postag",words_postag)
# 命名实体识别
words_netag = nlp.netag(words_postag)
# 依存句法分析
sentence = nlp.parse(words_netag)
print(sentence.to_string())
extractor = Extractor()
num = extractor.extract(origin_sentence, sentence, output_path,
num)
args = parser.parse_args()
BASE_DIR = get_rootDir(args.verbose)
if __name__ == "__main__":
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print("[I] Inicio - {} ({}).".format(now, sys.argv[0]))
# To test
## begin
args.treat_data = [
BASE_DIR + '/inputData/201805191230.csv', "knndictionary"
]
inputdata = args.treat_data
args.extractor = BASE_DIR + '/configinit/dbmongo_ci.json'
configure = args.extractor
##end
if args.treat_data:
#inputdata = str(args.treat_data[0])
Treats_ci_data(inputdata, args.multiprocess, args.verbose).run()
exit()
elif args.extractor:
configure = str(args.extractor[0])
configure = json.loads(open(configure, 'r').read())
extractdata = Extractor(configure, verbose=args.verbose).run()
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print("[I] Fim - %s (%s)." % (now, sys.argv[0]))
exit(0)
class Sys2syz(object):
FREEBSD = 1
supported_os = {'FreeBSD': FREEBSD}
def __init__(self, input_type, target, compile_commands, os_name,
log_level):
self.input_type = input_type
self.compile_commands = compile_commands
self.os = os_name
self.os_type = None
self.log_level = log_level
if input_type == "ioctl":
self.target = os.path.realpath(target)
self.out_dir = os.path.join(os.getcwd(), "out/preprocessed/",
basename(self.target), "out")
self.macro_details = ""
self.ioctls = []
self.bear = Bear(self)
self.c2xml = C2xml(self)
# initialize the sub classes
self.extractor = Extractor(self)
self.descriptions = Descriptions(self)
self.header_files = self.extractor.header_files
logging.debug("[+] Sys2syz init completed")
if input_type == "syscall":
self.target = target
self.out_dir = os.path.join(os.getcwd(), "out/preprocessed/",
basename(self.target), "out")
self.bear = Bear(self)
self.c2xml = C2xml(self)
self.syscall = Syscall(self)
self.descriptions = Descriptions(self)
if not self._sanity_check():
logging.critical("[+] Sys2syz failed to init")
sys.exit(-1)
def _sanity_check(self) -> bool:
"""Perform Sanity check on the arguments passed
Returns:
bool: Reflect passed or failed
"""
if self.input_type == "ioctl":
if not os.path.isdir(self.target):
logging.error("[+] The target file is not found at %s",
self.target)
return False
logging.debug("[+] The target file is %s", self.target)
if not os.path.isfile(self.compile_commands):
logging.error("[+] The compile commands not found at %s",
self.compile_commands)
return False
logging.debug("[+] The compile commands file is %s",
self.compile_commands)
for os_type in self.supported_os.keys():
if os_type.lower() == self.os.lower():
self.os_type = self.supported_os[os_type]
return True
logging.error("[+] Target OS not supported/found %s", self.os)
return False
def get_ioctls(self) -> bool:
""" Get's the IOCTL calls as a list and does sanity check and some stats
Returns:
bool: True is ioctls were found
"""
self.extractor.get_ioctls()
self.ioctls = self.extractor.ioctls
if len(self.ioctls) == 0:
return False
if self.log_level > 1:
ioctl_string = ""
for ioctl in self.ioctls:
ioctl_string += str(ioctl) + "\n"
open("test.log", "w+").write(ioctl_string)
logging.info(f"[+] {len(self.ioctls)} IOCTL calls were found!")
return True
@property
def undefined_macros(self) -> list:
und_macros = self.extractor.fetch_flags()
logging.info(
f"[+] {len(und_macros)} undefined macros were found from the file!"
)
return und_macros
def get_macro_details(self):
self.macro_details = self.extractor.flag_details(self.undefined_macros)
logging.info(
f"[+] Extracted details of {len(self.macro_details)} macros from c2xml!"
)
def preprocess_files(self, file_name=None) -> bool:
""" Preprocess the files
"""
try:
if self.bear.parse_compile_commands(file_name):
return True
except Exception as e:
logging.critical("Unable to run bear and parse compile commands")
logging.error(e)
return False
def create_xml_files(self):
try:
self.c2xml.run_c2xml()
return True
except Exception as e:
logging.critical("Failed to convert C files to XML")
return False
def generate_descriptions(self):
if self.input_type == "ioctl":
self.descriptions.ioctl_run()
#Store the descriptions in the syzkaller's syscall description file format
output_path = self.descriptions.make_file()
if Utils.file_exists(output_path, True):
logging.info("[+] Description file: " + output_path)
return True
return False
if self.input_type == "syscall":
self.descriptions.syscall_run()
return True
'''except Exception as e: