def load_data(json_file):
print("loading data...")
with open(json_file, 'rb') as f:
parser = cysimdjson.JSONParser()
data = parser.parse(f.read())
print("processing data...")
return data
def test_parser_resut(self):
parser = cysimdjson.JSONParser()
with open(os.path.join(THIS_DIR, 'document.json'), 'r') as fo:
json_parsed = parser.parse_string(fo.read())
self.assertEqual(
json_parsed.export(), {
'document': {
'key1': 1,
'key2': '2',
'key3': '3',
'key4': 40,
'key5': '50',
}
})
with open(os.path.join(THIS_DIR, 'document.json'), 'r') as fo:
json_parsed1 = parser.parse_string(fo.read())
self.assertEqual(
json_parsed1.export(), {
'document': {
'key1': 1,
'key2': '2',
'key3': '3',
'key4': 40,
'key5': '50',
}
})
def test_contains_01(self):
parser = cysimdjson.JSONParser()
with open(os.path.join(THIS_DIR, 'document.json'), 'rb') as fo:
json_parsed = parser.parse(fo.read())
self.assertEqual("notdocument" in json_parsed, False)
self.assertEqual("document" in json_parsed, True)
def test_scalar_01(self):
parser = cysimdjson.JSONParser()
with open(os.path.join(THIS_DIR, 'scalar_01.json'), 'rb') as fo:
json_parsed = parser.parse(fo.read())
self.assertEqual(json_parsed.get_value(), 1)
def test_iter_02(self):
parser = cysimdjson.JSONParser()
with open(os.path.join(THIS_DIR, 'top_array.json'), 'rb') as fo:
json_parsed = parser.parse(fo.read())
for i, n in enumerate(json_parsed, 1):
self.assertEqual(i, n)
def test_iter_01(self):
parser = cysimdjson.JSONParser()
with open(os.path.join(THIS_DIR, 'array.json'), 'rb') as fo:
json_parsed = parser.parse(fo.read())
ar = json_parsed['array']
self.assertEqual(len(ar), 10)
for i, n in enumerate(ar, 1):
self.assertEqual(i, n)
def perftest_cysimdjson_pad_parse(jsonfile, number):
import cysimdjson
with open(jsonfile, 'rb') as f:
jsonb = f.read() + (b' ' * 1024)
parser = cysimdjson.JSONParser()
return benchmark("cysimdjson pad parse",
lambda: parser.parse_in_place(jsonb),
number=number)
def test_get_01(self):
parser = cysimdjson.JSONParser()
with open(os.path.join(THIS_DIR, 'document.json'), 'r') as fo:
json_parsed = parser.parse_string(fo.read())
v1 = json_parsed.get('document')
self.assertEqual(v1.export(), {
'key1': 1,
'key2': '2',
'key3': '3',
'key4': 40,
'key5': '50',
})
v2 = json_parsed.get('not-present', 'miss')
self.assertEqual(v2, 'miss')
v3 = json_parsed.get('not-present')
self.assertEqual(v3, None)
def test_iter_01(self):
parser = cysimdjson.JSONParser()
with open(os.path.join(THIS_DIR, 'document.json'), 'rb') as fo:
json_parsed = parser.parse(fo.read())
_document = json_parsed['document']
_dict = {}
for key, value in _document.items():
_dict[key] = value
self.assertEqual(
{
"key1": 1,
"key2": "2",
"key3": "3",
"key4": 40,
"key5": "50",
}, _dict)
def test_string_01(self):
parser = cysimdjson.JSONParser()
with open(os.path.join(THIS_DIR, 'document.json'), 'r') as fo:
json_parsed = parser.parse_string(fo.read())
def test_parser_failure_01(self):
parser = cysimdjson.JSONParser()
with self.assertRaises(ValueError) as context:
json_parsed = parser.parse(b"Definitively not a JSON")
def test_loads_01(self):
parser = cysimdjson.JSONParser()
json_parsed = parser.loads('''{"foo":"bar"}''')
self.assertEqual(json_parsed['foo'], 'bar')
def parse_cve_items(exploits):
(exploitdb_names, exploitdb_map, secfocus_names, secfocus_map,
metasploit_names, metasploit_map, l337day_names, l337day_map) = exploits
vulns = []
parser = cysimdjson.JSONParser()
entries = None
for file in sorted(glob.glob('nvd/cve-items-*.json.gz')):
info('Parsing file {bgreen}{file}{rst}...')
with gzip.open(file, 'rb') as f:
entries = parser.parse_in_place(f.read()).at_pointer('/CVE_Items')
for entry in entries:
vuln = {
'id': None,
'date': None,
'description': None,
'availability': None,
'affected': [],
'vendor': [],
'_exploitdb': [],
'_securityfocus': [],
'_metasploit': [],
'_l337day': []
}
vuln['id'] = entry['cve']['CVE_data_meta']['ID'][4:]
vuln['date'] = entry['publishedDate']
vuln['description'] = entry['cve']['description'][
'description_data'][0]['value']
if 'baseMetricV2' in entry['impact']:
vuln['availability'] = entry['impact']['baseMetricV2'][
'cvssV2']['accessComplexity']
# TODO implement proper matching for CPEs with provided operators
for node in entry['configurations']['nodes']:
for child in node['children']:
for cpe in child['cpe_match']:
vuln['affected'].append(cpe['cpe23Uri'])
for reference in entry['cve']['references']['reference_data']:
url = reference['url']
source = reference['refsource']
tags = reference['tags']
if 'Vendor Advisory' in tags:
vuln['vendor'].append(url)
elif source == 'EXPLOIT-DB':
vuln['_exploitdb'].append(url)
elif source == 'BID':
vuln['_securityfocus'].append(url)
# TODO refactor this to use one code path for all lists
if exploitdb_map is not None and vuln['id'] in exploitdb_map:
for expid in exploitdb_map[vuln['id']]:
vuln['_exploitdb'].append(expid)
vuln['_exploitdb'] = set(vuln['_exploitdb'])
vuln['exploitdb'] = []
for exploit in vuln['_exploitdb']:
vuln['exploitdb'].append({
'id':
exploit,
'title':
exploitdb_names[exploit]
if exploit in exploitdb_names else None
})
vuln['_exploitdb'] = None
else:
vuln['exploitdb'] = []
for exploit in vuln['_exploitdb']:
vuln['exploitdb'].append({'id': exploit, 'title': None})
vuln['_exploitdb'] = None
if secfocus_map is not None and vuln['_securityfocus']:
exploits = []
for sfid in vuln['_securityfocus']:
if sfid in secfocus_map:
exploits.append(sfid)
vuln['securityfocus'] = []
for exploit in exploits:
vuln['securityfocus'].append({
'id':
exploit,
'title':
secfocus_names[exploit]
if exploit in secfocus_names else None
})
vuln['_securityfocus'] = None
else:
vuln['securityfocus'] = []
for exploit in vuln['_securityfocus']:
vuln['securityfocus'].append({
'id': exploit,
'title': None
})
vuln['_securityfocus'] = None
if metasploit_map is not None and vuln['id'] in metasploit_map:
for expid in metasploit_map[vuln['id']]:
vuln['_metasploit'].append(expid)
vuln['_metasploit'] = set(vuln['_metasploit'])
vuln['metasploit'] = []
for exploit in vuln['_metasploit']:
vuln['metasploit'].append({
'id':
exploit,
'title':
metasploit_names[exploit]
if exploit in metasploit_names else None
})
vuln['_metasploit'] = None
if l337day_map is not None and vuln['id'] in l337day_map:
for expid in l337day_map[vuln['id']]:
vuln['_l337day'].append(expid)
vuln['_l337day'] = set(vuln['_l337day'])
vuln['l337day'] = []
for exploit in vuln['_l337day']:
vuln['l337day'].append({
'id':
exploit,
'title':
l337day_names[exploit]
if exploit in l337day_names else None
})
vuln['_l337day'] = None
vulns.append(vuln)
info('Extracted {byellow}{vulncount:,}{rst} vulnerabilites.',
vulncount=len(vulns))
return vulns
def test_scalar_02(self):
parser = cysimdjson.JSONParser()
json_loaded = parser.load(os.path.join(THIS_DIR, 'scalar_01.json'))
self.assertEqual(json_loaded, 1)