def test_mutation_and_query(self):
"""Runs mutation and verifies queries see the results."""
txn = self.client.txn()
_ = txn.mutate(pydgraph.Mutation(commit_now=True), set_nquads="""
<_:alice> <name> \"Alice\" .
<_:greg> <name> \"Greg\" .
<_:alice> <follows> <_:greg> .
""")
query = """query me($a: string) {
me(func: anyofterms(name, "Alice"))
{
name
follows
{
name
}
}
}"""
response = self.client.txn().query(query, variables={'$a': 'Alice'})
self.assertEqual([{'name': 'Alice', 'follows': [{'name': 'Greg'}]}],
json.loads(response.json).get('me'))
self.assertTrue(is_number(response.latency.parsing_ns),
'Parsing latency is not available')
self.assertTrue(is_number(response.latency.processing_ns),
'Processing latency is not available')
self.assertTrue(is_number(response.latency.encoding_ns),
'Encoding latency is not available')
def set_json(client, val):
"""
_=set_json(client, feed_json)
:param client:
:param val:
:return:
"""
txn = client.txn()
return txn.mutate(pydgraph.Mutation(commit_now=True), set_obj=val)
def load(client, src):
filt = [
Filter('type', '=', 'attack-pattern'),
]
results = src.query(filt)
# print(len(results)) # 81
# print(results[0])
data = []
for attackItem in results:
tmpResult = {
"name": "",
"created_date": rfc3339.rfc3339(attackItem["created"]),
"last_modified_date": rfc3339.rfc3339(attackItem["modified"]),
"name_full": attackItem["name"],
"reference": [],
"kill_chain_phase": [],
"dgraph.type": "AttackTechnique"
}
if "description" in attackItem.keys():
tmpResult["description"] = [attackItem["description"]]
if "x_mitre_platforms" in attackItem.keys():
tmpResult["platform"] = attackItem["x_mitre_platforms"]
if "kill_chain_phases" in attackItem.keys():
for kill_chain_phase in attackItem["kill_chain_phases"]:
tmpResult["kill_chain_phase"].append(kill_chain_phase["phase_name"])
for external_reference in attackItem["external_references"]:
if external_reference["source_name"] == "mitre-ics-attack":
tmpResult["name"] = external_reference["external_id"]
tmpResult["url"] = external_reference["url"]
else: # no link with capec
tmpExternalReference = {
"description": external_reference["description"],
"refsource": external_reference["source_name"],
"dgraph.type": "Reference"
}
if "url" in external_reference.keys():
tmpExternalReference["url"] = external_reference["url"]
tmpResult["reference"].append(tmpExternalReference)
data.append(tmpResult)
# print(data)
txn = client.txn()
try:
mu = pydgraph.Mutation(set_json=json.dumps(data[:]).encode('utf8'))
txn.mutate(mu)
txn.commit()
except pydgraph.AbortedError:
print("error")
finally:
txn.discard()
print("ics att&ck technique data without relations loaded")
def test_mutation_and_query(self):
"""Runs mutation and verifies queries see the results."""
txn = self.client.txn()
_ = txn.mutate(pydgraph.Mutation(commit_now=True),
set_nquads="""
<_:alice> <name> \"Alice\" .
<_:greg> <name> \"Greg\" .
<_:alice> <follows> <_:greg> .
""")
query = """query me($a: string) {
me(func: anyofterms(name, "Alice"))
{
name
follows
{
name
}
}
}"""
queryRDF = """query q($a: string) {
q(func: anyofterms(name, "Alice"))
{
uid
name
}
}"""
response = self.client.txn().query(query, variables={'$a': 'Alice'})
self.assertEqual([{
'name': 'Alice',
'follows': [{
'name': 'Greg'
}]
}],
json.loads(response.json).get('me'))
self.assertTrue(is_number(response.latency.parsing_ns),
'Parsing latency is not available')
self.assertTrue(is_number(response.latency.processing_ns),
'Processing latency is not available')
self.assertTrue(is_number(response.latency.encoding_ns),
'Encoding latency is not available')
""" Run query with JSON and RDF resp_format and verify the result """
response = self.client.txn().query(queryRDF, variables={'$a': 'Alice'})
uid = json.loads(response.json).get('q')[0]['uid']
expected_rdf = '<{}> <name> \"Alice\" .\n'.format(uid)
response = self.client.txn().query(queryRDF,
variables={'$a': 'Alice'},
resp_format="RDF")
self.assertEqual(expected_rdf, response.rdf.decode('utf-8'))
def set_nquads(client, nquads):
'''
set_nquads(client, """
_:person1 <name> "Daniel" (वंश="स्पेनी", ancestry="Español") .
_:person2 <name> "Raj" (वंश="हिंदी", ancestry="हिंदी") .
_:person3 <name> "Zhang Wei" (वंश="चीनी", ancestry="中文") .
""")
:param client:
:param nquads:
:return:
'''
txn = client.txn()
return txn.mutate(pydgraph.Mutation(commit_now=True), set_nquads=nquads)
def testJsonLoad(client):
testCve = {
"name":
"CVE-test-test",
"cwe": [],
"reference": [{
"url": "http://houwenda.github.io",
"name": "houwenda",
"refsource": "houwenda",
"tag": [],
"dgraph.type": "Reference"
}],
"description": ["test description", "test description2"],
"impact": {
"severity": "MEDIUM",
"exploitabilityScore": 10.0,
"impactScore": 2.9,
"obtainAllPrivilege": False,
"obtainUserPrivilege": False,
"obtainOtherPrivilege": False,
"userInteractionRequired": False,
"dgraph.type": "Impact"
},
"publishedDate":
"1999-12-30T05:00Z",
"lastModifiedDate":
"2010-12-16T05:00Z",
"dgraph.type":
"Cve"
}
print(json.dumps(testCve).encode('utf8'))
txn = client.txn()
try:
mu = pydgraph.Mutation(set_json=json.dumps(testCve).encode('utf8'))
txn.mutate(mu)
txn.commit()
except pydgraph.AbortedError:
print("error")
finally:
txn.discard()
print("test json loaded")
def test_mutation_and_query(self):
"""Runs mutation and queries asyncronously."""
alter_future = self.client.async_alter(
pydgraph.Operation(schema="name: string @index(term) ."))
response = pydgraph.DgraphClient.handle_alter_future(alter_future)
txn = self.client.txn()
mutate_future = txn.async_mutate(pydgraph.Mutation(commit_now=True),
set_nquads="""
<_:alice> <name> \"Alice\" .
<_:greg> <name> \"Greg\" .
<_:alice> <follows> <_:greg> .
""")
_ = pydgraph.Txn.handle_mutate_future(txn, mutate_future, True)
query = """query me($a: string) {
me(func: anyofterms(name, "Alice"))
{
name
follows
{
name
}
}
}"""
txn = self.client.txn()
query_future = txn.async_query(query, variables={'$a': 'Alice'})
response = pydgraph.Txn.handle_query_future(query_future)
self.assertEqual([{
'name': 'Alice',
'follows': [{
'name': 'Greg'
}]
}],
json.loads(response.json).get('me'))
def load(client, src):
filt = [
Filter('type', '=', 'attack-pattern'),
Filter('external_references.source_name', '=', 'capec'),
]
results = src.query(filt)
data = []
for capecItem in results:
tmpResult = {
"name": "",
"created_date": rfc3339.rfc3339(capecItem["created"]),
"last_modified_date": rfc3339.rfc3339(capecItem["modified"]),
"name_full": capecItem["name"],
"description": [capecItem["description"]],
"reference": [],
"prerequisites": [],
"resources_required": [],
"dgraph.type": "Capec"
}
if "x_capec_likelihood_of_attack" in capecItem.keys():
tmpResult["likelihood"] = capecItem["x_capec_likelihood_of_attack"]
if "x_capec_prerequisites" in capecItem.keys():
tmpResult["prerequisites"] = capecItem["x_capec_prerequisites"]
if "x_capec_resources_required" in capecItem.keys():
tmpResult["resources_required"] = capecItem["x_capec_resources_required"]
if "x_capec_typical_severity" in capecItem.keys():
tmpResult["severity"] = capecItem["x_capec_typical_severity"]
for external_reference in capecItem["external_references"]:
if external_reference["source_name"] == "capec":
tmpResult["name"] = external_reference["external_id"]
elif external_reference["source_name"] == "cwe":
pass
else:
tmpExternalReference = {
"description": [external_reference["description"]],
"refsource": external_reference["source_name"],
"dgraph.type": "Reference"
}
if "url" in external_reference.keys():
tmpExternalReference["url"] = external_reference["url"]
tmpResult["reference"].append(tmpExternalReference)
data.append(tmpResult)
print("Total capec:", len(data))
totalCount = len(data)
index = 0
span = 100 # 100 records per transaction
while index + span < totalCount :
txn = client.txn()
try:
mu = pydgraph.Mutation(set_json=json.dumps(data[index:index+span]).encode('utf8'))
txn.mutate(mu)
txn.commit()
index += span
except pydgraph.AbortedError:
print("error")
break
finally:
txn.discard()
print(str(index+span)+".", end="")
txn = client.txn()
try:
mu = pydgraph.Mutation(set_json=json.dumps(data[index:]).encode('utf8'))
txn.mutate(mu)
txn.commit()
index += span
except pydgraph.AbortedError:
print("error")
finally:
txn.discard()
print("capec data without relations loaded")
def load(client, src):
filt = [
Filter('type', '=', 'attack-pattern'),
]
results = src.query(filt)
data = []
for attackItem in results:
tmpResult = {
"name": "",
"created_date": rfc3339.rfc3339(attackItem["created"]),
"last_modified_date": rfc3339.rfc3339(attackItem["modified"]),
"name_full": attackItem["name"],
"reference": [],
"kill_chain_phase": [],
"dgraph.type": "AttackTechnique"
}
if "description" in attackItem.keys():
tmpResult["description"] = [attackItem["description"]]
if "x_mitre_platforms" in attackItem.keys():
tmpResult["platform"] = attackItem["x_mitre_platforms"]
if "kill_chain_phases" in attackItem.keys():
for kill_chain_phase in attackItem["kill_chain_phases"]:
tmpResult["kill_chain_phase"].append(kill_chain_phase["phase_name"])
for external_reference in attackItem["external_references"]:
if external_reference["source_name"] == "mitre-attack":
tmpResult["name"] = external_reference["external_id"]
tmpResult["url"] = external_reference["url"]
elif external_reference["source_name"] == "capec":
# print(external_reference["external_id"], end=" ")
pass
else:
tmpExternalReference = {
"description": external_reference["description"],
"refsource": external_reference["source_name"],
"dgraph.type": "Reference"
}
if "url" in external_reference.keys():
tmpExternalReference["url"] = external_reference["url"]
tmpResult["reference"].append(tmpExternalReference)
data.append(tmpResult)
print("Total attach techniques:", len(results)) # 670
totalCount = len(data)
index = 0
span = 100 # 100 records per transaction
while index + span < totalCount :
txn = client.txn()
try:
mu = pydgraph.Mutation(set_json=json.dumps(data[index:index+span]).encode('utf8'))
txn.mutate(mu)
txn.commit()
index += span
except pydgraph.AbortedError:
print("error")
break
finally:
txn.discard()
print(str(index+span)+".", end="")
txn = client.txn()
try:
mu = pydgraph.Mutation(set_json=json.dumps(data[index:]).encode('utf8'))
txn.mutate(mu)
txn.commit()
index += span
except pydgraph.AbortedError:
print("error")
finally:
txn.discard()
print("enterprise att&ck technique data without relations loaded")
def set_json(client, val):
txn = client.txn()
return txn.mutate(pydgraph.Mutation(commit_now=True), set_obj=val)
def loadFromJsonFile(client, fileName):
with open(fileName) as f:
dataWithoutRelation = json.load(f) # json validation
# remove relation information
for i in range(len(dataWithoutRelation)):
dataWithoutRelation[i]["cve"] = []
totalCount = len(dataWithoutRelation)
index = 0
span = 100 # 100 records per transaction
while index + span < totalCount :
txn = client.txn()
try:
mu = pydgraph.Mutation(set_json=json.dumps(dataWithoutRelation[index:index+span]).encode('utf8'))
txn.mutate(mu)
txn.commit()
index += span
except pydgraph.AbortedError:
print("error")
break
finally:
txn.discard()
print(str(index+span)+".", end="")
txn = client.txn()
try:
mu = pydgraph.Mutation(set_json=json.dumps(dataWithoutRelation[index:]).encode('utf8'))
txn.mutate(mu)
txn.commit()
index += span
except pydgraph.AbortedError:
print("error")
finally:
txn.discard()
print()
print("json data without relation loaded into dgraph")
# create relations
querySuricataRule = '''query all($sid: string) {
q(func: eq(sid, $sid)) {
uid
sid
}
}
'''
queryCve = '''query all($name: string) {
q(func: eq(name, $name)) {
uid
name
}
}
'''
with open(fileName) as f:
data = json.load(f) # json validation
for suricataRule in data:
txn = client.txn()
res = txn.query(querySuricataRule, variables={'$sid':suricataRule["sid"]})
print(json.loads(res.json))
currentUid = json.loads(res.json)["q"][0]["uid"]
if len(suricataRule["cve"]) > 0:
for name in suricataRule["cve"]:
res = txn.query(queryCve, variables={'$name':name})
res = json.loads(res.json)["q"]
if len(res) == 0:
print("cve not found:", name)
continue
uid = res[0]["uid"]
txn.mutate(set_nquads='<' + currentUid + '> <cve> <' + uid + '> .')
txn.commit()
print("suricata rule to cve relations created")
def loadFromJsonFile(client, fileName):
with open(fileName) as f:
data = json.load(f) # json validation
with open(fileName) as f:
dataWithoutRelation = json.load(f)
# remove relation information
for i in range(len(dataWithoutRelation)):
dataWithoutRelation[i]["child_of"] = []
dataWithoutRelation[i]["peer_of"] = []
dataWithoutRelation[i]["can_precede"] = []
dataWithoutRelation[i]["can_also_be"] = []
dataWithoutRelation[i]["capec"] = []
totalCount = len(dataWithoutRelation)
index = 0
span = 100 # 100 records per transaction
while index + span < totalCount:
txn = client.txn()
try:
mu = pydgraph.Mutation(
set_json=json.dumps(dataWithoutRelation[index:index +
span]).encode('utf8'))
txn.mutate(mu)
txn.commit()
index += span
except pydgraph.AbortedError:
print("error")
break
finally:
txn.discard()
print(str(index + span) + ".", end="")
txn = client.txn()
try:
mu = pydgraph.Mutation(
set_json=json.dumps(dataWithoutRelation[index:]).encode('utf8'))
txn.mutate(mu)
txn.commit()
index += span
except pydgraph.AbortedError:
print("error")
finally:
txn.discard()
print()
print("json data without relations loaded into dgraph")
# create relations
query = '''query all($name: string) {
q(func: eq(name, $name)) {
uid
name
}
}
'''
for cwe in data:
txn = client.txn()
res = txn.query(query, variables={'$name': cwe["name"]})
print(json.loads(res.json))
currentUid = json.loads(res.json)["q"][0]["uid"]
if len(cwe["child_of"]) > 0:
for name in cwe["child_of"]:
res = txn.query(query, variables={'$name': name})
uid = json.loads(res.json)["q"][0]["uid"]
txn.mutate(set_nquads='<' + currentUid + '> <child_of> <' +
uid + '> .')
print("child_of")
if len(cwe["peer_of"]) > 0:
for name in cwe["peer_of"]:
res = txn.query(query, variables={'$name': name})
uid = json.loads(res.json)["q"][0]["uid"]
txn.mutate(set_nquads='<' + currentUid + '> <peer_of> <' +
uid + '> .')
print("peer_of")
if len(cwe["can_precede"]) > 0:
for name in cwe["can_precede"]:
res = txn.query(query, variables={'$name': name})
uid = json.loads(res.json)["q"][0]["uid"]
txn.mutate(set_nquads='<' + currentUid + '> <can_precede> <' +
uid + '> .')
print("can_precede")
if len(cwe["can_also_be"]) > 0:
for name in cwe["can_also_be"]:
res = txn.query(query, variables={'$name': name})
uid = json.loads(res.json)["q"][0]["uid"]
txn.mutate(set_nquads='<' + currentUid + '> <can_also_be> <' +
uid + '> .')
print("can_also_be")
# cannot create link to capec now
# if len(cwe["capec"]) > 0:
# for name in cwe["capec"]:
# res = txn.query(query, variables={'$name':name})
# uid = json.loads(res.json)["q"][0]["uid"]
# txn.mutate(set_nquads='<'+currentUid + '> <capec> <' + uid + '> .')
# print("capec")
txn.commit()
print("cwe to cwe relations created")
