def test_mutation_and_query(self): """Runs mutation and verifies queries see the results.""" txn = self.client.txn() _ = txn.mutate(pydgraph.Mutation(commit_now=True), set_nquads=""" <_:alice> <name> \"Alice\" . <_:greg> <name> \"Greg\" . <_:alice> <follows> <_:greg> . """) query = """query me($a: string) { me(func: anyofterms(name, "Alice")) { name follows { name } } }""" response = self.client.txn().query(query, variables={'$a': 'Alice'}) self.assertEqual([{'name': 'Alice', 'follows': [{'name': 'Greg'}]}], json.loads(response.json).get('me')) self.assertTrue(is_number(response.latency.parsing_ns), 'Parsing latency is not available') self.assertTrue(is_number(response.latency.processing_ns), 'Processing latency is not available') self.assertTrue(is_number(response.latency.encoding_ns), 'Encoding latency is not available')
def set_json(client, val): """ _=set_json(client, feed_json) :param client: :param val: :return: """ txn = client.txn() return txn.mutate(pydgraph.Mutation(commit_now=True), set_obj=val)
def load(client, src): filt = [ Filter('type', '=', 'attack-pattern'), ] results = src.query(filt) # print(len(results)) # 81 # print(results[0]) data = [] for attackItem in results: tmpResult = { "name": "", "created_date": rfc3339.rfc3339(attackItem["created"]), "last_modified_date": rfc3339.rfc3339(attackItem["modified"]), "name_full": attackItem["name"], "reference": [], "kill_chain_phase": [], "dgraph.type": "AttackTechnique" } if "description" in attackItem.keys(): tmpResult["description"] = [attackItem["description"]] if "x_mitre_platforms" in attackItem.keys(): tmpResult["platform"] = attackItem["x_mitre_platforms"] if "kill_chain_phases" in attackItem.keys(): for kill_chain_phase in attackItem["kill_chain_phases"]: tmpResult["kill_chain_phase"].append(kill_chain_phase["phase_name"]) for external_reference in attackItem["external_references"]: if external_reference["source_name"] == "mitre-ics-attack": tmpResult["name"] = external_reference["external_id"] tmpResult["url"] = external_reference["url"] else: # no link with capec tmpExternalReference = { "description": external_reference["description"], "refsource": external_reference["source_name"], "dgraph.type": "Reference" } if "url" in external_reference.keys(): tmpExternalReference["url"] = external_reference["url"] tmpResult["reference"].append(tmpExternalReference) data.append(tmpResult) # print(data) txn = client.txn() try: mu = pydgraph.Mutation(set_json=json.dumps(data[:]).encode('utf8')) txn.mutate(mu) txn.commit() except pydgraph.AbortedError: print("error") finally: txn.discard() print("ics att&ck technique data without relations loaded")
def test_mutation_and_query(self): """Runs mutation and verifies queries see the results.""" txn = self.client.txn() _ = txn.mutate(pydgraph.Mutation(commit_now=True), set_nquads=""" <_:alice> <name> \"Alice\" . <_:greg> <name> \"Greg\" . <_:alice> <follows> <_:greg> . """) query = """query me($a: string) { me(func: anyofterms(name, "Alice")) { name follows { name } } }""" queryRDF = """query q($a: string) { q(func: anyofterms(name, "Alice")) { uid name } }""" response = self.client.txn().query(query, variables={'$a': 'Alice'}) self.assertEqual([{ 'name': 'Alice', 'follows': [{ 'name': 'Greg' }] }], json.loads(response.json).get('me')) self.assertTrue(is_number(response.latency.parsing_ns), 'Parsing latency is not available') self.assertTrue(is_number(response.latency.processing_ns), 'Processing latency is not available') self.assertTrue(is_number(response.latency.encoding_ns), 'Encoding latency is not available') """ Run query with JSON and RDF resp_format and verify the result """ response = self.client.txn().query(queryRDF, variables={'$a': 'Alice'}) uid = json.loads(response.json).get('q')[0]['uid'] expected_rdf = '<{}> <name> \"Alice\" .\n'.format(uid) response = self.client.txn().query(queryRDF, variables={'$a': 'Alice'}, resp_format="RDF") self.assertEqual(expected_rdf, response.rdf.decode('utf-8'))
def set_nquads(client, nquads): ''' set_nquads(client, """ _:person1 <name> "Daniel" (वंश="स्पेनी", ancestry="Español") . _:person2 <name> "Raj" (वंश="हिंदी", ancestry="हिंदी") . _:person3 <name> "Zhang Wei" (वंश="चीनी", ancestry="中文") . """) :param client: :param nquads: :return: ''' txn = client.txn() return txn.mutate(pydgraph.Mutation(commit_now=True), set_nquads=nquads)
def testJsonLoad(client): testCve = { "name": "CVE-test-test", "cwe": [], "reference": [{ "url": "http://houwenda.github.io", "name": "houwenda", "refsource": "houwenda", "tag": [], "dgraph.type": "Reference" }], "description": ["test description", "test description2"], "impact": { "severity": "MEDIUM", "exploitabilityScore": 10.0, "impactScore": 2.9, "obtainAllPrivilege": False, "obtainUserPrivilege": False, "obtainOtherPrivilege": False, "userInteractionRequired": False, "dgraph.type": "Impact" }, "publishedDate": "1999-12-30T05:00Z", "lastModifiedDate": "2010-12-16T05:00Z", "dgraph.type": "Cve" } print(json.dumps(testCve).encode('utf8')) txn = client.txn() try: mu = pydgraph.Mutation(set_json=json.dumps(testCve).encode('utf8')) txn.mutate(mu) txn.commit() except pydgraph.AbortedError: print("error") finally: txn.discard() print("test json loaded")
def test_mutation_and_query(self): """Runs mutation and queries asyncronously.""" alter_future = self.client.async_alter( pydgraph.Operation(schema="name: string @index(term) .")) response = pydgraph.DgraphClient.handle_alter_future(alter_future) txn = self.client.txn() mutate_future = txn.async_mutate(pydgraph.Mutation(commit_now=True), set_nquads=""" <_:alice> <name> \"Alice\" . <_:greg> <name> \"Greg\" . <_:alice> <follows> <_:greg> . """) _ = pydgraph.Txn.handle_mutate_future(txn, mutate_future, True) query = """query me($a: string) { me(func: anyofterms(name, "Alice")) { name follows { name } } }""" txn = self.client.txn() query_future = txn.async_query(query, variables={'$a': 'Alice'}) response = pydgraph.Txn.handle_query_future(query_future) self.assertEqual([{ 'name': 'Alice', 'follows': [{ 'name': 'Greg' }] }], json.loads(response.json).get('me'))
def load(client, src): filt = [ Filter('type', '=', 'attack-pattern'), Filter('external_references.source_name', '=', 'capec'), ] results = src.query(filt) data = [] for capecItem in results: tmpResult = { "name": "", "created_date": rfc3339.rfc3339(capecItem["created"]), "last_modified_date": rfc3339.rfc3339(capecItem["modified"]), "name_full": capecItem["name"], "description": [capecItem["description"]], "reference": [], "prerequisites": [], "resources_required": [], "dgraph.type": "Capec" } if "x_capec_likelihood_of_attack" in capecItem.keys(): tmpResult["likelihood"] = capecItem["x_capec_likelihood_of_attack"] if "x_capec_prerequisites" in capecItem.keys(): tmpResult["prerequisites"] = capecItem["x_capec_prerequisites"] if "x_capec_resources_required" in capecItem.keys(): tmpResult["resources_required"] = capecItem["x_capec_resources_required"] if "x_capec_typical_severity" in capecItem.keys(): tmpResult["severity"] = capecItem["x_capec_typical_severity"] for external_reference in capecItem["external_references"]: if external_reference["source_name"] == "capec": tmpResult["name"] = external_reference["external_id"] elif external_reference["source_name"] == "cwe": pass else: tmpExternalReference = { "description": [external_reference["description"]], "refsource": external_reference["source_name"], "dgraph.type": "Reference" } if "url" in external_reference.keys(): tmpExternalReference["url"] = external_reference["url"] tmpResult["reference"].append(tmpExternalReference) data.append(tmpResult) print("Total capec:", len(data)) totalCount = len(data) index = 0 span = 100 # 100 records per transaction while index + span < totalCount : txn = client.txn() try: mu = pydgraph.Mutation(set_json=json.dumps(data[index:index+span]).encode('utf8')) txn.mutate(mu) txn.commit() index += span except pydgraph.AbortedError: print("error") break finally: txn.discard() print(str(index+span)+".", end="") txn = client.txn() try: mu = pydgraph.Mutation(set_json=json.dumps(data[index:]).encode('utf8')) txn.mutate(mu) txn.commit() index += span except pydgraph.AbortedError: print("error") finally: txn.discard() print("capec data without relations loaded")
def load(client, src): filt = [ Filter('type', '=', 'attack-pattern'), ] results = src.query(filt) data = [] for attackItem in results: tmpResult = { "name": "", "created_date": rfc3339.rfc3339(attackItem["created"]), "last_modified_date": rfc3339.rfc3339(attackItem["modified"]), "name_full": attackItem["name"], "reference": [], "kill_chain_phase": [], "dgraph.type": "AttackTechnique" } if "description" in attackItem.keys(): tmpResult["description"] = [attackItem["description"]] if "x_mitre_platforms" in attackItem.keys(): tmpResult["platform"] = attackItem["x_mitre_platforms"] if "kill_chain_phases" in attackItem.keys(): for kill_chain_phase in attackItem["kill_chain_phases"]: tmpResult["kill_chain_phase"].append(kill_chain_phase["phase_name"]) for external_reference in attackItem["external_references"]: if external_reference["source_name"] == "mitre-attack": tmpResult["name"] = external_reference["external_id"] tmpResult["url"] = external_reference["url"] elif external_reference["source_name"] == "capec": # print(external_reference["external_id"], end=" ") pass else: tmpExternalReference = { "description": external_reference["description"], "refsource": external_reference["source_name"], "dgraph.type": "Reference" } if "url" in external_reference.keys(): tmpExternalReference["url"] = external_reference["url"] tmpResult["reference"].append(tmpExternalReference) data.append(tmpResult) print("Total attach techniques:", len(results)) # 670 totalCount = len(data) index = 0 span = 100 # 100 records per transaction while index + span < totalCount : txn = client.txn() try: mu = pydgraph.Mutation(set_json=json.dumps(data[index:index+span]).encode('utf8')) txn.mutate(mu) txn.commit() index += span except pydgraph.AbortedError: print("error") break finally: txn.discard() print(str(index+span)+".", end="") txn = client.txn() try: mu = pydgraph.Mutation(set_json=json.dumps(data[index:]).encode('utf8')) txn.mutate(mu) txn.commit() index += span except pydgraph.AbortedError: print("error") finally: txn.discard() print("enterprise att&ck technique data without relations loaded")
def set_json(client, val): txn = client.txn() return txn.mutate(pydgraph.Mutation(commit_now=True), set_obj=val)
def loadFromJsonFile(client, fileName): with open(fileName) as f: dataWithoutRelation = json.load(f) # json validation # remove relation information for i in range(len(dataWithoutRelation)): dataWithoutRelation[i]["cve"] = [] totalCount = len(dataWithoutRelation) index = 0 span = 100 # 100 records per transaction while index + span < totalCount : txn = client.txn() try: mu = pydgraph.Mutation(set_json=json.dumps(dataWithoutRelation[index:index+span]).encode('utf8')) txn.mutate(mu) txn.commit() index += span except pydgraph.AbortedError: print("error") break finally: txn.discard() print(str(index+span)+".", end="") txn = client.txn() try: mu = pydgraph.Mutation(set_json=json.dumps(dataWithoutRelation[index:]).encode('utf8')) txn.mutate(mu) txn.commit() index += span except pydgraph.AbortedError: print("error") finally: txn.discard() print() print("json data without relation loaded into dgraph") # create relations querySuricataRule = '''query all($sid: string) { q(func: eq(sid, $sid)) { uid sid } } ''' queryCve = '''query all($name: string) { q(func: eq(name, $name)) { uid name } } ''' with open(fileName) as f: data = json.load(f) # json validation for suricataRule in data: txn = client.txn() res = txn.query(querySuricataRule, variables={'$sid':suricataRule["sid"]}) print(json.loads(res.json)) currentUid = json.loads(res.json)["q"][0]["uid"] if len(suricataRule["cve"]) > 0: for name in suricataRule["cve"]: res = txn.query(queryCve, variables={'$name':name}) res = json.loads(res.json)["q"] if len(res) == 0: print("cve not found:", name) continue uid = res[0]["uid"] txn.mutate(set_nquads='<' + currentUid + '> <cve> <' + uid + '> .') txn.commit() print("suricata rule to cve relations created")
def loadFromJsonFile(client, fileName): with open(fileName) as f: data = json.load(f) # json validation with open(fileName) as f: dataWithoutRelation = json.load(f) # remove relation information for i in range(len(dataWithoutRelation)): dataWithoutRelation[i]["child_of"] = [] dataWithoutRelation[i]["peer_of"] = [] dataWithoutRelation[i]["can_precede"] = [] dataWithoutRelation[i]["can_also_be"] = [] dataWithoutRelation[i]["capec"] = [] totalCount = len(dataWithoutRelation) index = 0 span = 100 # 100 records per transaction while index + span < totalCount: txn = client.txn() try: mu = pydgraph.Mutation( set_json=json.dumps(dataWithoutRelation[index:index + span]).encode('utf8')) txn.mutate(mu) txn.commit() index += span except pydgraph.AbortedError: print("error") break finally: txn.discard() print(str(index + span) + ".", end="") txn = client.txn() try: mu = pydgraph.Mutation( set_json=json.dumps(dataWithoutRelation[index:]).encode('utf8')) txn.mutate(mu) txn.commit() index += span except pydgraph.AbortedError: print("error") finally: txn.discard() print() print("json data without relations loaded into dgraph") # create relations query = '''query all($name: string) { q(func: eq(name, $name)) { uid name } } ''' for cwe in data: txn = client.txn() res = txn.query(query, variables={'$name': cwe["name"]}) print(json.loads(res.json)) currentUid = json.loads(res.json)["q"][0]["uid"] if len(cwe["child_of"]) > 0: for name in cwe["child_of"]: res = txn.query(query, variables={'$name': name}) uid = json.loads(res.json)["q"][0]["uid"] txn.mutate(set_nquads='<' + currentUid + '> <child_of> <' + uid + '> .') print("child_of") if len(cwe["peer_of"]) > 0: for name in cwe["peer_of"]: res = txn.query(query, variables={'$name': name}) uid = json.loads(res.json)["q"][0]["uid"] txn.mutate(set_nquads='<' + currentUid + '> <peer_of> <' + uid + '> .') print("peer_of") if len(cwe["can_precede"]) > 0: for name in cwe["can_precede"]: res = txn.query(query, variables={'$name': name}) uid = json.loads(res.json)["q"][0]["uid"] txn.mutate(set_nquads='<' + currentUid + '> <can_precede> <' + uid + '> .') print("can_precede") if len(cwe["can_also_be"]) > 0: for name in cwe["can_also_be"]: res = txn.query(query, variables={'$name': name}) uid = json.loads(res.json)["q"][0]["uid"] txn.mutate(set_nquads='<' + currentUid + '> <can_also_be> <' + uid + '> .') print("can_also_be") # cannot create link to capec now # if len(cwe["capec"]) > 0: # for name in cwe["capec"]: # res = txn.query(query, variables={'$name':name}) # uid = json.loads(res.json)["q"][0]["uid"] # txn.mutate(set_nquads='<'+currentUid + '> <capec> <' + uid + '> .') # print("capec") txn.commit() print("cwe to cwe relations created")