g = Graph("bolt://127.0.0.1:7687",
username="******",
password="******")
iter = g.begin()
nodes = []
a = Node("文献名", title=t.title)
iter.create(a)
nodes.append(a)
au = t.author.split(', ')
for th in au:
th = th.replace(" \n\r", "")
th.strip(" ")
b = Node("作者", author=th)
nodes.append(b)
ab = Relationship(a, "本文作者", b)
nodes.append(ab)
key = t.keywords.split(', ')
for ke in key:
ke = ke.replace("\r", "")
ke = ke.replace("\n", "")
if ke == "" or ke == " ":
break
if ke[-1] == ' ':
ke = ke[:-1]
ke.strip(' ')
c = Node("关键词", keywords=ke)
nodes.append(c)
ac = Relationship(a, "本文关键词", c)
nodes.append(ac)
while line.__len__() > 0:
want_line = line.split("\t")[-1]
want_line = want_line.replace("(", "").replace(")",
"").replace("\n", "")
line_data = want_line.split(",")
print(line_data)
data.append(line_data)
line = f.readline()
return data
graph = Graph('http://119.23.224.186:7474', username='******', password='******')
data = load_data()
d_dict = {}
for d in data: # type:list
a = Node("test2", name=d[0])
if d[0] in d_dict:
a = d_dict[d[0]]
else:
d_dict[d[0]] = a
graph.create(a)
b = Node("test2", name=d[2], size=10)
if d[1] in d_dict:
b = d_dict[d[1]]
else:
d_dict[d[1]] = b
graph.create(b)
r = Relationship(a, d[1], b)
graph.create(r)
def cti_mitigations_graph(scenario,
graph_uri=default_uri,
graph_user=default_user,
graph_password=default_password,
network=None):
cve_list = scenario['cve']
capec_list = scenario['capec']
technique_list = scenario['technique']
mitigation_list = scenario['mitigation']
host_list = ['localhost']
if network != None:
host_list = list(network.keys())
g = Graph(uri=graph_uri, user=graph_user, password=graph_password)
matcher = NodeMatcher(g)
g.delete_all()
tx = g.begin()
for host in host_list:
host_node = Node("Host", name=host)
tx.create(host_node)
for cve_id in cve_list:
cve_node = Node("CVE",
name=cve_id,
cwe=cve.cwe(cve_id),
base_score=cve.cvss_basescore(cve_id),
exploitability_score=cve.cvss_exscore(cve_id),
impact_score=cve.cvss_impactscore(cve_id),
vector_string=cve.cvss_vector(cve_id))
tx.create(cve_node)
for capec_id in capec_list:
capec_node = Node("CAPEC",
name=('CAPEC-' + str(capec_id)),
cwe=capec.cwe(capec_id),
likelihood=capec.likelihood(capec_id),
severity=capec.severity(capec_id),
prerequisites=capec.prerequisites(capec_id),
skill=capec.skill(capec_id),
consequences=capec.consequences(capec_id),
resources=capec.resources(capec_id))
tx.create(capec_node)
for technique in technique_list:
technique_node = Node(
"TECHNIQUE",
name=attack.attack_id(technique),
extended_name=attack.name(technique),
platforms=attack.technique_platforms(technique),
permissions=attack.permissions(technique),
description=attack.description(technique),
datasource=attack.data_sources(technique),
kill_chain_phase=attack.kill_chain_phases(technique))
tx.create(technique_node)
for mitigation in mitigation_list:
if attack.attack_id(mitigation)[0] == 'M':
mitigation_node = Node(
"MITIGATION",
name=attack.attack_id(mitigation),
extended_name=attack.mitigation_name(mitigation),
description=attack.mitigation_description(mitigation))
tx.create(mitigation_node)
tx.commit()
tx = g.begin()
for host in host_list:
host_node = matcher.match("Host").where("_.name =~ '" + host +
"'").first()
if network == None:
for cve_id in cve_list:
cve_node = matcher.match("CVE").where("_.name =~ '" +
str(cve_id) +
"'").first()
host_cve = Relationship(host_node, "PRESENT", cve_node)
tx.merge(host_cve, "PRESENT", "name")
else:
for cve_id in network[host]['cve_list']:
cve_node = matcher.match("CVE").where("_.name =~ '" +
str(cve_id) +
"'").first()
host_cve = Relationship(host_node, "PRESENT", cve_node)
tx.merge(host_cve, "PRESENT", "name")
for cve_id in cve_list:
cve_node = matcher.match("CVE").where("_.name =~ '" + cve_id +
"'").first()
for capec_id in cve.capec(cve_id):
capec_node = matcher.match("CAPEC").where("_.name =~ '" +
('CAPEC-' +
str(capec_id)) +
"'").first()
cve_capec = Relationship(cve_node, "EXPLOITED", capec_node)
tx.merge(cve_capec, "EXPLOITED", "name")
for capec_id in capec_list:
capec_node = matcher.match("CAPEC").where("_.name =~ '" +
('CAPEC-' + str(capec_id)) +
"'").first()
for technique in capec.attack_technique(capec_id):
if technique in technique_list:
technique_node = matcher.match("TECHNIQUE").where(
"_.name =~ '" + attack.attack_id(technique) + "'").first()
capec_technique = Relationship(capec_node, "IMPLEMENTED",
technique_node)
tx.merge(capec_technique, "IMPLEMENTED", "name")
for technique in technique_list:
technique_node = matcher.match("TECHNIQUE").where(
"_.name =~ '" + attack.attack_id(technique) + "'").first()
for mitigation in attack.relationship_with(technique,
"course-of-action"):
if attack.attack_id(mitigation)[0] == 'M':
mitigation_node = matcher.match("MITIGATED").where(
"_.name =~ '" + attack.attack_id(mitigation) +
"'").first()
technique_mitigation = Relationship(technique_node,
"MITIGATED",
mitigation_node)
tx.merge(technique_mitigation)
tx.commit()
return
def require_ingredients(self, ingredients):
for ingredient in ingredients:
iNode = ingredient.add()
rel = Relationship(self.node, 'REQUIRES', iNode)
graph.create_unique(rel)
def process(self, parameters=None, data=None):
if parameters is None:
parameters = {}
if data is None:
data = {}
if 'verbose' in parameters:
self.config['verbose'] = parameters['verbose']
# for this facets, do not add additional entity to connect with, but write to properties of the entity
properties = ['content_type_ss',
'content_type_group_ss', 'language_ss', 'language_s']
host = 'localhost'
if 'neo4j_host' in parameters:
host = parameters['neo4j_host']
user = '******'
if 'neo4j_user' in parameters:
user = parameters['neo4j_user']
password = '******'
if 'neo4j_password' in parameters:
password = parameters['neo4j_password']
graph = Graph(host=host, user=user, password=password)
document_node = Node('Document', name=parameters['id'])
if 'title' in data:
document_node['title'] = data['title']
# add properties from facets
for entity_class in parameters['facets']:
if entity_class in data:
entity_class_label = parameters['facets'][entity_class]['label']
if entity_class in properties:
document_node[entity_class_label] = data[entity_class]
graph.merge(document_node)
# add / connect linked entities from facets
for entity_class in parameters['facets']:
if entity_class in data:
entity_class_label = entity_class
if parameters['facets'][entity_class]['label']:
entity_class_label = parameters['facets'][entity_class]['label']
if not entity_class in properties:
relationship_label = entity_class_label
if entity_class in ['person_ss', 'organization_ss', 'location_ss']:
relationship_label = "Named Entity Recognition"
# convert to array, if single entity / not multivalued field
if isinstance(data[entity_class], list):
entities = data[entity_class]
else:
entities = [data[entity_class]]
for entity in entities:
if self.config['verbose']:
print("Export to Neo4j: Merging entity {} of class {}".format(
entity, entity_class_label))
# if not yet there, add the entity to graph
entity_node = Node(entity_class_label, name=entity)
graph.merge(entity_node)
# if not yet there, add relationship to graph
relationship = Relationship(
document_node, relationship_label, entity_node)
graph.merge(relationship)
return parameters, data
def create_KG():
g.delete_all()
i = 0
with open(
'G:/Programming files/Python/code/KG/KG-MITRE ATT&CK Matrices/ATT&CK MATRICES Tac.csv',
'r') as f1:
Tactics_info_list = csv.reader(f1)
for tac in Tactics_info_list:
if Tactics_info_list.line_num == 1:
continue
Tactics_attrs = {
'Name': tac[0].replace(' ', ''),
'Intro': tac[1],
'ID': tac[2],
'Created': tac[3],
'Last_Modified': tac[4]
}
Tactics_node = Node("Tactics", **Tactics_attrs)
g.create(Tactics_node)
# 建立Tactics节点之间关系
i = i + 1
if 1 < i < 13:
pre_node = cur_node
cur_node = Tactics_node
relation1 = Relationship(pre_node, 'ATT&CK step', cur_node)
g.create(relation1)
else:
cur_node = Tactics_node
with open(
'G:/Programming files/Python/code/KG/KG-MITRE ATT&CK Matrices/ATT&CK MATRICES Tec.csv',
'r') as f2:
Techniques_info_list = csv.reader(f2)
for tec in Techniques_info_list:
if Techniques_info_list.line_num == 1:
continue
Techniques_attrs = {
'Name': tec[0],
'ID': tec[1],
'Sub-Tec': tec[2],
'Tactic': tec[3],
'Platforms': tec[4],
'Data Sources': tec[5],
'Permissions Required': tec[6]
}
Techniques_node = Node("Techniques", **Techniques_attrs)
g.merge(Techniques_node, "Techniques", "Name")
# 建立Tactics和Techniques节点之间关系
if ',' not in tec[3]:
Tempnode = matcher.match('Tactics').where("_.Name=~'(?i)" +
tec[3] +
"'").first()
if Tempnode != None:
relation2 = Relationship(Techniques_node, 'Accomplishes',
Tempnode)
g.create(relation2)
else:
Temp_tac_list = tec[3].split(',')
for i in range(0, len(Temp_tac_list)):
Tempnode = matcher.match('Tactics').where(
"_.Name=~'(?i)" + Temp_tac_list[i] + "'").first()
if Tempnode != None:
relation2 = Relationship(Techniques_node,
'Accomplishes', Tempnode)
g.create(relation2)
with open(
'G:/Programming files/Python/code/KG/KG-MITRE ATT&CK Matrices/ATT&CK MATRICES Miti.csv',
'r') as f3:
Mitigations_info_list = csv.reader(f3)
for miti in Mitigations_info_list:
if Mitigations_info_list.line_num == 1:
continue
# 格式化miti[3]数据
if miti[3] == '[]':
miti[3] = ['None']
else:
miti[3] = miti[3].replace("'", "").replace("[", "").replace(
"]", "").split(", ")
# print(miti[3])
Mitigations_attrs = {
'Name': miti[0],
'ID': miti[1],
'Description': miti[2],
'Tecs Addressed by Mitigation': miti[3]
}
Mitigations_node = Node("Mitigations", **Mitigations_attrs)
g.create(Mitigations_node)
# 建立Mitigations和Techniques节点之间关系
for addressed_tec in miti[3]:
Tempnode = matcher.match('Techniques').where("_.Name=~'" +
addressed_tec +
"'").first()
if Tempnode != None:
relation3 = Relationship(Mitigations_node, 'Prevents',
Tempnode)
g.create(relation3)
with open(
'G:/Programming files/Python/code/KG/KG-MITRE ATT&CK Matrices/ATT&CK MATRICES Group.csv',
'r') as f4:
Groups_info_list = csv.reader(f4)
for group in Groups_info_list:
if Groups_info_list.line_num == 1:
continue
if group[3] == '':
group[3] = ['None']
if group[2] == '[]':
group[2] = ['None']
else:
group[2] = group[2].replace("'", "").replace("[", "").replace(
"]", "").split(", ")
Groups_attrs = {
'Name': group[0],
'ID': group[1],
'Associated groups': group[3],
'Tecs Used by Group': group[2]
}
Groups_node = Node("Groups", **Groups_attrs)
g.create(Groups_node)
# 建立Groups和Techniques节点之间关系
for Used_tec in group[2]:
Tempnode = matcher.match('Techniques').where("_.ID=~'" +
Used_tec +
"'").first()
if Tempnode != None:
relation4 = Relationship(Groups_node, 'Uses', Tempnode)
g.create(relation4)
def create_relationship(node1, node2, name=None, swap=False):
if swap:
node1, node2 = node2, node1
if name:
return Relationship(node1, name, node2)
def create_permission(db, entity, resource, permissions):
sec = Relationship(entity,
Security.SECURITY,
resource,
permissions=permissions)
db.create(sec)
def add_person_to_group(db, person, group):
sec = Relationship(group, Security.IS_MEMBER_OF, person)
db.create(sec)
def create_group(db, name, owner):
node = Node(Groups.LABEL, name=name)
db.create(node)
rel = Relationship(owner, Relationships.OWNS, node)
db.create(rel)
def add_user(db, group, user):
rel = Relationship(user, Relationships.IS_MEMBER, group)
db.create(rel)
date_tweet = read['date']
datetime = read['datetime']
tid = read['tid']
author_profile_image = read['author_profile_image']
author_screen_name = read['author_screen_name']
author_id = read['author_id']
retweet_source_id = read['retweet_source_id']
reply_source_id = read['replyto_source_id']
user_node = Node("user",author_id = author_id,author = author,author_screen_name = author_screen_name)
tweet_node = Node("tweet",tid = tid,date = date_tweet,datetime= datetime)
tx.merge(user_node, primary_key="author_screen_name")
tx.merge(tweet_node, primary_key="tid")
relation_user_tweet = Relationship(user_node,"tweets",tweet_node)
tx.create(relation_user_tweet)
if location is not None:
location_node = Node("location", location=location)
tx.merge(location_node, primary_key="location")
relation_tweet_location = Relationship(location_node,"hastweet",tweet_node)
tx.create(relation_tweet_location)
if hashtags is not None:
for hashtag in hashtags:
count = count+1
hashtag_node = Node("hashtag",hashtag=hashtag)
tx.merge(hashtag_node,primary_key="hashtag")
relation_tweet_hashtag = Relationship(tweet_node,"contains_hashtag",hashtag_node)
tx.create(relation_tweet_hashtag)
def update_relationship(rels, key):
labels_node1 = rels.get('labels_node1')
if labels_node1 in ('', None):
return None, 201
name_node1 = rels.get('name_node1')
if name_node1 in ('', None):
return None, 201
labels_node2 = rels.get('labels_node2')
if labels_node2 in ('', None):
return None, 201
name_node2 = rels.get('name_node2')
if name_node2 in ('', None):
return None, 201
labels = rels.get('labels')
name = key
if labels in ('', None) or name in ('', None):
return None, 201
full_name = rels.get('full_name')
if full_name == '':
full_name = None
image = rels.get('image')
if image == '':
image == None
object_id = rels.get('object_id')
if object_id == 0:
object_id == None
weight = rels.get('weight')
if weight == 0:
weight == None
keywords = rels.get('keywords')
if keywords == '':
keywords == None
areas = rels.get('areas')
if areas == '':
areas == None
from_date = rels.get('from_date')
if from_date == '':
from_date == None
to_date = rels.get('to_date')
if to_date == '':
to_date == None
node1 = find_node(labels_node1, name_node1)
print(node1)
node2 = find_node(labels_node2, name_node2)
print(node2)
if node1 in ('', None) or node2 in ('', None):
return None, 201
relationship = Relationship(node1,
labels,
node2,
name=name,
full_name=full_name,
image=image,
object_id=object_id,
weight=weight,
keywords=keywords,
areas=areas,
from_date=from_date,
to_date=to_date)
relationship.__primarylabel__ = labels
relationship.__primarykey__ = name
print(relationship)
graph = neo4j_connect()
transaction = graph.begin()
transaction.merge(relationship, primary_key=name, primary_label=labels)
transaction.commit()
transaction.finished()
graph.exists(relationship)
return relationship, 200
def create_relationship(rels):
labels_node1 = rels.get('labels_node1')
if labels_node1 in ('', None):
return None, 201
name_node1 = rels.get('name_node1')
if name_node1 in ('', None):
return None, 201
labels_node2 = rels.get('labels_node2')
if labels_node2 in ('', None):
return None, 201
name_node2 = rels.get('name_node2')
if name_node2 in ('', None):
return None, 201
labels = rels.get('labels')
name = rels.get('name')
if labels in ('', None) or name in ('', None):
return None, 201
full_name = rels.get('full_name')
if full_name == '':
full_name = None
image = rels.get('image')
if image == '':
image == None
object_id = rels.get('object_id')
if object_id == 0:
object_id == None
weight = rels.get('weight')
if weight == 0:
weight == None
keywords = rels.get('keywords')
if keywords == '':
keywords == None
areas = rels.get('areas')
if areas == '':
areas == None
from_date = rels.get('from_date')
if from_date == '':
from_date == None
to_date = rels.get('to_date')
if to_date == '':
to_date == None
node1 = find_node(labels_node1, name_node1)
print(node1)
node2 = find_node(labels_node2, name_node2)
print(node2)
if node1 in ('', None) or node2 in ('', None):
return None, 201
relationship = Relationship(node1,
labels,
node2,
name=name,
full_name=full_name,
image=image,
object_id=object_id,
weight=weight,
keywords=keywords,
areas=areas,
from_date=from_date,
to_date=to_date)
relationship.__primarylabel__ = labels
relationship.__primarykey__ = name
graph = neo4j_connect()
transaction = graph.begin()
transaction.create(relationship)
transaction.commit()
transaction.finished()
graph.exists(relationship)
# constrain = f"""CREATE CONSTRAINT ON ()-[r:{labels}]-() ASSERT exists(r.name)"""
# graph.run(constrain)
return relationship, 200
def test_relationship_equality_with_none(graph):
rel = Relationship({}, "KNOWS", {})
none = None
assert rel != none
from py2neo import Graph, Node, Relationship
graph = Graph()
a = Node("Author", name="Alice")
b = Node("Author", name="Bob")
a_knows_b = Relationship(a, "CO-AUTHORED", b)
a_knows_b = Relationship(a, "KNOWS", b)
a_knows_b = Relationship(a, "FRIEND", b)
graph.create(a_knows_b)
class User(StructuredNode):
uuid = UniqueIdProperty()
email = StringProperty(unique_index=True)
pw = StringProperty()
depth = IntegerProperty(index=True, default=0)
domains = Relationship('tradespace.models.aux.Domain', 'PERMITS')
def logged_in(self, *args, **kwargs):
# check session # session.
if (self.email and 'userinfo' in session):
return True
else:
return False
def invited_user(self, *args, **kwargs):
return self.email in official_invites
@classmethod
def merge(*args, **kwargs):
graph = dbutil.get_graph()
user_instance = User(*args, **kwargs)
tx = graph.begin()
n = Node("User", **kwargs)
tx.merge(n, primary_label='User', primary_key='email')
tx.commit()
user_instance.node = user_instance
return user_instance
def generate_registration(self, *args, **kwargs):
from tradespace.models.aux import Registration
token = secrets.token_urlsafe(16)
r = Registration(uuid=token,
ip=xweb.get_ip(),
domain=xweb.get_domain(),
create_date=datetime.now()).save()
rel = r.user.connect(self)
try:
xmail.send(recipient=self.email,
subject=xweb.get_domain() + ' Activation',
message=xweb.get_protocol() + xweb.get_url() +
'/activate/' + token)
except Exception as e:
self.log_event(tag='auth_error')
def activate_registration(self, registration):
from tradespace.models.aux import Domain
d = Domain.nodes.get(name=registration.domain)
self.domains.connect(d)
def permission_on_domain(self):
cur_domain = xweb.get_domain()
if len(list(self.domains.filter(name=cur_domain))) > 0:
return True
else:
return False
def log_event(self, *args, **kwargs):
graph = dbutil.get_graph()
tx = graph.begin()
tag = kwargs['tag']
u = graph.nodes.match("User", email=self.email)
e = Node("Event", domain=domain, tag=tag)
r = Relationship(u, "EMANATES", e)
tx.create(e)
tx.create(r)
tx.commit()
def termify_domain(self, *args, **kwargs):
domain = kwargs['domain']
ip = kwargs['ip']
graph = dbutil.get_graph()
tx = graph.begin()
u = graph.nodes.match("User", email=self.email)
if (u.email in official_invites):
t = Node("Terms", domain=domain, ip=ip)
tx.create(Relationship(u, "AGREES", t))
tx.commit()
return True
else:
return False
def activation_on_domain(self, *args, **kwargs):
graph = dbutil.get_graph()
tx = graph.begin()
u = graph.nodes.match("User", email=self.email).first()
a = graph.nodes.match("Activation", domain=xweb.get_domain()).first()
if (u and a):
rlt = Relationship(u, a)
records = graph.run(
"MATCH ( u:User { email:$email} )-[r]-( a:Activation ) RETURN r",
email=self.email,
domain=xweb.get_domain())
recs = records.data()
recslen = len(recs)
if (len(recs) > 0):
return True
else:
return False
else:
return False
def terms_on_domain(self, *args, **kwargs):
graph = dbutil.get_graph()
domain = args[0]
graph = dbutil.get_graph()
u = graph.nodes.match("User", email=self.email)
t = graph.nodes.match("Terms", domain=domain)
ut = Relationship(u, "HAS", t)
return graph.exists(ut)
def apps_on_domain(self, *args, **kwargs):
graph = dbutil.get_graph()
apps = []
domain = args[0] if args[0] else self.domain
if self.logged_in() and self.activation_on_domain(domain):
records = graph.run(
"MATCH ( a:App )--( d:Domain { url:$url} ) RETURN a",
email=self.email,
url=domain)
apps += [dict(x['a']) for x in records.data()]
elif self.logged_in() and not self.activation_on_domain(domain):
records = graph.run(
"MATCH ( a:App ) WHERE a.identifier='logout' XOR a.type='contingency' RETURN a"
)
apps += [dict(x['a']) for x in records.data()]
else:
records = graph.run("MATCH ( a:App { type:'auth'} ) RETURN a",
email=self.email)
apps += [dict(x['a']) for x in records.data()]
return apps
# JWT
#This info is often referred to as JWT Claims.
# We utilize the following “claims”:
#exp: expiration date of the token
#iat: the time the token is generated
#sub: the subject of the token (the user whom it identifies)
#The secret key must be random and only accessible server-side. Use the Python interpreter to generate a key:
# >>> import os
# >>> os.urandom(24)
# b"\xf9'\xe4p(\xa9\x12\x1a!\x94\x8d\x1c\x99l\xc7\xb7e\xc7c\x86\x02MJ\
def encode_auth_token(self, user_id):
"""
Generates the Auth Token
:return: string
"""
try:
payload = {
'exp':
datetime.datetime.utcnow() +
datetime.timedelta(days=0, seconds=5),
'iat':
datetime.datetime.utcnow(),
'sub':
user_id
}
return jwt.encode(payload,
app.config.get('SECRET_KEY'),
algorithm='HS256')
except Exception as e:
return e
@staticmethod
def decode_auth_token(auth_token):
"""
Decodes the auth token
:param auth_token:
:return: integer|string
"""
try:
payload = jwt.decode(auth_token, app.config.get('SECRET_KEY'))
return payload['sub']
except jwt.ExpiredSignatureError:
return 'Signature expired. Please log in again.'
except jwt.InvalidTokenError:
return 'Invalid token. Please log in again.'
#更新实体属性
from py2neo import Node, Relationship
a = Node('Person', name='Alice')
b = Node('Person', name='Bob')
r = Relationship(a, 'KNOWS', b)
a['age'] = 20
a.setdefault('location', '北京')
data = {'name': 'Amy', 'age': 21}
a.update(data)
b['age'] = 21
r['time'] = '2017/08/31'
print(a, b, r)
def make_child(node_parent, node_child):
# print("make child?")
relationship = Relationship(node_parent, edge_word, node_child)
graph.merge(relationship)
def create_neo4jgraph(graph, cm_file):
sheet = cm_file.sheet_by_index(0)
graph.schema.create_index("Study", "study_id")
graph.schema.create_index("Subject", "subject_id")
graph.schema.create_index("Compound", "ae_compound")
graph.schema.create_index("Therapeutic","ae_therapeutic_area")
graph.schema.create_index("AdverseEffect","ae_seq")
for r in range(1, 200):
# Extract the required values from the excel sheets and store them.
# At present the functionality extracts all values from a single excel sheet
ae_id = str(sheet.cell(r, 0).value)
study_id = str(sheet.cell(r, 1).value)
domain = str(sheet.cell(r,2).value)
subject_id = str(sheet.cell(r,3).value)
ae_seq = str(sheet.cell(r,4).value)
ae_group_id = str(sheet.cell(r,5).value)
ae_reference_id = str(sheet.cell(r, 6).value)
ae_sp_id = str(sheet.cell(r, 7).value)
ae_term = str(sheet.cell(r, 8).value)
ae_modify = str(sheet.cell(r, 9).value)
ae_llt = str(sheet.cell(r, 10).value)
ae_decode = str(sheet.cell(r, 12).value)
ae_hlt = str(sheet.cell(r, 14).value)
ae_hlgt = str(sheet.cell(r, 16).value)
ae_category = str(sheet.cell(r, 18).value)
ae_subcategory = str(sheet.cell(r, 19).value)
ae_bodysystem = str(sheet.cell(r, 21).value)
ae_systemorganclass = str(sheet.cell(r, 23).value)
ae_location = str(sheet.cell(r, 25).value)
ae_severity = str(sheet.cell(r, 26).value)
ae_serious_event = str(sheet.cell(r, 27).value)
ae_action_taken_with_study_treatment = str(sheet.cell(r, 28).value)
ae_other_action_taken = str(sheet.cell(r, 29).value)
ae_causality = str(sheet.cell(r, 30).value)
ae_relationship_to_nonstudy_tratment = str(sheet.cell(r, 31).value)
ae_pattern = str(sheet.cell(r, 32).value)
ae_outcome = str(sheet.cell(r, 33).value)
ae_involves_cancer = str(sheet.cell(r, 34).value)
ae_birth_defect = str(sheet.cell(r, 35).value)
ae_disability = str(sheet.cell(r, 36).value)
ae_results_in_death = str(sheet.cell(r, 37).value)
ae_prolong_hospitalization = str(sheet.cell(r, 38).value)
ae_life_threatning = str(sheet.cell(r, 39).value)
ae_overdose = str(sheet.cell(r, 40).value)
ae_other_medical_serios_event = str(sheet.cell(r, 41).value)
ae_other_treatment_given = str(sheet.cell(r, 42).value)
ae_standard_toxicity_grade = str(sheet.cell(r, 43).value)
ae_duration = str(sheet.cell(r, 48).value)
ae_end_relative_reference_period = str(sheet.cell(r, 49).value)
ae_end_reference_time_point = str(sheet.cell(r, 51).value)
ae_start_datetime = None if (sheet.cell(r, 44).value == '') else str(datetime.datetime(
*xlrd.xldate_as_tuple(sheet.cell(r, 44).value, cm_file.datemode)))
ae_end_datetime = None if (sheet.cell(r, 45).value == '') else str(datetime.datetime(
*xlrd.xldate_as_tuple(sheet.cell(r, 45).value, cm_file.datemode)))
ae_study_startday = str(sheet.cell(r, 46).value)
ae_study_endday = str(sheet.cell(r, 47).value)
ae_therapeutic_area = str(sheet.cell(r, 52).value)
ae_compound = str(sheet.cell(r, 53).value)
#Create the Node structure for Study, Subject, Compound, Therapeutic,AdverseEffects.
study_node=Node("Study", name=study_id)
graph.merge(study_node,"Study")
compound_node=Node("Compound",name=ae_compound)
graph.merge(compound_node,"Compound")
therapeutic_node=Node("Therapeutic", name=ae_therapeutic_area)
graph.merge(therapeutic_node,"Therapeutic")
subject_node=Node("Subject", name=subject_id)
graph.merge(subject_node, "Subject")
adverse_effect_node=Node("AdverseEffect", name=ae_seq, id=ae_sp_id,domain=domain,reported_term=ae_term,modified_reported_term=ae_modify,low_level_term=ae_llt,derived_term=ae_decode,high_level_term=ae_hlt,
high_level_group_term=ae_hlgt,category=ae_category,subcategory=ae_subcategory,bodysystem=ae_bodysystem,systemorganclass=ae_systemorganclass,location=ae_location,
severity=ae_severity,serious_event=ae_serious_event,action_taken_with_study_treatment=ae_action_taken_with_study_treatment,ae_other_action_taken=ae_other_action_taken,
causality=ae_causality,relationship_to_nonstudy_tratment=ae_relationship_to_nonstudy_tratment,pattern=ae_pattern,outcome=ae_outcome,involves_cancer=ae_involves_cancer,
birth_defect=ae_birth_defect,disability=ae_disability,results_in_death=ae_results_in_death,prolong_hospitalization=ae_prolong_hospitalization,life_threatning=ae_life_threatning,
overdose=ae_overdose,other_medical_serios_event=ae_other_medical_serios_event,other_treatment_given=ae_other_treatment_given,standard_toxicity_grade=ae_standard_toxicity_grade,
duration=ae_duration,end_relative_reference_period=ae_end_relative_reference_period,end_reference_time_point=ae_end_reference_time_point)
graph.merge(adverse_effect_node, "AdverseEffect")
#Create relationship between nodes
study_compound_relationship = Relationship(study_node, "HAS", compound_node)
graph.merge(study_compound_relationship, study_node, compound_node)
therapeutic_compound_relationship = Relationship(therapeutic_node, "HAS", compound_node)
graph.merge(therapeutic_compound_relationship, therapeutic_node,compound_node)
compound_therapeutic_relationship = Relationship(compound_node, "HAS", therapeutic_node)
graph.merge(compound_therapeutic_relationship,compound_node, therapeutic_node)
study_subject_relationship = Relationship(study_node, "HAS", subject_node)
graph.merge(study_subject_relationship,study_node, subject_node)
subject_ae_relationship = Relationship(subject_node,"HAS",adverse_effect_node,start_datetime=ae_start_datetime,end_datetime=ae_end_datetime,study_startday=ae_study_startday,study_endday=ae_study_endday)
graph.merge(subject_ae_relationship,subject_node,adverse_effect_node)
graph.create(study_compound_relationship | therapeutic_compound_relationship | subject_ae_relationship | compound_therapeutic_relationship | study_subject_relationship)
def require_ingredient(self, ingredient):
rel = Relationship(self.node, 'REQUIRES', ingredient.node)
graph.create(rel)
df = pd.read_excel('./excels/santi.xlsx')
# 取前610行作为图谱数据
# df = df.iloc[:706, :]
# 连接Neo4j服务
graph = Graph(host="localhost://7474", auth=("neo4j", "jc147369"))
# 创建节点
nodes = set(df['S'].tolist()+df['O'].tolist())
for node in nodes:
node = Node("Node", name=node)
graph.create(node)
print('create nodes successfully!')
# 创建关系
matcher = NodeMatcher(graph)
for i in range(df.shape[0]):
S = df.iloc[i, :]['S'] # S节点
O = df.iloc[i, :]['O'] # O节点
s_node = matcher.match("Node", name=S).first()
o_node = matcher.match("Node", name=O).first()
# 创建关系
P = df.iloc[i, :]['P']
relationship = Relationship(s_node, P, o_node)
graph.create(relationship)
print('create relationships successfully!')
print('You can check Neo4j now!')
def topic_word_graph(topic_words):
for n in range(config.NMF_settings['num_topics']):
topic = Node("Topic", name="Topic "+str(n))
for i in range(config.NMF_settings['num_topic_words']):
word = Node("Word", name=topic_words[n][i])
graph.merge(Relationship(topic, "CONSISTS_OF", word))
def test_cannot_delete_uncreated_relationship(graph):
a = Node()
b = Node()
graph.create(a | b)
r = Relationship(a, "TO", b)
graph.delete(r)
dest="neopassword",
help="Account Password for neo4j Login")
parser.add_argument('--neohost', dest="neohost", help="adresse du neo4j")
parser.add_argument('--neoport', dest="neoport", help="port du serveur neo4j")
parser.add_argument('-v', dest="verbose", action='count', help="mode verbeux")
options = parser.parse_args()
url = os.environ.get('GRAPHENEDB_URL',
'http://' + options.neohost + ':' + options.neoport)
graph = Graph(url + '/db/data/',
username=options.neousername,
password=options.neopassword)
list = {}
testcsv = open('pf-clients.csv', 'r')
csvreader = csv.reader(testcsv, delimiter=';')
# on saute le header
csvreader.next()
for i in csvreader:
list.update({i[0]: i[1]})
testcsv.close()
pprint.pprint(list)
for pf in list:
neo_client = Node('Compte', name=list[pf])
graph.merge(neo_client)
neo_pf = Node('Plateform', name=pf)
graph.merge(neo_pf)
link = Relationship(neo_client, 'have', neo_pf)
graph.merge(link)
def test_relationship_exists(graph):
a = Node()
b = Node()
r = Relationship(a, "TO", b)
graph.create(r)
assert graph.exists(r)
def cti_groups_graph(scenario,
graph_uri=default_uri,
graph_user=default_user,
graph_password=default_password,
network=None):
cve_list = scenario['cve']
capec_list = scenario['capec']
technique_list = scenario['technique']
software_list = scenario['software']
group_list = scenario['group']
host_list = ['localhost']
if network != None:
host_list = list(network.keys())
g = Graph(uri=graph_uri, user=graph_user, password=graph_password)
matcher = NodeMatcher(g)
g.delete_all()
tx = g.begin()
for host in host_list:
host_node = Node("Host", name=host)
tx.create(host_node)
for cve_id in cve_list:
cve_node = Node("CVE",
name=cve_id,
cwe=cve.cwe(cve_id),
base_score=cve.cvss_basescore(cve_id),
exploitability_score=cve.cvss_exscore(cve_id),
impact_score=cve.cvss_impactscore(cve_id),
vector_string=cve.cvss_vector(cve_id))
tx.create(cve_node)
for capec_id in capec_list:
capec_node = Node("CAPEC",
name=('CAPEC-' + str(capec_id)),
cwe=capec.cwe(capec_id),
likelihood=capec.likelihood(capec_id),
severity=capec.severity(capec_id),
prerequisites=capec.prerequisites(capec_id),
skill=capec.skill(capec_id),
consequences=capec.consequences(capec_id),
resources=capec.resources(capec_id))
tx.create(capec_node)
for technique in technique_list:
technique_node = Node(
"TECHNIQUE",
name=attack.attack_id(technique),
extended_name=attack.name(technique),
platforms=attack.technique_platforms(technique),
permissions=attack.permissions(technique),
description=attack.description(technique),
datasource=attack.data_sources(technique),
kill_chain_phase=attack.kill_chain_phases(technique),
impact=attack.technique_impact(technique))
tx.create(technique_node)
for software in software_list:
sofware_node = Node("SOFTWARE",
name=attack.attack_id(software),
extended_name=attack.software_name(software),
description=attack.software_description(software),
platforms=attack.software_platforms(software),
software_type=attack.software_type(software))
tx.create(sofware_node)
for group in group_list:
group_node = Node("GROUP",
name=attack.attack_id(group),
extended_name=attack.group_name(group),
description=attack.group_description(group))
tx.create(group_node)
tx.commit()
tx = g.begin()
for host in host_list:
host_node = matcher.match("Host").where("_.name =~ '" + host +
"'").first()
if network == None:
for cve_id in cve_list:
cve_node = matcher.match("CVE").where("_.name =~ '" +
str(cve_id) +
"'").first()
host_cve = Relationship(host_node, "PRESENT", cve_node)
tx.merge(host_cve, "PRESENT", "name")
else:
for cve_id in network[host]['cve_list']:
cve_node = matcher.match("CVE").where("_.name =~ '" +
str(cve_id) +
"'").first()
host_cve = Relationship(host_node, "PRESENT", cve_node)
tx.merge(host_cve, "PRESENT", "name")
for cve_id in cve_list:
cve_node = matcher.match("CVE").where("_.name =~ '" + cve_id +
"'").first()
for capec_id in cve.capec(cve_id):
capec_node = matcher.match("CAPEC").where("_.name =~ '" +
('CAPEC-' +
str(capec_id)) +
"'").first()
cve_capec = Relationship(cve_node, "EXPLOITED", capec_node)
tx.merge(cve_capec, "EXPLOITED", "name")
for capec_id in capec_list:
capec_node = matcher.match("CAPEC").where("_.name =~ '" +
('CAPEC-' + str(capec_id)) +
"'").first()
for technique in capec.attack_technique(capec_id):
if technique in technique_list:
technique_node = matcher.match("TECHNIQUE").where(
"_.name =~ '" + attack.attack_id(technique) + "'").first()
capec_technique = Relationship(capec_node, "IMPLEMENTED",
technique_node)
tx.merge(capec_technique, "IMPLEMENTED", "name")
for technique in technique_list:
technique_node = matcher.match("TECHNIQUE").where(
"_.name =~ '" + attack.attack_id(technique) + "'").first()
for software in attack.relationship_with(technique, "software"):
if software in software_list:
software_node = matcher.match("SOFTWARE").where(
"_.name =~ '" + attack.attack_id(software) + "'").first()
technique_software = Relationship(technique_node, "ENABLED",
software_node)
tx.merge(technique_software, "ENABLED", "name")
for software in software_list:
software_node = matcher.match("SOFTWARE").where(
"_.name =~ '" + attack.attack_id(software) + "'").first()
for group in attack.relationship_with(software, "intrusion-set"):
group_node = matcher.match("GROUP").where("_.name =~ '" +
attack.attack_id(group) +
"'").first()
software_group = Relationship(software_node, "USED", group_node)
tx.merge(software_group, "USED", "name")
tx.commit()
return
def test_only_one_relationship_in_a_relationship(graph):
rel = Relationship({}, "KNOWS", {})
assert len(rel.relationships) == 1
FLAGS = str(data['flags'])
com_node = graph.find_one(label='COMPANY', property_key='entname', property_value=entname)
if not com_node:
com_node = Node('COMPANY')
com_node['entname'] = entname
com_node['regcap_CNY'] = regcap_CNY
com_node['FLAGS'] = FLAGS
graph.create(com_node)
else:
com_node['regcap_CNY'] = regcap_CNY
graph.push(com_node)
if holder_label == 'COMPANY':
holder_node = graph.find_one(label='COMPANY', property_key='entname', property_value=holder)
if not holder_node:
holder_node = Node('COMPANY')
holder_node['entname'] = holder
holder_node['FLAGS'] = FLAGS
graph.create(holder_node)
else:
holder_node = graph.find_one(label='PERSON', property_key='name', property_value=holder)
if not holder_node:
holder_node = Node('PERSON')
holder_node['name'] = holder
holder_node['FLAGS'] = FLAGS
graph.create(holder_node)
rel = Relationship(holder_node, 'Share', com_node)
rel['money_CNY'] = money_CNY
rel['FLAGS'] = FLAGS
graph.merge(rel)
print(i+1, rel.start_node(), rel.relationships(), rel.end_node())
def create_wordnet_rel(self, synset1, synset2, ptype):
"""
Pointer symbols
http://wordnet.princeton.edu/wordnet/man/wninput.5WN.html
The pointer_symbol s for nouns are:
! Antonym
@ Hypernym
@i Instance Hypernym
~ Hyponym
~i Instance Hyponym
#m Member holonym
#s Substance holonym
#p Part holonym
%m Member meronym
%s Substance meronym
%p Part meronym
= Attribute
+ Derivationally related form
;c Domain of synset - TOPIC
-c Member of this domain - TOPIC
;r Domain of synset - REGION
-r Member of this domain - REGION
;u Domain of synset - USAGE
-u Member of this domain - USAGE
The pointer_symbol s for verbs are:
! Antonym
@ Hypernym
~ Hyponym
* Entailment
> Cause
^ Also see
$ Verb Group
+ Derivationally related form
;c Domain of synset - TOPIC
;r Domain of synset - REGION
;u Domain of synset - USAGE
The pointer_symbol s for adjectives are:
! Antonym
& Similar to
< Participle of verb
\ Pertainym (pertains to noun)
= Attribute
^ Also see
;c Domain of synset - TOPIC
;r Domain of synset - REGION
;u Domain of synset - USAGE
The pointer_symbol s for adverbs are:
! Antonym
\ Derived from adjective
;c Domain of synset - TOPIC
;r Domain of synset - REGION
;u Domain of synset - USAGE
"""
node1 = self.graph_db.find_one(self.nodelabel,
property_key="synset_id",
property_value=synset1)
node2 = self.graph_db.find_one(self.nodelabel,
property_key="synset_id",
property_value=synset2)
if (node1 is not None) and (node2 is not None):
rel = Relationship(node1,
self.reltype,
node2,
pointer_symbol=ptype)
return rel
else:
raise Exception(
"Could not create Wordnet relation (%s) - [%s] -> (%s)" %
(synset1, ptype, synset2))
