def create_elb():
elbs = loadbalancer.describe_load_balancers()['LoadBalancerDescriptions']
for elb in elbs:
tx = graph.begin()
graphElb = Node("ELB", name=elb['LoadBalancerName'])
tx.merge(graphElb)
tx.commit()
for subnet in elb['Subnets']:
tx = graph.begin()
selector = NodeSelector(graph)
graphSubnet = selector.select("Subnet", subnetId=subnet).first()
rel = Relationship(graphElb, "BELONGS", graphSubnet)
tx.create(rel)
tx.commit()
for instance in elb["Instances"]:
try:
tx = graph.begin()
selector = NodeSelector(graph)
graphInstance = selector.select(
"EC2", instanceId=instance['InstanceId']).first()
rel = Relationship(graphInstance, "BELONGS", graphElb)
tx.create(rel)
tx.commit()
except:
pass
def create_alb():
albs = elbv2.describe_load_balancers()['LoadBalancers']
for alb in albs:
# print(alb)
selector = NodeSelector(graph)
graphVPC = selector.select("VPC", vpcId=alb['VpcId']).first()
tx = graph.begin()
graphAlb = Node("ALB",
name=alb['LoadBalancerName'],
fqdn=alb['DNSName'],
type=alb['Type'],
parent=graphVPC['name'])
rel = Relationship(graphAlb, "EXIST_IN", graphVPC)
tx.create(rel)
tx.merge(graphAlb)
graphFQDN = Node("FQDN", name=alb['DNSName'])
relFQDN = Relationship(graphAlb, "AVAILABLE_ON", graphFQDN)
tx.create(relFQDN)
tx.commit()
albArn = alb['LoadBalancerArn']
# for subnet in alb['AvailabilityZones']:
# tx = graph.begin()
# selector = NodeSelector(graph)
# graphSubnet = selector.select("Subnet",subnetId=subnet['SubnetId']).first()
# rel = Relationship(graphAlb, "BELONGS", graphSubnet)
# tx.create(rel)
# tx.commit()
tgs = elbv2.describe_target_groups(
LoadBalancerArn=albArn)['TargetGroups']
for tg in tgs:
# print(tg)
tgArn = tg['TargetGroupArn']
targets = elbv2.describe_target_health(
TargetGroupArn=tgArn)['TargetHealthDescriptions']
tx = graph.begin()
graphTG = Node("TargetGroup",
name=tg['TargetGroupName'],
protocol=tg['Protocol'],
port=tg['Port'],
parent=graphVPC['name'])
tx.merge(graphTG)
rel = Relationship(graphTG, "EXPOSE_PORT_" + str(tg['Port']),
graphAlb)
tx.create(rel)
tx.commit()
for target in targets:
try:
tx = graph.begin()
selector = NodeSelector(graph)
graphInstance = selector.select(
"EC2", instanceId=target['Target']['Id']).first()
rel = Relationship(graphInstance, "BELONGS", graphTG)
tx.create(rel)
tx.commit()
except:
pass
def node_exists(graph, nodeType, propertyname, propertyvalue):
"""
Check if a node exists given it's type and label
On success will return the node.
On failure, will return none
"""
graph = graphs[graph]
#TODO: rewrite this using NodeSelector, find_one is deprecated.
# print('debug')
# print(propertyname)
# print(propertyvalue)
selector = NodeSelector(graph)
#TODO: Test this!
# search1 = selector.select(nodeType).where("_."+propertyname+"="+propertyvalue).first()
search1 = graph.find_one(nodeType,
property_key=propertyname,
property_value=propertyvalue)
if search1:
search1["nodeType"] = list(search1.labels())[0]
return search1
else:
return None
def get_all_projects(current_user):
#if not current_user.is_admin:
#return jsonify({'message': 'User not authorized!'})
selector = NodeSelector(graph)
projects = selector.select("Project")
projects_list = []
for project in projects:
users_list = []
for rel in graph.match(end_node=project, rel_type="IS_USER"):
user_umcn = rel.start_node()["umcn"]
users_list.append(user_umcn)
admin = ""
for rel in graph.match(end_node=project, rel_type="IS_ADMIN"):
admin_umcn = rel.start_node()["umcn"]
admin = admin_umcn
project_obj = ({
"id": remote(project)._id,
"name": project['name'],
"client": project['client'],
"deadline": project['deadline'],
"priority": project['priority'],
"users": users_list,
"admin": admin
})
projects_list.append(project_obj)
return jsonify(Projects=projects_list)
def __init__(self, password="******", userid="A0001"):
self.graph = Graph("http://localhost:7474/db/data/", password=password)
self.selector = NodeSelector(self.graph)
# self.locations = get_navigation_location()
self.is_scene = False
self.user = None
# self.user = self.selector.select("User", userid=userid).first()
# self.usertopics = self.get_usertopics(userid=userid)
# self.address = get_location_by_ip(self.user['city'])
self.topic = ""
self.behavior = 0 # 场景类型 Add in 2018-6-7
self.last_step_error = False # 在场景内上一个问答是否正常 Add in 2018-6-12
self.qa_id = get_current_time()
self.qmemory = deque(maxlen=10)
self.amemory = deque(maxlen=10)
self.pmemory = deque(maxlen=10)
# TODO：判断意图是否在当前话题领域内
self.change_attention = ["换个话题吧", "太无聊了", "没意思", "别说了"]
self.cmd_end_scene = ["退出业务场景", "退出场景", "退出", "返回", "结束", "发挥"]
self.cmd_previous_step = ["上一步", "上一部", "上一页", "上一个"]
self.cmd_next_step = ["下一步", "下一部", "下一页", "下一个"]
self.cmd_repeat = ["重复", "再来一个", "再来一遍", "你刚说什么", "再说一遍", "重来"]
self.yes = ["是", "是的", "对", "对的", "好的", "YES", "yes", "结束了"]
self.no = ["没", "没有", "否", "不", "没有结束", "没结束"]
self.do_not_know = [
"这个问题太难了，{robotname}还在学习中", "这个问题{robotname}不会，要么我去问下",
"您刚才说的是什么，可以再重复一遍吗", "{robotname}刚才走神了，一不小心没听清",
"{robotname}理解的不是很清楚啦，你就换种方式表达呗", "不如我们换个话题吧", "咱们聊点别的吧",
"{robotname}正在学习中", "{robotname}正在学习哦", "不好意思请问您可以再说一次吗",
"额，这个问题嘛。。。", "{robotname}得好好想一想呢", "请问您说什么", "您问的问题好有深度呀",
"{robotname}没有听明白，您能再说一遍吗"
]
def find_by_name_exactly_match_for_wikidata(self, name):
try:
selector = NodeSelector(self.graph)
return selector.select("wikidata", labels_en=name).first()
except Exception, error:
_logger.exception()
return None
def node_exist(self, number):
client = Graph(password=os.getenv('NEO4J_PASSWORD', 'Neo4j'))
selector = NodeSelector(client)
selected = selector.select("number", **{"number": number}).first()
if selected:
return selected
return False
def node_exists_label_in_list(graph, nodeType, propertyname, propertyvalue,
**kwargs):
"""
Check if a node exists given:
- type
- a property of type list:
- a value to search for in it
On success will return the node.
On failure, will return none
"""
graph = graphs[graph]
selector = NodeSelector(graph)
search1 = selector.select(nodeType)
if search1:
search_res = list(search1)
for ind_res in search_res:
if propertyname in ind_res:
for ind_prop in ind_res[propertyname]:
if ind_prop == propertyvalue:
ind_res["nodeType"] = list(ind_res.labels())[0]
return ind_res
return None
else:
return None
def __init__(self, auth_tweepy, auth_neo4j, n, seed, next_node, w=50):
# sets current account on Twitter, as specified in the secrets file. Only uses one.
self.curr_acc = 0
self.accounts = list(auth_tweepy.items())
self.api = MHCrawler.auth_tweepy(self.accounts[self.curr_acc][1])
# authenticates neo4j
self.graph = MHCrawler.auth_neo4j(auth_neo4j)
# starts node selector from py2neo
self.node_selector = NodeSelector(self.graph)
# starts random walk parameters
self.seed, self.n, self.w = seed, n, w
self.next_node = list(self.node_selector.select("User", id=next_node["id"]))[0] \
if next_node is not None else next_node
self.previous_node = None
# checks for uniqueness constraints/index keys in the database and creates them if they don't exist
if self.graph.schema.get_uniqueness_constraints("User") != ["id"]:
self.graph.schema.create_uniqueness_constraint("User", "id")
if self.graph.schema.get_uniqueness_constraints("Tweet") != ["id"]:
self.graph.schema.create_uniqueness_constraint("Tweet", "id")
if self.graph.schema.get_uniqueness_constraints("Media") != ["url"]:
self.graph.schema.create_uniqueness_constraint("Media", "url")
if "number" not in self.graph.schema.get_indexes("User"):
self.graph.schema.create_index("User", "number")
def store_in_neo4j(triple):
from py2neo import Graph, Node, Relationship ,NodeSelector
graph = Graph('http://52.83.213.55:7474',user ='******',password = '******')
# graph = Graph('http://localhost:7474',user = '******',password='******')
select = NodeSelector(graph)
# 加载entity
with open('entity.pkl','rb') as f:
entities = pickle.load(f)
entities = list(flatten(entities))
# 添加所有实体为结点
for en in entities:
node = Node('Entity',name= en)
graph.create(node)
# 遍历三元组，添加节点的属性，结点间关系等
for en, kw in triple.items():
node_1 = select.select('Entity').where(name = en).first()
for item in kw:
if item[1] in triple.keys():
node_2 = select.select('Entity').where(name = item[1]).first()
relate = Relationship(node_1,item[0],node_2)
graph.create(relate)
else:
node_1[item[0]] = item[1]
graph.push(node_1)
print('数据存储完毕')
def get_projects_of_user(current_user, umcn):
selector = NodeSelector(graph)
users = selector.select("Person", umcn=umcn)
user = list(users)[0]
myprojects = []
for rel in graph.match(start_node=user, rel_type="IS_USER"):
project = rel.end_node()
myprojects.append(project)
projects_list = []
for project in myprojects:
users_list = []
for rel in graph.match(end_node=project, rel_type="IS_USER"):
user_umcn = rel.start_node()["umcn"]
users_list.append(user_umcn)
admin = ""
for rel in graph.match(end_node=project, rel_type="IS_ADMIN"):
admin_umcn = rel.start_node()["umcn"]
admin = admin_umcn
project_obj = ({"id": remote(project)._id,
"name": project['name'],
"client": project['client'],
"deadline": project['deadline'],
"priority": project['priority'],
"users": users_list,
"admin": admin
})
projects_list.append(project_obj)
return jsonify(Projects=projects_list)
def fill_attribute_graph(graph):
# graph.delete_all()
attribute_df = pd.read_csv('data/attributes.csv', usecols=['attribute', 'frequency'])
coexistence_df = pd.read_csv('data/coexistencesProb.csv', usecols=['attribute1','attribute2','totals','exp','diff','weight'])
freq_lookup_dict = dict(zip(attribute_df.attribute.values, attribute_df.frequency.values))
# create nodes
for attribute, frequency in freq_lookup_dict.items():
x = Node('Attribute', attribute=str(attribute), frequency=str(frequency)) # create cypher object
graph.merge(x, 'Attribute', 'attribute') # merge is important to prevent duplicates
# create relationships
for index, row in coexistence_df.iterrows():
rel_type = 'COOCCURS_WITH'
try:
selector = NodeSelector(graph)
attrubute1_node = selector.select("Attribute", attribute=row.attribute1).first()
attrubute2_node = selector.select("Attribute", attribute=row.attribute2).first()
if (attrubute1_node != None
and attrubute2_node != None
and row.totals != None
and row.weight != None):
graph.merge(Relationship(attrubute1_node, rel_type, attrubute2_node, coexistance=row.totals, weight=row.weight))
except AttributeError:
print(index)
print(row)
def __init__(self, password="******", userid="A0001"):
self.graph = Graph("http://localhost:7474/db/data/", password=password)
self.selector = NodeSelector(self.graph)
# self.locations = get_navigation_location()
self.is_scene = False
self.user = self.selector.select("User", userid=userid).first()
self.usertopics = self.get_usertopics(userid=userid)
self.address = get_location_by_ip(self.user['city'])
self.topic = ""
self.qa_id = get_current_time()
self.qmemory = deque(maxlen=10)
self.amemory = deque(maxlen=10)
self.pmemory = deque(maxlen=10)
self.cmd_end_scene = ["退出业务场景", "退出场景", "退出", "返回", "结束", "发挥"]
self.cmd_previous_step = ["上一步", "上一部", "上一页", "上一个"]
self.cmd_next_step = ["下一步", "下一部", "下一页", "下一个"]
self.cmd_repeat = ['重复', '再来一个', '再来一遍', '你刚说什么', '再说一遍', '重来']
self.do_not_know = [
"这个问题太难了，{robotname}还在学习中", "这个问题{robotname}不会，要么我去问下",
"您刚才说的是什么，可以再重复一遍吗", "{robotname}刚才走神了，一不小心没听清",
"{robotname}理解的不是很清楚啦，你就换种方式表达呗", "不如我们换个话题吧", "咱们聊点别的吧",
"{robotname}正在学习中", "{robotname}正在学习哦", "不好意思请问您可以再说一次吗",
"额，这个问题嘛。。。", "{robotname}得好好想一想呢", "请问您说什么", "您问的问题好有深度呀",
"{robotname}没有听明白，您能再说一遍吗"
]
def find_wikidata_node(self, wd_item_id):
try:
selector = NodeSelector(self.graph)
return selector.select('wikidata', wd_item_id=wd_item_id).first()
except Exception, error:
_logger.exception()
return None
def exists_node(self, label, name):
''' checks to see if a node exists by name '''
print("lookup ", label, " ", name)
g = Graph(password=self.password)
selector = NodeSelector(g)
selected = selector.select(label, name=name)
ret = list(selected)
return len(ret), ret
def search():
print(request.json)
search_type = request.json['search_type']
search_input = request.json['search_input']
database_config = current_app.config['SERVER_CONFIG']['database']
print("[api_v1_app] /search: connecting to neo4j...")
database = Graph("{protocol}://{ip}:{port}".format(
protocol=database_config['connection']['protocol'],
ip=database_config['connection']['ip'],
port=database_config['connection']['port']),
user=database_config['auth']['user'],
password=database_config['auth']['password'])
selector = NodeSelector(database)
if search_type == "operation_system":
records = selector.select("OperationSystem") \
.where("(any(prop in keys(_) where _[prop] CONTAINS '{input}'))".format(input=search_input))
elif search_type == "ftp":
records = selector.select("FTPServer") \
.where("(any(prop in keys(_) where _[prop] CONTAINS '{input}'))".format(input=search_input))
elif search_type == "product":
records = selector.select("ProductSet") \
.where("(any(prop in keys(_) where _[prop] CONTAINS '{input}'))".format(input=search_input))
elif search_type == "all":
records = selector.select() \
.where("(any(prop in keys(_) where _[prop] CONTAINS '{input}'))".format(input=search_input))
else:
result = {
'app': 'npdp-server',
'api': 'search',
'timestamp': time.time(),
'data': {
'status': 'error',
'message': 'type not supported.',
'request': request.form
}
}
return jsonify(result)
result_list = []
for record in records:
node_dict = get_node_dict(record)
result_list.append(node_dict)
result = {
'app': 'npdp-server',
'api': 'search',
'timestamp': time.time(),
'data': {
'status': 'ok',
'request': request.form,
'results': result_list
}
}
return jsonify(result)
def find_node(graph, node_name, output=None):
s = NodeSelector(graph)
try:
if (output):
return output(s.select(name=node_name).first())
else:
return s.select(name=node_name).first()
except:
return '没有这个' + node_name + '节点，查询失败'
def connect(self):
print("http://" + self.host + ":" + str(self.port), self.username,
self.password)
self.graph = Graph("http://" + self.host + ":" + str(self.port),
username=self.username,
password=self.password)
if self.graph != None:
print("Neo4j Database Connected.")
self.selector = NodeSelector(self.graph)
def is_persisted(self):
"""
Check if this already is persisted to neo4j
TODO: Make code less hateable
"""
neo = self.caller.to_neo()
label = list(neo.labels())[0]
neo.update(self.caller.props)
selector = NodeSelector(client)
return len(list(selector.select(label, **dict(neo)))) > 0
def add_constant_relationship_test2(self, dict_entities, article_num):
# key = type, "src" = initial , "dst" = destination
selector = NodeSelector(self.graph)
selected = selector.select(article=article_num)
for i, n1 in enumerate(list(selected)):
print("Building links... node ", i, "/", len(list(selected)))
for n2 in list(selected):
if n1 != n2:
self.graph.create(Relationship(n1, "SEMANTIC", n2))
def enum_vul(self, TaskID, Cypher_Conditions=None):
if Cypher_Conditions:
# selector.select.where not good for use , not support zh_cn just pure cypher
cypher = 'MATCH (n:HostVul) where n.TaskID="%s" %s RETURN n ' % (TaskID, Cypher_Conditions)
for data in self.graph.data(cypher):
yield data["n"]
else:
selector = NodeSelector(self.graph)
selected = selector.select("HostVul", TaskID=TaskID)
for data in list(selected):
yield data
def new_person(n,y):
selector = NodeSelector(connection.g)
node = selector.select("Person",name=n,born=y)
nodelist=list(node)
if len(nodelist) >1:
EditMovie.popupmsg("Person exists!")
else:
query = "CREATE (a:Person{name:'"+str(n)+"', born:'"+str(y)+"'})"
connection.g.run(query)
time.sleep(5)
EditMovie.controller.show_page("EditMovie")
def get_node_by_uuid(self, node_id):
"""
Retrieve a node from the neo4j database.
:param node_id: THe node to retrieve.
:return: The node
"""
selector = NodeSelector(self.graph_db)
selected = list(selector.select().where('_.name="{}"'.format(node_id)))
if selected:
return selected[0]
return None
def get_nodes_by_properties(self, properties):
"""
:param properties: Dictionary of properties, keys and values.
:return: List of node instances.
"""
conditions = list()
for key in properties.keys():
conditions.append('_.{} = "{}"'.format(key, properties[key]))
selector = NodeSelector(self.graph_db)
selected = selector.select().where(*conditions)
return list(selected)
def add_constant_relationship(self, dict_entities, article_num):
# key = type, "src" = initial , "dst" = destination
selector = NodeSelector(self.graph)
selected = selector.select(article=article_num)
document = Node("DOCUMENT", name=str(article_num), article=article_num)
self.graph.create(document)
for i, n1 in enumerate(list(selected)):
print("Building links... node ", i, "/", len(list(selected)))
self.graph.create(Relationship(n1, "MENTIONED IN", document))
def new_gk(n,type):
selector = NodeSelector(connection.g)
node = selector.select(type,name=n)
nodelist=list(node)
if len(nodelist) >1:
EditMovie.popupmsg("Input already exists!")
else:
query = "CREATE (a:"+type+"{name:'"+str(n)+"'})"
connection.g.run(query)
#EditMovie.popupmsg("done!")
time.sleep(5)
EditMovie.controller.show_page("EditMovie")
def __init__(self,
user='******',
password='******',
host='localhost',
port=7687):
# self.db = Database("bolt://camelot.example.com:7687")
self.graph = Graph(bolt=True,
host=host,
bolt_port=port,
user=user,
password=password)
self.selector = NodeSelector(self.graph)
def query_node_by_id(node_id):
database_config = current_app.config['SERVER_CONFIG']['database']
print("[api_v1_app] /search: connecting to neo4j...")
database = Graph("{protocol}://{ip}:{port}".format(
protocol=database_config['connection']['protocol'],
ip=database_config['connection']['ip'],
port=database_config['connection']['port']),
user=database_config['auth']['user'],
password=database_config['auth']['password'])
selector = NodeSelector(database)
records = list(selector.select().where(
"id(_)={node_id}".format(node_id=node_id)))
if len(records) == 0:
result = {
'app': 'npdp-server',
'api': 'node/ids',
'timestamp': time.time(),
'data': {
'status': 'error',
'message': "can't find node by id.",
'request': {
'id': node_id
}
}
}
return jsonify(result)
record = records[0]
node_dict = {
'labels': list(record.labels()),
'props': dict(record),
'id': node_id
}
result = {
'app': 'npdp-server',
'api': 'node/ids',
'timestamp': time.time(),
'data': {
'status': 'ok',
'request': {
'id': node_id
},
'node': node_dict
}
}
return jsonify(result)
def __init__(self, **neo4j_params):
"""
Method to instantiate the class in an object for the neostore.
:param neo4j_params: dictionary with Neo4J User, Pwd and Database. If host is not default localhost, it also
needs to be defined in the dictionary.
:return: Object to handle neostore commands.
"""
self.graph = self.connect2db(**neo4j_params)
self.calendar = GregorianCalendar(self.graph)
self.selector = NodeSelector(self.graph)
return
def create_sg():
securityGroups = ec2.describe_security_groups()
for sg in securityGroups['SecurityGroups']:
# print(sg)
selector = NodeSelector(graph)
graphVPC = selector.select("VPC", vpcId=sg['VpcId']).first()
tx = graph.begin()
graphSg = Node("SecurityGroup",
securityGroupId=sg['GroupId'],
name=sg['GroupName'],
parent=graphVPC['name'])
tx.merge(graphSg)
tx.commit()
