def form_update_node(user, handle_id, form, property_keys=None): """ Take a node, a form and the property keys that should be used to fill the node if the property keys are omitted the form.base_fields will be used. Returns True if all non-empty properties where added. """ meta_fields = ['relationship_location', 'relationship_end_a', 'relationship_end_b', 'relationship_parent', 'relationship_provider', 'relationship_end_user', 'relationship_customer', 'relationship_depends_on', 'relationship_user', 'relationship_owner', 'relationship_located_in', 'relationship_ports', 'services_checked', 'relationship_responsible_for', 'relationship_connected_to'] nh, node = get_nh_node(handle_id) if not property_keys: property_keys = [] for field in form.base_fields.keys(): if field not in meta_fields: property_keys.append(field) for key in property_keys: if form.cleaned_data.get(key, None) or form.cleaned_data.get(key, None) == 0: pre_value = node.data.get(key, '') if pre_value != form.cleaned_data[key]: node.data[key] = form.cleaned_data[key] # Handle dates if hasattr(form.cleaned_data[key], 'isoformat'): node.data[key] = form.cleaned_data[key].isoformat() if key == 'name': nh.node_name = form.cleaned_data[key] activitylog.update_node_property(user, nh, key, pre_value, form.cleaned_data[key]) elif form.cleaned_data.get(key, None) == '' and key in node.data.keys(): if key != 'name': # Never delete name pre_value = node.data.get(key, '') del node.data[key] activitylog.update_node_property(user, nh, key, pre_value, form.cleaned_data[key]) nc.set_node_properties(nc.graphdb.manager, node.handle_id, node.data) return True
def dict_update_node(user, handle_id, properties, keys=None, filtered_keys=list()): nh, node = get_nh_node(handle_id) if not keys: keys = properties.keys() for key in keys: if key in filtered_keys: continue if properties.get(key, None) or properties.get(key, None) == 0: pre_value = node.data.get(key, '') if pre_value != properties[key]: node.data[key] = properties[key] if key == 'name': nh.node_name = properties[key] nh.modifier = user nh.save() activitylog.update_node_property(user, nh, key, pre_value, properties[key]) elif properties.get(key, None) == '' and key in node.data.keys(): if key != 'name': # Never delete name pre_value = node.data.get(key, '') del node.data[key] activitylog.update_node_property(user, nh, key, pre_value, properties[key]) nc.set_node_properties(nc.graphdb.manager, handle_id, node.data) return True
def consume_noclook(json_list): """ Inserts the backup made with NOCLook producer. """ tot_items = len(json_list) tot_nodes = 0 print('Adding {!s} items.'.format(tot_items)) # Loop through all files starting with node for i in json_list: if i['host']['name'].startswith('node'): item = i['host']['noclook_producer'] properties = item.get('properties') node_name = properties.get('name') handle_id = item.get('handle_id') node_type = item.get('node_type') meta_type = item.get('meta_type') # Get a node handle nh = restore_node(handle_id, node_name, node_type, meta_type) nc.set_node_properties(nc.graphdb.manager, nh.handle_id, properties) logger.info('Added node {handle_id}.'.format(handle_id=handle_id)) tot_nodes += 1 print('Added {!s} nodes.'.format(tot_nodes)) # Loop through all files starting with relationship x = 0 with nc.graphdb.manager.transaction as t: for i in json_list: if i['host']['name'].startswith('relationship'): item = i['host']['noclook_producer'] properties = item.get('properties') q = """ MATCH (start:Node { handle_id:{start_id} }),(end:Node {handle_id: {end_id} }) CREATE UNIQUE (start)-[r:%s { props } ]->(end) """ % item.get('type') query_data = { 'props': properties, 'start_id': item.get('start'), 'end_id': item.get('end') } t.run(q, query_data) logger.info('{start}-[{rel_type}]->{end}'.format(start=item.get('start'), rel_type=item.get('type'), end=item.get('end'))) x += 1 if x >= 1000: t.commit() x = 0 tot_rels = tot_items - tot_nodes print('Added {!s} relationships.'.format(tot_rels))
def _consume_node(item): try: properties = item.get('properties') node_name = properties.get('name') handle_id = item.get('handle_id') node_type = item.get('node_type') meta_type = item.get('meta_type') # Get a node handle nh = restore_node(handle_id, node_name, node_type, meta_type) nc.set_node_properties(nc.graphdb.manager, nh.handle_id, properties) logger.info('Added node {handle_id}.'.format(handle_id=handle_id)) except Exception as e: import traceback traceback.print_exc() ex_type = type(e).__name__ logger.error( 'Could not add node {} (handle_id={}, node_type={}, meta_type={}) got {}: {})' .format(node_name, handle_id, node_type, meta_type, ex_type, str(e)))
def update_noclook_auto_manage(item): """ Updates the noclook_auto_manage and noclook_last_seen properties. If noclook_auto_manage is not set, it is set to True. :param item: norduclient model :return: None """ auto_manage_data = {} auto_manage = item.data.get('noclook_auto_manage', None) if auto_manage or auto_manage is None: auto_manage_data['noclook_auto_manage'] = True auto_manage_data['noclook_last_seen'] = datetime.now().isoformat() if isinstance(item, nc.models.BaseNodeModel): node = nc.get_node_model(nc.graphdb.manager, item.handle_id) node.data.update(auto_manage_data) nc.set_node_properties(nc.graphdb.manager, node.handle_id, node.data) elif isinstance(item, nc.models.BaseRelationshipModel): relationship = nc.get_relationship_model(nc.graphdb.manager, item.id) relationship.data.update(auto_manage_data) nc.set_relationship_properties(nc.graphdb.manager, relationship.id, relationship.data)
def set_noclook_auto_manage(item, auto_manage): """ Sets the node or relationship noclook_auto_manage flag to True or False. Also sets the noclook_last_seen flag to now. :param item: norduclient model :param auto_manage: boolean :return: None """ auto_manage_data = { 'noclook_auto_manage': auto_manage, 'noclook_last_seen': datetime.now().isoformat() } if isinstance(item, nc.models.BaseNodeModel): node = nc.get_node_model(nc.graphdb.manager, item.handle_id) node.data.update(auto_manage_data) nc.set_node_properties(nc.graphdb.manager, node.handle_id, node.data) elif isinstance(item, nc.models.BaseRelationshipModel): relationship = nc.get_relationship_model(nc.graphdb.manager, item.id) relationship.data.update(auto_manage_data) nc.set_relationship_properties(nc.graphdb.manager, relationship.id, relationship.data)
def insert_graph_data(json_list): with nc.neo4jdb.transaction as w: w.execute('CREATE CONSTRAINT ON (n:Node) ASSERT n.old_node_id IS UNIQUE').fetchall() # Loop through all files starting with node for i in json_list: if i['host']['name'].startswith('node'): item = i['host']['noclook_producer'] properties = item.get('properties') node_name = properties.get('name') handle_id = properties.get('handle_id') node_type = properties.get('node_type') meta_type = item.get('meta_type').capitalize() # Labels are nicer capitalized # Get a node handle nh = restore_node(handle_id, node_name, node_type, meta_type) # We need the old node id to create relationships, this will be removed later. properties['old_node_id'] = item.get('id') # Add all properties node = nc.set_node_properties(nc.neo4jdb, nh.handle_id, properties) try: print u'Added node {meta_type} {node_type} {name} with handle ID: {handle_id}'.format( name=node['name'], node_type=node_type, meta_type=meta_type, handle_id=node['handle_id']) except KeyError as e: print e print 'Failed at handle ID: {handle_id}'.format(handle_id=nh.handle_id) sys.exit(1) except UnicodeEncodeError as e: print "Added node", node # Loop through all files starting with relationship x = 0 with nc.neo4jdb.write as w: for i in json_list: if i['host']['name'].startswith('relationship'): item = i['host']['noclook_producer'] properties = item.get('properties') props = {'props': properties} q = """ MATCH (start:Node { old_node_id:{start_id} }),(end:Node {old_node_id: {end_id} }) CREATE UNIQUE (start)-[r:%s { props } ]->(end) RETURN start.name, type(r), end.name """ % item.get('type') w.execute(q, start_id=item.get('start'), end_id=item.get('end'), **props).fetchall() print '{start} -[{rel_type}]-> {end}'.format(start=item.get('start'), rel_type=item.get('type'), end=item.get('end')) x += 1 if x >= 1000: w.connection.commit() x = 0 # Remove the 'old_node_id' and 'node_type' property from all nodes q = """ MATCH (n:Node) REMOVE n.old_node_id REMOVE n.node_type """ with nc.neo4jdb.write as w: w.execute(q).fetchall() w.connection.commit() w.execute('DROP CONSTRAINT ON (n:Node) ASSERT n.old_node_id IS UNIQUE').fetchall()