def fix_organizations_address(self):
self.user = get_user()
address_type = self.get_nodetype(type='Address',
slug='address',
hidden=True)
organization_type = self.get_nodetype(type='Organization',
slug='organization')
all_organizations = NodeHandle.objects.filter(
node_type=organization_type)
logical_meta_type = 'Logical'
phone_field = 'phone'
for organization in all_organizations:
organization_node = organization.get_node()
address_name = 'Address: {}'.format(organization.node_name)
old_phone = organization_node.data.get(phone_field, None)
if old_phone:
# create an Address and asociate it to the Organization
new_address = NodeHandle.objects.get_or_create(
node_name=address_name,
node_type=address_type,
node_meta_type=logical_meta_type,
creator=self.user,
modifier=self.user,
)[0]
self.add_community_context(new_address)
new_address.get_node().add_property(phone_field, old_phone)
organization_node.remove_property(phone_field)
organization_node.add_address(new_address.handle_id)
def fix_website_field(self):
self.user = get_user()
address_type = self.get_nodetype(type='Address',
slug='address',
hidden=True)
organization_type = self.get_nodetype(type='Organization',
slug='organization')
all_organizations = NodeHandle.objects.filter(
node_type=organization_type)
website_field = 'website'
for organization in all_organizations:
orgnode = organization.get_node()
relations = orgnode.get_outgoing_relations()
address_relations = relations.get('Has_address', None)
if address_relations:
for rel in address_relations:
address_end = rel['relationship'].end_node
if website_field in address_end._properties:
website_str = address_end._properties[website_field]
handle_id = address_end._properties['handle_id']
address_node = NodeHandle.objects.get(
handle_id=handle_id).get_node()
# remove if it already exists
orgnode.remove_property(website_field)
orgnode.add_property(website_field, website_str)
# remove value in address_node
address_node.remove_property(website_field)
def __init__(self, seed=None):
locales = OrderedDict([
('en_GB', 1),
('sv_SE', 2),
])
self.fake = Faker(locales)
if seed:
self.fake.seed_instance(seed)
self.user = get_user()
def fix_organizations_fields(self):
self.user = get_user()
organization_type = self.get_nodetype(type='Organization',
slug='organization')
all_organizations = NodeHandle.objects.filter(
node_type=organization_type)
old_field1 = 'customer_id'
new_field1 = 'organization_id'
for organization in all_organizations:
orgnode = organization.get_node()
org_id_val = orgnode.data.get(old_field1, None)
if org_id_val:
orgnode.remove_property(old_field1)
orgnode.add_property(new_field1, org_id_val)
def test_query(self):
from apps.nerds.lib.consumer_util import get_user
from apps.noclook.tests.schema.base import TestContext
# this query should be possible to execute without any privileges
noc_user = get_user()
nonprivileged_context = TestContext(noc_user)
# as the result is a special type we can't query any relationship
query = '''
{
getPlainGroups{
id
name
}
}
'''
result = schema.execute(query, context=nonprivileged_context)
assert not result.errors, pformat(result.errors, indent=1)
plaingroups_data = result.data['getPlainGroups']
# now query as a privileged user that can query community's groups
query = '''
{
all_groups{
id
name
}
}
'''
result = schema.execute(query, context=self.context)
assert not result.errors, pformat(result.errors, indent=1)
regulargroups_data = result.data['all_groups']
self.assertEqual(plaingroups_data, regulargroups_data)
def handle(self, *args, **options):
# check if the fixroles option has been called, do it and exit
if options['fixroles']:
self.fix_roles()
return
# check if the emailphones option has been called, do it and exit
if options['emailphones']:
self.fix_emails_phones()
return
# check if the addressfix option has been called, do it and exit
if options['addressfix']:
self.fix_organizations_address()
return
# check if the movewebsite option has been called, do it and exit
if options['movewebsite']:
self.fix_website_field()
return
# check if the reorgprops option has been called, do it and exit
if options['reorgprops']:
self.fix_organizations_fields()
return
# check if the reorgprops option has been called, do it and exit
if options['contextfix']:
self.fix_community_context()
return
relation_meta_type = 'Relation'
logical_meta_type = 'Logical'
self.delimiter = ';'
if options['delimiter']:
self.delimiter = options['delimiter']
## (We'll use handle_id on to get the node on cql code)
# check if new types exists
if options['verbosity'] > 0:
self.stdout.write('Checking if the types are already in the db')
for type in self.new_types:
dbtype = NodeType.objects.filter(type=type)
if not dbtype:
dbtype = NodeType(
type=type,
slug=type.lower(),
)
dbtype.save()
else:
dbtype = dbtype.first()
total_lines = 0
csv_organizations = None
csv_contacts = None
csv_secroles = None
self.user = get_user()
# IMPORT ORGANIZATIONS
if options['organizations']:
# py: count lines
csv_organizations = options['organizations']
org_lines = self.count_lines(csv_organizations)
if options['verbosity'] > 0:
self.stdout.write('Importing {} Organizations from file "{}"'\
.format(org_lines, csv_organizations.name))
total_lines = total_lines + org_lines
# IMPORT CONTACTS AND ROLES
if options['contacts']:
# py: count lines
csv_contacts = options['contacts']
con_lines = self.count_lines(csv_contacts)
if options['verbosity'] > 0:
self.stdout.write('Importing {} Contacts from file "{}"'\
.format(con_lines, csv_contacts.name))
total_lines = total_lines + con_lines
# IMPORT SECURITY ROLES
if options['secroles']:
# py: count lines
csv_secroles = options['secroles']
srl_lines = self.count_lines(csv_secroles)
if options['verbosity'] > 0:
self.stdout.write('Importing {} Security Roles from file "{}"'\
.format(srl_lines, csv_secroles.name))
total_lines = total_lines + srl_lines
imported_lines = 0
# print progress bar
if options['verbosity'] > 0:
self.printProgressBar(imported_lines, total_lines)
# process organizations
if options['organizations']:
# contact
node_type = NodeType.objects.filter(type=self.new_types[0]).first()
csv_organizations = options['organizations']
node_list = self.read_csv(csv_organizations, delim=self.delimiter)
for node in node_list:
account_name = node['account_name']
# dj: organization exist?: create or get (using just the name)
new_organization = NodeHandle.objects.get_or_create(
node_name=account_name,
node_type=node_type,
node_meta_type=relation_meta_type,
creator=self.user,
modifier=self.user,
)[0]
# n4: add attributes
graph_node = new_organization.get_node()
for key in node.keys():
if key not in ['account_name', 'parent_account'
] and node[key]:
graph_node.add_property(key, node[key])
# dj: if parent organization: create or get (using just the name)
if key == 'parent_account' and node['parent_account']:
parent_org_name = node['parent_account']
parent_organization = NodeHandle.objects.get_or_create(
node_name=parent_org_name,
node_type=node_type,
node_meta_type=relation_meta_type,
creator=self.user,
modifier=self.user,
)[0]
parent_node = parent_organization.get_node()
# n4: add relation between org and parent_org
graph_node.set_parent(parent_organization.handle_id)
# Print iterations progress
if options['verbosity'] > 0:
imported_lines = imported_lines + 1
self.printProgressBar(imported_lines, total_lines)
csv_organizations.close()
# process contacts
if options['contacts']:
node_type = NodeType.objects.filter(
type=self.new_types[2]).first() # contact
node_list = self.read_csv(csv_contacts, delim=self.delimiter)
for node in node_list:
full_name = '{} {}'.format(node['first_name'],
node['last_name'])
# dj: contact exists?: create or get
new_contact = NodeHandle.objects.get_or_create(
node_name=full_name,
node_type=node_type,
node_meta_type=relation_meta_type,
creator=self.user,
modifier=self.user,
)[0]
# n4: add attributes
graph_node = new_contact.get_node()
for key in node.keys():
if key not in [
'node_type', 'contact_role', 'name',
'account_name', 'salutation'
] and node[key]:
graph_node.add_property(key, node[key])
# dj: organization exist?: create or get
organization_name = node.get('account_name', None)
if organization_name:
org_type = NodeType.objects.filter(
type=self.new_types[0]).first() # organization
new_org = NodeHandle.objects.get_or_create(
node_name=organization_name,
node_type=org_type,
node_meta_type=relation_meta_type,
creator=self.user,
modifier=self.user,
)[0]
# add role relatioship, use employee role if empty
role_name = node['contact_role']
if role_name:
role = Role.objects.get_or_create(name=role_name)[0]
else:
role = Role.objects.get(slug=DEFAULT_ROLE_KEY)
nc.models.RoleRelationship.link_contact_organization(
new_contact.handle_id, new_org.handle_id, role.name)
# Print iterations progress
if options['verbosity'] > 0:
imported_lines = imported_lines + 1
self.printProgressBar(imported_lines, total_lines)
csv_contacts.close()
# process security roles
if options['secroles']:
orga_type = NodeType.objects.filter(
type=self.new_types[0]).first() # organization
cont_type = NodeType.objects.filter(
type=self.new_types[2]).first() # contact
role_type = NodeType.objects.filter(
type=self.new_types[4]).first() # role
node_list = self.read_csv(csv_secroles, delim=self.delimiter)
for node in node_list:
# create or get nodes
organization = NodeHandle.objects.get_or_create(
node_name=node['organisation'],
node_type=orga_type,
node_meta_type=relation_meta_type,
creator=self.user,
modifier=self.user,
)[0]
contact = NodeHandle.objects.get_or_create(
node_name=node['contact'],
node_type=cont_type,
node_meta_type=relation_meta_type,
creator=self.user,
modifier=self.user,
)[0]
role_name = node['role']
role = Role.objects.get_or_create(name=role_name)[0]
nc.models.RoleRelationship.link_contact_organization(
contact.handle_id, organization.handle_id, role.name)
csv_secroles.close()
def fix_emails_phones(self):
self.user = get_user()
work_type_str = 'work'
personal_type_str = 'personal'
logical_meta_type = 'Logical'
old_email_fields = {
'email': work_type_str,
'other_email': personal_type_str
}
old_phone_fields = {
'phone': work_type_str,
'mobile': personal_type_str
}
# check that the options are available
phone_type_val = Dropdown.objects.get(
name="phone_type").as_values(False)
email_type_val = Dropdown.objects.get(
name="email_type").as_values(False)
if not ((work_type_str in phone_type_val) and\
(personal_type_str in phone_type_val) and\
(personal_type_str in email_type_val) and\
(personal_type_str in email_type_val)):
raise Exception('Work/Personal values are not available for the \
Email/phone dropdown types')
contact_type = self.get_nodetype(type='Contact', slug='contact')
# set hidden types
email_type = self.get_nodetype(type='Email', slug='email', hidden=True)
phone_type = self.get_nodetype(type='Phone', slug='phone', hidden=True)
all_contacts = NodeHandle.objects.filter(node_type=contact_type)
for contact in all_contacts:
contact_node = contact.get_node()
for old_phone_field, assigned_type in old_phone_fields.items():
# phones
if old_phone_field in contact_node.data:
old_phone_value = contact_node.data.get(old_phone_field)
new_phone = NodeHandle.objects.get_or_create(
node_name=old_phone_value,
node_type=phone_type,
node_meta_type=logical_meta_type,
creator=self.user,
modifier=self.user,
)[0]
self.add_community_context(new_phone)
contact_node.add_phone(new_phone.handle_id)
contact_node.remove_property(old_phone_field)
new_phone.get_node().add_property('type', assigned_type)
for old_email_field, assigned_type in old_email_fields.items():
# emails
if old_email_field in contact_node.data:
old_email_value = contact_node.data.get(old_email_field)
new_email = NodeHandle.objects.get_or_create(
node_name=old_email_value,
node_type=email_type,
node_meta_type=logical_meta_type,
creator=self.user,
modifier=self.user,
)[0]
self.add_community_context(new_email)
contact_node.add_email(new_email.handle_id)
contact_node.remove_property(old_email_field)
new_email.get_node().add_property('type', assigned_type)
def __init__(self):
self.user = get_user()
def backwards_func(apps, schema_editor):
NodeType = apps.get_model('noclook', 'NodeType')
NodeHandle = apps.get_model('noclook', 'NodeHandle')
Dropdown = apps.get_model('noclook', 'Dropdown')
Choice = apps.get_model('noclook', 'Choice')
User = apps.get_model('auth', 'User')
user = get_user(usermodel=User)
# get the values from the old group dropdown
groups_dropname = 'responsible_groups'
groupdropdown, created = \
Dropdown.objects.get_or_create(name=groups_dropname)
choices = Choice.objects.filter(dropdown=groupdropdown)
# get group options
groups_opts_dict = {}
host_type_objs = []
for host_type_str in host_types:
host_type, created = NodeType.objects.get_or_create(
type=host_type_str, slug=slugify(host_type_str))
host_type_objs.append(host_type)
if NodeHandle.objects.filter(node_type__in=host_type_objs).exists():
# fill choice dict
for choice in choices:
choice_name = choice.name
groups_opts_dict[choice_name] = choice
# loop over entity types
for host_type_str in host_types:
host_type, created = NodeType.objects.get_or_create(
type=host_type_str, slug=slugify(host_type_str))
nhs = NodeHandle.objects.filter(node_type=host_type)
# loop over entities of this type
for nh in nhs:
host_node = nc.get_node_model(nc.graphdb.manager, nh.handle_id)
attr_val_dict = {
'responsible_group':
host_node.incoming.get('Takes_responsibility'),
'support_group':
host_node.incoming.get('Supports'),
}
for attr_name, rels in attr_val_dict.items():
if rels:
# unlink group and get its name
relationship = rels[0]
node = relationship['node']
ngroup_name = node.data.get('name', None)
if ngroup_name in groups_opts_dict:
group_choice = groups_opts_dict[ngroup_name]
# add old property value
host_node.add_property(attr_name,
group_choice.value)
# delete relationship anyways
nc.delete_relationship(nc.graphdb.manager,
relationship['relationship_id'])
# delete created groups (both postgresql and neo4j)
group_type, created = NodeType.objects.get_or_create(type='Group',
slug='group')
groups_nhs = NodeHandle.objects.filter(node_type=group_type)
for group_nh in groups_nhs:
q = """
MATCH (n:Group {handle_id:{handle_id}}) DETACH DELETE n
"""
nc.query_to_dict(nc.graphdb.manager, q, handle_id=group_nh.handle_id)
group_nh.delete()
def fix_test_data(self):
self.user = get_user()
organization_type = NodeType.objects.get_or_create(
type='Organization', slug='organization')[0] # organization
contact_type = NodeType.objects.get_or_create(
type='Contact', slug='contact')[0] # organization
# fix all organizations
all_organizations = NodeHandle.objects.filter(node_type=organization_type)
total_lines = all_organizations.count()
current_line = 0
org_types_drop = Dropdown.objects.get(name='organization_types')
org_types = Choice.objects.filter(dropdown=org_types_drop)
first_field = 'type'
second_field = 'organization_id'
organization_ids = {}
# get all the possible organizations id
for organization in all_organizations:
orgnode = organization.get_node()
organization_id = orgnode.data.get(second_field, None)
if organization_id not in organization_ids:
organization_ids[organization_id] = [organization.handle_id]
else:
organization_ids[organization_id].append(organization.handle_id)
total_lines = total_lines + len(organization_ids)
# fix all contacts
all_contacts = NodeHandle.objects.filter(node_type=contact_type)
con_types_drop = Dropdown.objects.get(name='contact_type')
con_types = Choice.objects.filter(dropdown=con_types_drop)
total_lines = total_lines + all_contacts.count()
# fix organization type
for organization in all_organizations:
self.printProgressBar(current_line, total_lines)
orgnode = organization.get_node()
org_type = orgnode.data.get(first_field, None)
# get the first value as default
selected_type = org_types.first()
if org_type:
correct_type = org_types.filter(value=org_type).exists()
if not correct_type:
# check if exist choice with that name
if org_types.filter(name__icontains=org_type).exists():
selected_type = org_types.filter(name__icontains=org_type).first()
elif org_types.filter(value__icontains=org_type).exists():
# if not, check if exists with that value
selected_type = org_types.filter(value__icontains=org_type).first()
else:
selected_type = random.choice(org_types)
orgnode.remove_property(first_field)
orgnode.add_property(first_field, selected_type.value)
current_line = current_line + 1
# fix contact type
con_type_field = 'contact_type'
for contact in all_contacts:
self.printProgressBar(current_line, total_lines)
connode = contact.get_node()
con_type = connode.data.get(con_type_field, None)
if con_type:
correct_type = con_types.filter(value=con_type).exists()
if not correct_type:
# get the first value as default
selected_type = con_types.first()
# check if exist choice with that name
if con_types.filter(name__icontains=con_type).exists():
selected_type = con_types.filter(name__icontains=con_type).first()
elif con_types.filter(value__icontains=org_type).exists():
# if not, check if exists with that value
selected_type = con_types.filter(value__icontains=con_type).first()
connode.remove_property(con_type_field)
connode.add_property(con_type_field, selected_type.value)
current_line = current_line + 1
for organization_id, org_handle_ids in organization_ids.items():
self.printProgressBar(current_line, total_lines)
# check if the array lenght is more than one
if len(org_handle_ids) > 1:
for org_handle_id in org_handle_ids:
# if it is so, iterate and check if the uppercased name of the
# organization is already set as organization_id
organization = NodeHandle.objects.get(handle_id=org_handle_id)
orgnode = organization.get_node()
possible_org_id = organization.node_name.upper()
count = self.checkOrgInUse(possible_org_id)
# if it's alredy in use, append handle_id
if count > 0:
possible_org_id = '{}_{}'.format(
possible_org_id, organization.handle_id
)
orgnode.remove_property(second_field)
orgnode.add_property(second_field, possible_org_id)
current_line = current_line + 1
self.printProgressBar(current_line, total_lines)
def load_nodes(self):
generator = CommunityFakeDataGenerator()
# create organization's file
org_list = []
for i in range(self.organization_num):
organization = generator.create_fake_organization()
organization['organization_number'] = str(i + 1)
str_value = '"{}"'.format('";"'.join(organization.values()))
org_list.append(str_value)
org_str = '{}\n{}'.format(self.org_csv_head, '\n'.join(org_list))
# write string to file
org_file = write_string_to_disk(org_str)
# create contacts's file
con_list = []
for i in range(self.contact_num):
contact = generator.create_fake_contact()
str_value = '"{}"'.format('";"'.join(contact.values()))
con_list.append(str_value)
con_str = '{}\n{}'.format(self.con_csv_head, '\n'.join(con_list))
# write string to file
con_file = write_string_to_disk(con_str)
# import organizations file
call_command(
self.import_cmd,
organizations=org_file,
verbosity=0,
)
# import contacts file
call_command(
self.import_cmd,
contacts=con_file,
verbosity=0,
)
# call data modifiers
call_command(self.import_cmd, emailphones=True, verbosity=0)
call_command(self.import_cmd, addressfix=True, verbosity=0)
call_command(self.import_cmd, movewebsite=True, verbosity=0)
call_command(self.import_cmd, reorgprops=True, verbosity=0)
# create groups and add members
group_list = []
group_type = NodeType.objects.filter(type='Group').first()
contact_type = NodeType.objects.filter(type='Contact').first()
user = get_user()
contact_ids = [
x.handle_id
for x in NodeHandle.objects.filter(node_type=contact_type)
]
for i in range(self.group_num):
group_dict = generator.create_fake_group()
# create group
group = NodeHandle.objects.get_or_create(
node_name=group_dict['name'],
node_type=group_type,
node_meta_type=nc.META_TYPES[1],
creator=user,
modifier=user,
)[0]
group_node = group.get_node()
group_node.add_property('description', group_dict['description'])
# add contacts (get them randomly)
hids = random.sample(
contact_ids, min(len(contact_ids), self.contacts_per_group))
gcontacts = NodeHandle.objects.filter(handle_id__in=hids)
for contact in gcontacts:
set_member_of(user, contact.get_node(), group.handle_id)
# add members to organizations
organization_type = NodeType.objects.filter(
type='Organization').first()
role_ids = [x.handle_id for x in Role.objects.all()]
for organization in NodeHandle.objects.filter(
node_type=organization_type):
hids = random.sample(
contact_ids,
min(len(contact_ids), self.contacts_per_organization))
ocontacts = NodeHandle.objects.filter(handle_id__in=hids)
for contact in ocontacts:
rand_role = Role.objects.get(handle_id=random.choice(role_ids))
set_works_for(user, contact.get_node(), organization.handle_id,
rand_role.name)