def EntityTypes(request, entitytypeid): entityschema = [] if entitytypeid == '': return HttpResponse(JSONSerializer().serialize({}, ensure_ascii=True, indent=4)) else: if request.GET.get('f') is None: return render(request, 'graph.htm', {}) else: entityschema = {entitytypeid: Entity.get_mapping_schema(entitytypeid)} if request.GET.get('f') == 'json': return HttpResponse(JSONSerializer().serialize(entityschema, ensure_ascii=True, indent=4), content_type='application/json') if request.GET.get('f') == 'd3': d3Schema = d3Obj() d3Schema.name = entitytypeid for assestAttr in entityschema[entitytypeid]: d3ObjAssestAttr = d3Obj() d3ObjAssestAttr.name = assestAttr for step in entityschema[entitytypeid][assestAttr]['steps']: d3ObjStep = d3Obj() d3ObjStep.name = step['entitytypedomain'] + ' ' + step['propertyid'] + ' ' + step['entitytyperange'] d3ObjAssestAttr.children.append(d3ObjStep) d3Schema.children.append(d3ObjAssestAttr) return HttpResponse(JSONSerializer().serialize(d3Schema, ensure_ascii=True, indent=4))
def EntityTypes(request, entitytypeid): entityschema = [] if entitytypeid == '': return HttpResponse(JSONSerializer().serialize({}, ensure_ascii=True, indent=4)) else: if request.GET.get('f') is None: return render_to_response('graph.htm', {}, context_instance=RequestContext(request)) else: entityschema = {entitytypeid: Entity.get_mapping_schema(entitytypeid)} if request.GET.get('f') == 'json': return HttpResponse(JSONSerializer().serialize(entityschema, ensure_ascii=True, indent=4), content_type='application/json') if request.GET.get('f') == 'd3': d3Schema = d3Obj() d3Schema.name = entitytypeid for assestAttr in entityschema[entitytypeid]: d3ObjAssestAttr = d3Obj() d3ObjAssestAttr.name = assestAttr for step in entityschema[entitytypeid][assestAttr]['steps']: d3ObjStep = d3Obj() d3ObjStep.name = step['entitytypedomain'] + ' ' + step['propertyid'] + ' ' + step['entitytyperange'] d3ObjAssestAttr.children.append(d3ObjStep) d3Schema.children.append(d3ObjAssestAttr) return HttpResponse(JSONSerializer().serialize(d3Schema, ensure_ascii=True, indent=4))
def update_nodes(self, entitytypeid, data): self.resource.prune(entitytypes=[entitytypeid]) if self.schema == None: self.schema = Entity.get_mapping_schema(self.resource.entitytypeid) for value in data[entitytypeid]: baseentity = None for newentity in value['nodes']: entity = Entity() if newentity['entitytypeid'] in self.schema: entity.create_from_mapping(self.resource.entitytypeid, self.schema[newentity['entitytypeid']]['steps'], newentity['entitytypeid'], newentity['value'], newentity['entityid']) if baseentity == None: baseentity = entity else: baseentity.merge(entity) if entitytypeid == 'COMPONENT.E18': production_entities = self.resource.find_entities_by_type_id('PRODUCTION.E12') if len(production_entities) > 0: self.resource.merge_at(baseentity, 'PRODUCTION.E12') else: self.resource.merge_at(baseentity, self.resource.entitytypeid) else: self.resource.merge_at(baseentity, self.resource.entitytypeid) self.resource.trim()
def schema(self): try: return self._schema except AttributeError: self._schema = Entity.get_mapping_schema( self.resource.entitytypeid) return self._schema
def insert_actors(settings=None): if not settings: from django.conf import settings logging.warning("INSERTING ACTORS") resource_entity_type = 'HERITAGE_RESOURCE_GROUP.E27' mapping_schema = Entity.get_mapping_schema(resource_entity_type) # access settings to determine which actor nodes should correspond to editors of which pre-existing nodes for entry in settings.ACTOR_NODES: # find all entities of the parent type actor_entitytypeid = entry[0] parent_entitytypeid = entry[1] source_entitytypeid = entry[2] mapping_step_to_actor = mapping_schema[actor_entitytypeid]['steps'][-1] parent_entities = models.Entities.objects.filter( entitytypeid=parent_entitytypeid).iterator() for parent_entity_model in parent_entities: # check whether an actor node already exists parent_entity = Entity().get(parent_entity_model.entityid) actors = parent_entity.find_entities_by_type_id(actor_entitytypeid) if (len(actors) == 0): # get the root resource root_resource_model = get_resource_for_entity( parent_entity_model, resource_entity_type) if not root_resource_model: continue # find the last edit to the node that the data originated at edits = models.EditLog.objects.filter( resourceid=root_resource_model.entityid, attributeentitytypeid=source_entitytypeid).order_by( 'timestamp') first_edit = edits[0] actor_name = '%s %s' % (edits[0].user_firstname, edits[0].user_lastname) # create the actor node parent_entity.add_child_entity( actor_entitytypeid, mapping_step_to_actor['propertyid'], actor_name, '') # logging.warning("\n\nParent after insert") log_entity(parent_entity) parent_entity._save() root_resource = Resource() root_resource.get(root_resource_model.entityid)
def update_nodes(self, entitytypeid, data, dataKey=None): if dataKey == None: dataKey = entitytypeid self.resource.prune(entitytypes=[entitytypeid]) if self.schema == None: self.schema = Entity.get_mapping_schema(self.resource.entitytypeid) for value in data[entitytypeid]: baseentity = None for newentity in value['nodes']: if type(newentity) is list: for newentityitem in newentity: baseentitygroup = None for newsubentity in newentityitem: entity = Entity() if newsubentity['entitytypeid'] in self.schema: entity.create_from_mapping( self.resource.entitytypeid, self.schema[ newsubentity['entitytypeid']]['steps'], newsubentity['entitytypeid'], newsubentity['value'], '') if baseentitygroup == None: baseentitygroup = entity else: baseentitygroup.merge(entity) if baseentity == None: baseentity = baseentitygroup else: baseentity.merge_at(baseentitygroup, entitytypeid) else: entity = Entity() if newentity['entitytypeid'] in self.schema: entity.create_from_mapping( self.resource.entitytypeid, self.schema[newentity['entitytypeid']]['steps'], newentity['entitytypeid'], newentity['value'], '') if baseentity == None: baseentity = entity else: baseentity.merge(entity) self.resource.merge_at(baseentity, self.resource.entitytypeid) self.resource.trim()
def update_nodes(self, entitytypeid, data, dataKey=None): if dataKey == None: dataKey = entitytypeid self.resource.prune(entitytypes=[entitytypeid]) if self.schema == None: self.schema = Entity.get_mapping_schema(self.resource.entitytypeid) for value in data[entitytypeid]: baseentity = None for newentity in value['nodes']: entity = Entity() if newentity['entitytypeid'] in self.schema: entity.create_from_mapping(self.resource.entitytypeid, self.schema[newentity['entitytypeid']]['steps'], newentity['entitytypeid'], newentity['value'], newentity['entityid']) if baseentity == None: baseentity = entity else: baseentity.merge(entity) self.resource.merge_at(baseentity, self.resource.entitytypeid) self.resource.trim()
def schema(self): try: return self._schema except AttributeError: self._schema = Entity.get_mapping_schema(self.resource.entitytypeid) return self._schema
def migrate(settings=None): if not settings: from django.conf import settings suffix = '_altered_nodes.csv' errors = [] for path in settings.ADDITIONAL_RESOURCE_GRAPH_LOCATIONS: if os.path.exists(path): print '\nLOADING NODE MIGRATION INFO (%s)' % (path) print '--------------' for f in listdir(path): if isfile(join(path, f)) and f.endswith(suffix): path_to_file = join(path, f) basepath = path_to_file[:-18] name = basepath.split(os.sep)[-1] migrations = get_list_dict( basepath + '_altered_nodes.csv', [ 'OLDENTITYTYPEID', 'NEWENTITYTYPEID', 'GROUPROOTNODEOLD', 'GROUPROOTNODENEW' ]) # Identify nodes which must be migrated resource_entity_type = 'HERITAGE_RESOURCE_GROUP.E27' mapping_schema = Entity.get_mapping_schema( resource_entity_type) # group migrations by groupRootNodeNew groups = groupby( migrations, lambda x: (x['GROUPROOTNODEOLD'], x['GROUPROOTNODENEW'])) for group_root_node_ids, group_migrations in groups: #Convert group_migrations to a list as we need to iterate it multiple times group_migrations_list = [] for group_migration in group_migrations: group_migrations_list.append(group_migration) group_root_node_id = group_root_node_ids[0] new_group_root_node_id = group_root_node_ids[1] #Find all entities with the old group root node group_root_entities = models.Entities.objects.filter( entitytypeid=group_root_node_id) print "ENTITIES COUNT: ", group_root_entities.count() for group_root_entity_model in group_root_entities.iterator( ): # Create a new subgraph for each of the migration steps, then merge them together at the group root node #get full resource graph for the root entity try: group_root_entity = Entity( group_root_entity_model.pk) except: print "Faulty group entity's ID %s and entitytype %s" % ( group_root_entity_model.pk, group_root_entity_model.entitytypeid) continue new_group_root_entity = Entity( ).create_from_mapping( resource_entity_type, mapping_schema[new_group_root_node_id] ['steps'], new_group_root_node_id, '') if group_migrations_list[0][ 'NEWENTITYTYPEID'] != new_group_root_node_id: # create a node for the new group root group_root_is_new_data_node = False else: group_root_is_new_data_node = True # get the root resource graph for this entity resource_model = get_resource_for_entity( group_root_entity, resource_entity_type) if not resource_model: continue resource = Resource().get(resource_model.entityid) for group_migration in group_migrations_list: # get individual entities to be migrated in the source group old_entities = group_root_entity.find_entities_by_type_id( group_migration['OLDENTITYTYPEID']) for old_entity in old_entities: date_on = False # Create the corresponding entity in the new schema new_entity = Entity() #Disturbance dates need to be mapped to different nodes depending on the value of the now obsolete DISTURBANCE_DATE_TYPE.E55 if group_migration['OLDENTITYTYPEID'] in [ 'DISTURBANCE_DATE_END.E49', 'DISTURBANCE_DATE_START.E49' ]: date_type_node = group_root_entity.find_entities_by_type_id( 'DISTURBANCE_DATE_TYPE.E55') if date_type_node: if date_type_node[ 0].label == 'Occurred before': new_entity_type_id = 'DISTURBANCE_DATE_OCCURRED_BEFORE.E61' elif date_type_node[ 0].label == 'Occurred on': if group_migration[ 'OLDENTITYTYPEID'] == 'DISTURBANCE_DATE_START.E49': date_on = True else: new_entity_type_id = 'DISTURBANCE_DATE_OCCURRED_ON.E61' else: new_entity_type_id = group_migration[ 'NEWENTITYTYPEID'] else: new_entity_type_id = group_migration[ 'NEWENTITYTYPEID'] old_value = old_entity.value if old_entity.businesstablename == 'domains': # in some cases we move from domains to strings. newEntityType = models.EntityTypes.objects.get( entitytypeid=new_entity_type_id) if newEntityType.businesstablename == 'strings': old_value = old_entity.label if not date_on: new_entity.create_from_mapping( resource_entity_type, mapping_schema[new_entity_type_id] ['steps'], new_entity_type_id, old_value) # In some cases a newly created data node is the new group root. In this case we should discard the previously created new group root and use this one instead. if new_group_root_node_id == new_entity_type_id: new_group_root_entity = new_entity group_root_is_new_data_node = True # UNUSED # # If there is a node to be inserted, do it here # # if 'INSERT_NODE_RULE' in group_migration: # # entityttypeid_to_insert = group_migration['INSERT_NODE_RULE'][1][1] # # value_to_insert = group_migration['INSERT_NODE_RULE'][1][2] # # # # inserted_entity = Entity() # # inserted_entity.create_from_mapping(resource_entity_type, mapping_schema[entityttypeid_to_insert]['steps'], entityttypeid_to_insert, value_to_insert) # # # # new_entity.merge(inserted_entity) # If there is a node in common with the existing node further down the chain than the group root node, merge there # follow links back from the parent shouldnt_merge_with_group_root = group_root_is_new_data_node and new_group_root_node_id == new_entity_type_id if not shouldnt_merge_with_group_root: has_merged = False reversed_steps = mapping_schema[ new_entity_type_id]['steps'][::-1] for step in reversed_steps: # find the entitytypedomain in the new_group_root_entity if not has_merged: mergeable_nodes = new_group_root_entity.find_entities_by_type_id( step['entitytypedomain']) if len(mergeable_nodes) > 0: new_group_root_entity.merge_at( new_entity, step[ 'entitytypedomain'] ) has_merged = True new_entity = None # gc.collect() if not has_merged: logging.warning( "Unable to merge newly created entity" ) # merge the new group root entity into the resource resource.merge_at(new_group_root_entity, resource_entity_type) logging.warning("SAVING RESOURCE, %s", resource) # save the resource resource.trim() try: resource._save() resource = None except Exception as e: logging.warning("Error saving resource") logging.warning(e) errors.append("Error saving %s. Error was %s" % (resource, e)) group_root_entity.clear() group_root_entity = None new_group_root_entity.clear() new_group_root_entity = None # end for group root # resource.index() # logging.warning("SAVED RESOURCE, %s", resource) utils.write_to_file( os.path.join(settings.PACKAGE_ROOT, 'logs', 'migration_errors.txt'), '') if len(errors) > 0: # utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'migration_errors.txt'), '\n'.join(errors)) print "\n\nERROR: There were errors migrating some resources. See below" print errors