def relate_resources(self, relationship, legacyid_to_entityid, archesjson): start_date = None if relationship['START_DATE'] in ( '', 'None') else relationship['START_DATE'] end_date = None if relationship['END_DATE'] in ( '', 'None') else relationship['END_DATE'] if archesjson == False: relationshiptype_concept = Concept.objects.get( legacyoid=relationship['RELATION_TYPE']) concept_value = Value.objects.filter( concept=relationshiptype_concept.conceptid).filter( valuetype='prefLabel') entityid1 = legacyid_to_entityid[relationship['RESOURCEID_FROM']] entityid2 = legacyid_to_entityid[relationship['RESOURCEID_TO']] else: concept_value = Value.objects.filter( valueid=relationship['RELATION_TYPE']) entityid1 = relationship['RESOURCEID_FROM'] entityid2 = relationship['RESOURCEID_TO'] related_resource_record = ResourceXResource( entityid1=entityid1, entityid2=entityid2, notes=relationship['NOTES'], relationshiptype=concept_value[0].valueid, datestarted=start_date, dateended=end_date, ) related_resource_record.save() self.se.index_data(index='resource_relations', doc_type='all', body=model_to_dict(related_resource_record), idfield='resourcexid')
def import_relations(self, relations=None): def get_resourceid_from_legacyid(legacyid): ret = Resource.objects.filter(legacyid=legacyid) if len(ret) > 1 or len(ret) == 0: return None else: return ret[0].resourceinstanceid for relation_count, relation in enumerate(relations): relation_count = relation_count + 2 if relation_count % 500 == 0: print("{0} relations saved".format(str(relation_count))) def validate_resourceinstanceid(resourceinstanceid, key): # Test if resourceinstancefrom is a uuid it is for a resource or if it is not a uuid that get_resourceid_from_legacyid found a resourceid. try: # Test if resourceinstanceid from relations file is a UUID. newresourceinstanceid = uuid.UUID(resourceinstanceid) try: # If resourceinstanceid is a UUID then test that it is assoicated with a resource instance Resource.objects.get(resourceinstanceid=resourceinstanceid) except: # If resourceinstanceid is not associated with a resource instance then set resourceinstanceid to None newresourceinstanceid = None except: # If resourceinstanceid is not UUID then assume it's a legacyid and pass it into get_resourceid_from_legacyid function newresourceinstanceid = get_resourceid_from_legacyid(resourceinstanceid) # If resourceinstancefrom is None then either: # 1.) a legacyid was passed in and get_resourceid_from_legacyid could not find a resource or found multiple resources with the indicated legacyid or # 2.) a uuid was passed in and it is not associated with a resource instance if newresourceinstanceid is None: errors = [] # self.errors.append({'datatype':'legacyid', 'value':relation[key], 'source':'', 'message':'either multiple resources or no resource have this legacyid\n'}) errors.append( { "type": "ERROR", "message": "Relation not created, either zero or multiple resources found with legacyid: {0}".format( relation[key] ), } ) if len(errors) > 0: self.errors += errors return newresourceinstanceid resourceinstancefrom = validate_resourceinstanceid(relation["resourceinstanceidfrom"], "resourceinstanceidfrom") resourceinstanceto = validate_resourceinstanceid(relation["resourceinstanceidto"], "resourceinstanceidto") if relation["datestarted"] == "" or relation["datestarted"] == "None": relation["datestarted"] = None if relation["dateended"] == "" or relation["dateended"] == "None": relation["dateended"] = None if resourceinstancefrom is not None and resourceinstanceto is not None: relation = ResourceXResource( resourceinstanceidfrom=Resource(resourceinstancefrom), resourceinstanceidto=Resource(resourceinstanceto), relationshiptype=str(relation["relationshiptype"]), datestarted=relation["datestarted"], dateended=relation["dateended"], notes=relation["notes"], ) relation.save() self.report_errors()
def import_business_data(self, business_data, mapping=None): reporter = ResourceImportReporter(business_data) try: if mapping == None or mapping == '': for resource in business_data['resources']: if resource['resourceinstance'] != None: resourceinstance, created = ResourceInstance.objects.update_or_create( resourceinstanceid=uuid.UUID( str(resource['resourceinstance'] ['resourceinstanceid'])), graph_id=uuid.UUID( str(resource['resourceinstance']['graph_id'])), legacyid=resource['resourceinstance']['legacyid']) if len( ResourceInstance.objects.filter( resourceinstanceid=resource[ 'resourceinstance'] ['resourceinstanceid'])) == 1: reporter.update_resources_saved() if resource['tiles'] != []: reporter.update_tiles(len(resource['tiles'])) for tile in resource['tiles']: tile['parenttile_id'] = uuid.UUID( str(tile['parenttile_id']) ) if tile['parenttile_id'] else None tile, created = Tile.objects.update_or_create( resourceinstance=resourceinstance, parenttile=Tile( uuid.UUID(str(tile['parenttile_id']))) if tile['parenttile_id'] else None, nodegroup=NodeGroup( uuid.UUID(str(tile['nodegroup_id']))) if tile['nodegroup_id'] else None, tileid=uuid.UUID(str(tile['tileid'])), data=tile['data']) if len(Tile.objects.filter( tileid=tile.tileid)) == 1: reporter.update_tiles_saved() for relation in business_data['relations']: resource_x_resource_relation = ResourceXResource( resourcexid=str(uuid.UUID(str( relation['resourcexid']))), resourceinstanceidfrom=ResourceInstance( str(relation['resourceinstanceidfrom_id'])), resourceinstanceidto=ResourceInstance( str(relation['resourceinstanceidto_id'])), relationshiptype=Value( uuid.UUID(str(relation['relationshiptype_id']))), datestarted=relation['datestarted'], dateended=relation['dateended'], notes=relation['notes']) resource_x_resource_relation.save() if len( ResourceXResource.objects.filter( resourcexid=relation['resourcexid'])) == 1: reporter.update_relations_saved() else: blanktilecache = {} target_nodegroup_cardinalities = {} for nodegroup in JSONSerializer().serializeToPython( NodeGroup.objects.all()): target_nodegroup_cardinalities[ nodegroup['nodegroupid']] = nodegroup['cardinality'] def replace_source_nodeid(tiles, mapping): for tile in tiles: new_data = [] for sourcekey in tile['data'].keys(): for row in mapping['nodes']: if row['file_field_name'] == sourcekey: d = {} d[row['arches_nodeid']] = tile['data'][ sourcekey] new_data.append(d) # tile['data'][row['targetnodeid']] = tile['data'][sourcekey] # del tile['data'][sourcekey] tile['data'] = new_data return tiles def cache(blank_tile): if blank_tile.data != {}: for tile in blank_tile.tiles.values(): if isinstance(tile, Tile): for key in tile.data.keys(): blanktilecache[key] = blank_tile # else: # print blank_tile for resource in business_data['resources']: reporter.update_tiles(len(resource['tiles'])) parenttileids = [] populated_tiles = [] resourceinstanceid = uuid.uuid4() populated_nodegroups = [] target_resource_model = mapping['resource_model_id'] for tile in resource['tiles']: if tile['data'] != {}: def get_tiles(tile): if tile['parenttile_id'] != None: if tile['parenttile_id'] not in parenttileids: parenttileids.append( tile['parenttile_id']) ret = [] for sibling_tile in resource['tiles']: if sibling_tile[ 'parenttile_id'] == tile[ 'parenttile_id']: ret.append(sibling_tile) else: ret = None else: ret = [tile] #deletes nodes that don't have values if ret is not None: for tile in ret: for key, value in tile[ 'data'].iteritems(): if value == "": del tile['data'][key] return ret def get_blank_tile(sourcetilegroup): if len(sourcetilegroup[0]['data']) > 0: if sourcetilegroup[0]['data'][0] != {}: if sourcetilegroup[0]['data'][0].keys( )[0] not in blanktilecache: blank_tile = Tile.get_blank_tile( tiles[0]['data'][0].keys()[0], resourceid=resourceinstanceid) cache(blank_tile) else: blank_tile = blanktilecache[ tiles[0]['data'][0].keys()[0]] else: blank_tile = None else: blank_tile = None return blank_tile tiles = get_tiles(tile) if tiles is not None: mapped_tiles = replace_source_nodeid( tiles, mapping) blank_tile = get_blank_tile(tiles) def populate_tile(sourcetilegroup, target_tile): need_new_tile = False target_tile_cardinality = target_nodegroup_cardinalities[ str(target_tile.nodegroup_id)] if str(target_tile.nodegroup_id ) not in populated_nodegroups: if target_tile.data != {}: for source_tile in sourcetilegroup: for tiledata in source_tile[ 'data']: for nodeid in tiledata.keys( ): if nodeid in target_tile.data: if target_tile.data[ nodeid] == None: target_tile.data[ nodeid] = tiledata[ nodeid] for key in tiledata.keys( ): if key == nodeid: del tiledata[ nodeid] for tiledata in source_tile[ 'data']: if tiledata == {}: source_tile[ 'data'].remove( tiledata) elif target_tile.tiles != None: populated_child_nodegroups = [] for nodegroupid, childtile in target_tile.tiles.iteritems( ): childtile_empty = True child_tile_cardinality = target_nodegroup_cardinalities[ str(childtile[0]. nodegroup_id)] if str( childtile[0]. nodegroup_id ) not in populated_child_nodegroups: prototype_tile = childtile.pop( ) prototype_tile.tileid = None for source_tile in sourcetilegroup: if prototype_tile.nodegroup_id not in populated_child_nodegroups: prototype_tile_copy = deepcopy( prototype_tile) for data in source_tile[ 'data']: for nodeid in data.keys( ): if nodeid in prototype_tile.data.keys( ): if prototype_tile.data[ nodeid] == None: prototype_tile_copy.data[ nodeid] = data[ nodeid] for key in data.keys( ): if key == nodeid: del data[ nodeid] if child_tile_cardinality == '1': populated_child_nodegroups.append( prototype_tile . nodegroup_id ) for data in source_tile[ 'data']: if data == {}: source_tile[ 'data'].remove( data ) for key in prototype_tile_copy.data.keys( ): if prototype_tile_copy.data[ key] != None: childtile_empty = False if prototype_tile_copy.data == {} or childtile_empty: prototype_tile_copy = None if prototype_tile_copy is not None: childtile.append( prototype_tile_copy ) else: break if target_tile.data: if target_tile.data == {} and target_tile.tiles == {}: target_tile = None populated_tiles.append(target_tile) for source_tile in sourcetilegroup: if source_tile['data']: for data in source_tile[ 'data']: if len(data) > 0: need_new_tile = True if need_new_tile: if get_blank_tile( sourcetilegroup) != None: populate_tile( sourcetilegroup, get_blank_tile( sourcetilegroup)) if target_tile_cardinality == '1': populated_nodegroups.append( str(target_tile.nodegroup_id)) else: target_tile = None if blank_tile != None: populate_tile(mapped_tiles, blank_tile) newresourceinstance = ResourceInstance.objects.create( resourceinstanceid=resourceinstanceid, graph_id=target_resource_model, legacyid=None) if len( ResourceInstance.objects.filter( resourceinstanceid=resourceinstanceid)) == 1: reporter.update_resources_saved() # print JSONSerializer().serialize(populated_tiles) for populated_tile in populated_tiles: populated_tile.resourceinstance = newresourceinstance saved_tile = populated_tile.save() # tile_saved = count parent tile and count of tile array if tile array != {} # reporter.update_tiles_saved(tile_saved) except (KeyError, TypeError) as e: print e finally: reporter.report_results()