def relate_resources(self, relationship, legacyid_to_entityid, archesjson):
start_date = None if relationship['START_DATE'] in (
'', 'None') else relationship['START_DATE']
end_date = None if relationship['END_DATE'] in (
'', 'None') else relationship['END_DATE']
if archesjson == False:
relationshiptype_concept = Concept.objects.get(
legacyoid=relationship['RELATION_TYPE'])
concept_value = Value.objects.filter(
concept=relationshiptype_concept.conceptid).filter(
valuetype='prefLabel')
entityid1 = legacyid_to_entityid[relationship['RESOURCEID_FROM']]
entityid2 = legacyid_to_entityid[relationship['RESOURCEID_TO']]
else:
concept_value = Value.objects.filter(
valueid=relationship['RELATION_TYPE'])
entityid1 = relationship['RESOURCEID_FROM']
entityid2 = relationship['RESOURCEID_TO']
related_resource_record = ResourceXResource(
entityid1=entityid1,
entityid2=entityid2,
notes=relationship['NOTES'],
relationshiptype=concept_value[0].valueid,
datestarted=start_date,
dateended=end_date,
)
related_resource_record.save()
self.se.index_data(index='resource_relations',
doc_type='all',
body=model_to_dict(related_resource_record),
idfield='resourcexid')
def import_relations(self, relations=None):
def get_resourceid_from_legacyid(legacyid):
ret = Resource.objects.filter(legacyid=legacyid)
if len(ret) > 1 or len(ret) == 0:
return None
else:
return ret[0].resourceinstanceid
for relation_count, relation in enumerate(relations):
relation_count = relation_count + 2
if relation_count % 500 == 0:
print("{0} relations saved".format(str(relation_count)))
def validate_resourceinstanceid(resourceinstanceid, key):
# Test if resourceinstancefrom is a uuid it is for a resource or if it is not a uuid that get_resourceid_from_legacyid found a resourceid.
try:
# Test if resourceinstanceid from relations file is a UUID.
newresourceinstanceid = uuid.UUID(resourceinstanceid)
try:
# If resourceinstanceid is a UUID then test that it is assoicated with a resource instance
Resource.objects.get(resourceinstanceid=resourceinstanceid)
except:
# If resourceinstanceid is not associated with a resource instance then set resourceinstanceid to None
newresourceinstanceid = None
except:
# If resourceinstanceid is not UUID then assume it's a legacyid and pass it into get_resourceid_from_legacyid function
newresourceinstanceid = get_resourceid_from_legacyid(resourceinstanceid)
# If resourceinstancefrom is None then either:
# 1.) a legacyid was passed in and get_resourceid_from_legacyid could not find a resource or found multiple resources with the indicated legacyid or
# 2.) a uuid was passed in and it is not associated with a resource instance
if newresourceinstanceid is None:
errors = []
# self.errors.append({'datatype':'legacyid', 'value':relation[key], 'source':'', 'message':'either multiple resources or no resource have this legacyid\n'})
errors.append(
{
"type": "ERROR",
"message": "Relation not created, either zero or multiple resources found with legacyid: {0}".format(
relation[key]
),
}
)
if len(errors) > 0:
self.errors += errors
return newresourceinstanceid
resourceinstancefrom = validate_resourceinstanceid(relation["resourceinstanceidfrom"], "resourceinstanceidfrom")
resourceinstanceto = validate_resourceinstanceid(relation["resourceinstanceidto"], "resourceinstanceidto")
if relation["datestarted"] == "" or relation["datestarted"] == "None":
relation["datestarted"] = None
if relation["dateended"] == "" or relation["dateended"] == "None":
relation["dateended"] = None
if resourceinstancefrom is not None and resourceinstanceto is not None:
relation = ResourceXResource(
resourceinstanceidfrom=Resource(resourceinstancefrom),
resourceinstanceidto=Resource(resourceinstanceto),
relationshiptype=str(relation["relationshiptype"]),
datestarted=relation["datestarted"],
dateended=relation["dateended"],
notes=relation["notes"],
)
relation.save()
self.report_errors()
def import_business_data(self, business_data, mapping=None):
reporter = ResourceImportReporter(business_data)
try:
if mapping == None or mapping == '':
for resource in business_data['resources']:
if resource['resourceinstance'] != None:
resourceinstance, created = ResourceInstance.objects.update_or_create(
resourceinstanceid=uuid.UUID(
str(resource['resourceinstance']
['resourceinstanceid'])),
graph_id=uuid.UUID(
str(resource['resourceinstance']['graph_id'])),
legacyid=resource['resourceinstance']['legacyid'])
if len(
ResourceInstance.objects.filter(
resourceinstanceid=resource[
'resourceinstance']
['resourceinstanceid'])) == 1:
reporter.update_resources_saved()
if resource['tiles'] != []:
reporter.update_tiles(len(resource['tiles']))
for tile in resource['tiles']:
tile['parenttile_id'] = uuid.UUID(
str(tile['parenttile_id'])
) if tile['parenttile_id'] else None
tile, created = Tile.objects.update_or_create(
resourceinstance=resourceinstance,
parenttile=Tile(
uuid.UUID(str(tile['parenttile_id'])))
if tile['parenttile_id'] else None,
nodegroup=NodeGroup(
uuid.UUID(str(tile['nodegroup_id'])))
if tile['nodegroup_id'] else None,
tileid=uuid.UUID(str(tile['tileid'])),
data=tile['data'])
if len(Tile.objects.filter(
tileid=tile.tileid)) == 1:
reporter.update_tiles_saved()
for relation in business_data['relations']:
resource_x_resource_relation = ResourceXResource(
resourcexid=str(uuid.UUID(str(
relation['resourcexid']))),
resourceinstanceidfrom=ResourceInstance(
str(relation['resourceinstanceidfrom_id'])),
resourceinstanceidto=ResourceInstance(
str(relation['resourceinstanceidto_id'])),
relationshiptype=Value(
uuid.UUID(str(relation['relationshiptype_id']))),
datestarted=relation['datestarted'],
dateended=relation['dateended'],
notes=relation['notes'])
resource_x_resource_relation.save()
if len(
ResourceXResource.objects.filter(
resourcexid=relation['resourcexid'])) == 1:
reporter.update_relations_saved()
else:
blanktilecache = {}
target_nodegroup_cardinalities = {}
for nodegroup in JSONSerializer().serializeToPython(
NodeGroup.objects.all()):
target_nodegroup_cardinalities[
nodegroup['nodegroupid']] = nodegroup['cardinality']
def replace_source_nodeid(tiles, mapping):
for tile in tiles:
new_data = []
for sourcekey in tile['data'].keys():
for row in mapping['nodes']:
if row['file_field_name'] == sourcekey:
d = {}
d[row['arches_nodeid']] = tile['data'][
sourcekey]
new_data.append(d)
# tile['data'][row['targetnodeid']] = tile['data'][sourcekey]
# del tile['data'][sourcekey]
tile['data'] = new_data
return tiles
def cache(blank_tile):
if blank_tile.data != {}:
for tile in blank_tile.tiles.values():
if isinstance(tile, Tile):
for key in tile.data.keys():
blanktilecache[key] = blank_tile
# else:
# print blank_tile
for resource in business_data['resources']:
reporter.update_tiles(len(resource['tiles']))
parenttileids = []
populated_tiles = []
resourceinstanceid = uuid.uuid4()
populated_nodegroups = []
target_resource_model = mapping['resource_model_id']
for tile in resource['tiles']:
if tile['data'] != {}:
def get_tiles(tile):
if tile['parenttile_id'] != None:
if tile['parenttile_id'] not in parenttileids:
parenttileids.append(
tile['parenttile_id'])
ret = []
for sibling_tile in resource['tiles']:
if sibling_tile[
'parenttile_id'] == tile[
'parenttile_id']:
ret.append(sibling_tile)
else:
ret = None
else:
ret = [tile]
#deletes nodes that don't have values
if ret is not None:
for tile in ret:
for key, value in tile[
'data'].iteritems():
if value == "":
del tile['data'][key]
return ret
def get_blank_tile(sourcetilegroup):
if len(sourcetilegroup[0]['data']) > 0:
if sourcetilegroup[0]['data'][0] != {}:
if sourcetilegroup[0]['data'][0].keys(
)[0] not in blanktilecache:
blank_tile = Tile.get_blank_tile(
tiles[0]['data'][0].keys()[0],
resourceid=resourceinstanceid)
cache(blank_tile)
else:
blank_tile = blanktilecache[
tiles[0]['data'][0].keys()[0]]
else:
blank_tile = None
else:
blank_tile = None
return blank_tile
tiles = get_tiles(tile)
if tiles is not None:
mapped_tiles = replace_source_nodeid(
tiles, mapping)
blank_tile = get_blank_tile(tiles)
def populate_tile(sourcetilegroup,
target_tile):
need_new_tile = False
target_tile_cardinality = target_nodegroup_cardinalities[
str(target_tile.nodegroup_id)]
if str(target_tile.nodegroup_id
) not in populated_nodegroups:
if target_tile.data != {}:
for source_tile in sourcetilegroup:
for tiledata in source_tile[
'data']:
for nodeid in tiledata.keys(
):
if nodeid in target_tile.data:
if target_tile.data[
nodeid] == None:
target_tile.data[
nodeid] = tiledata[
nodeid]
for key in tiledata.keys(
):
if key == nodeid:
del tiledata[
nodeid]
for tiledata in source_tile[
'data']:
if tiledata == {}:
source_tile[
'data'].remove(
tiledata)
elif target_tile.tiles != None:
populated_child_nodegroups = []
for nodegroupid, childtile in target_tile.tiles.iteritems(
):
childtile_empty = True
child_tile_cardinality = target_nodegroup_cardinalities[
str(childtile[0].
nodegroup_id)]
if str(
childtile[0].
nodegroup_id
) not in populated_child_nodegroups:
prototype_tile = childtile.pop(
)
prototype_tile.tileid = None
for source_tile in sourcetilegroup:
if prototype_tile.nodegroup_id not in populated_child_nodegroups:
prototype_tile_copy = deepcopy(
prototype_tile)
for data in source_tile[
'data']:
for nodeid in data.keys(
):
if nodeid in prototype_tile.data.keys(
):
if prototype_tile.data[
nodeid] == None:
prototype_tile_copy.data[
nodeid] = data[
nodeid]
for key in data.keys(
):
if key == nodeid:
del data[
nodeid]
if child_tile_cardinality == '1':
populated_child_nodegroups.append(
prototype_tile
.
nodegroup_id
)
for data in source_tile[
'data']:
if data == {}:
source_tile[
'data'].remove(
data
)
for key in prototype_tile_copy.data.keys(
):
if prototype_tile_copy.data[
key] != None:
childtile_empty = False
if prototype_tile_copy.data == {} or childtile_empty:
prototype_tile_copy = None
if prototype_tile_copy is not None:
childtile.append(
prototype_tile_copy
)
else:
break
if target_tile.data:
if target_tile.data == {} and target_tile.tiles == {}:
target_tile = None
populated_tiles.append(target_tile)
for source_tile in sourcetilegroup:
if source_tile['data']:
for data in source_tile[
'data']:
if len(data) > 0:
need_new_tile = True
if need_new_tile:
if get_blank_tile(
sourcetilegroup) != None:
populate_tile(
sourcetilegroup,
get_blank_tile(
sourcetilegroup))
if target_tile_cardinality == '1':
populated_nodegroups.append(
str(target_tile.nodegroup_id))
else:
target_tile = None
if blank_tile != None:
populate_tile(mapped_tiles, blank_tile)
newresourceinstance = ResourceInstance.objects.create(
resourceinstanceid=resourceinstanceid,
graph_id=target_resource_model,
legacyid=None)
if len(
ResourceInstance.objects.filter(
resourceinstanceid=resourceinstanceid)) == 1:
reporter.update_resources_saved()
# print JSONSerializer().serialize(populated_tiles)
for populated_tile in populated_tiles:
populated_tile.resourceinstance = newresourceinstance
saved_tile = populated_tile.save()
# tile_saved = count parent tile and count of tile array if tile array != {}
# reporter.update_tiles_saved(tile_saved)
except (KeyError, TypeError) as e:
print e
finally:
reporter.report_results()
