def getMatch_LatLong_Threshold(comp_0, comp_1, methodVal = 50000):
"""Gets the separation between two points, and then checks if distance is
smaller than **methodVal**. If Trued, then returns 100, if false, then returns 0
"""
# Initialization
RetVal = 0
# Netz_0 is empty
if comp_0 == '':
pass
# Netz_1 is empty
elif comp_1 == '':
pass
elif comp_0.long == None:
pass
elif comp_1.long == None:
pass
# Both Netze contain components
else:
# Creation of LatLong "vector" from component latlong
latlong_Netz_0 = K_Comp.PolyLine(lat = [comp_0.lat], long = [comp_0.long] ) #M_Netze.get_latlongPairs_Points(comp_0)
thisLatLong = K_Comp.PolyLine(lat = comp_1.lat, long = comp_1.long ) #M_Netze.get_latlongPairs_Points(comp_1)
[pos, minVal] = M_FindPos.find_pos_closestLatLongInList(thisLatLong, latlong_Netz_0)
if math.isnan(minVal):
RetVal = 0
elif minVal <= methodVal:
RetVal = 100
else:
RetVal = 0
# Testig if nan, if so then set to zero
if math.isnan(RetVal) :
RetVal = 0
return RetVal
def get_latlongPairs_Points(Components):
"""Returns lat long valus from **Components**, in format latlong.lat and latlong.long
\n.. comments:
Input:
Components list of elements of a single components
Return:
latlong: latlong.lat and latlong.long are lists of lat and long values
"""
latlong = K_Component.PolyLine(lat=[], long=[])
for comp in Components:
latlong.lat.append(comp.lat)
latlong.long.append(comp.long)
return latlong
def read_component(DataType = '', NumDataSets = 1e+100, RelDirName = None):
""" Method of reading in LKD components from shape files. **RelDirName** supplies the relative location of the shape files, whereas **DataType** specifies which component is to be reaad in with options 'PipeSegments', 'Nodes', 'Storages', and 'Productions'.
\n.. comments:
Input:
self: self
RelDirName: string, containing the relative path name of where data will be loaded from
Default = 'Eingabe/LKD/'
Return:
[]
"""
ReturnComponent = []
inCoord = 'epsg:31468'
outCoord = 'epsg:4326'
count = 0
if DataType in 'PipeSegments':
# start = time.time()
FileName_Shape = str(RelDirName / 'pipelines_utf8.shp')
# Loading from shape file
Shapes = shapefile.Reader(FileName_Shape, encoding = "utf8")
# Malen der Europa Karte
# print('there are pipesegments: ' + str(len(Shapes.shapeRecords())))
for shape in Shapes.shapeRecords():
# Getting PolyLine
parts = sorted(shape.shape.parts)
# Joining X and Y coordinates from Shape.shape.points
vec = shape.shape.points
polyLine = K_Component.PolyLine(lat = [], long = [])
for x,y in vec:
polyLine.long.append(x)
polyLine.lat.append(y)
# Converting to LatLong
polyLine = M_Projection.XY2LatLong(polyLine, inCoord, outCoord)
# Generation of PipeLine
PipeLine = M_Shape.PolyLine2PipeLines(polyLine, parts, source = C_Code, country_code = C_Code)
lat = PipeLine[0].lat
long = PipeLine[0].long
# Getting Meta data
id = str(shape.record[0])
source_id = [ID_Add + str(id)]
name = replaceString(shape.record[1])
if len(name) == 0:
name = 'PS_' + str(id)
# Converting gas_type to boolean
is_H_gas = shape.record[2]
if is_H_gas == 'L':
is_H_gas = 0
elif is_H_gas == 'H':
is_H_gas = 1
length = float(shape.record[3])/1000
pipe_class_type = shape.record[6]
if pipe_class_type == '':
pipe_class_type = None
# is_virtualPipe
is_virtualPipe = False
if len(shape.record[4]) > 0:
if shape.record[4] == 1:
is_virtualPipe = True
# diameter_mm
if len(shape.record[5]) > 0:
if 'NULL' == shape.record[5]:
diameter_mm = float('nan')
else:
diameter_mm = float(shape.record[5])
else:
diameter_mm = float('nan')
# max_pressure_bar
if shape.record[7] == None:
max_pressure_bar = float('nan')
elif type(shape.record[7]) == int:
max_pressure_bar = float(shape.record[7])
if max_pressure_bar > 200:
max_pressure_bar = float('nan')
elif len(shape.record[7]) > 0:
if 'NULL' == shape.record[7]:
max_pressure_bar = float('nan')
else:
max_pressure_bar = float(shape.record[7])
if max_pressure_bar > 200:
max_pressure_bar = float('nan')
else:
max_pressure_bar = float('nan')
diam_est = shape.record[8]
class_est = shape.record[9]
press_est = shape.record[10]
if isinstance(diam_est, str):
if diam_est == 'NULL':
diam_est = float('nan')
diam_est_method = 'raw'
diam_est_uncertainty = 0
else:
diam_est = diam_est
diam_est_method = 'raw'
diam_est_uncertainty = 0
else:
if diam_est == 1:
diam_est_method = 'estimated'
diam_est_uncertainty = 1
else:
diam_est_method = 'raw'
diam_est_uncertainty = 0
if isinstance(class_est, str):
if class_est == 'NULL':
class_est = float('nan')
class_est_method = 'raw'
class_est_uncertainty = 0
else:
class_est = class_est
class_est_method = 'raw'
class_est_uncertainty = 0
else:
if class_est == 1:
class_est_method = 'estimated'
class_est_uncertainty = 1
else:
class_est_method = 'raw'
class_est_uncertainty = 0
if isinstance(press_est, str):
if press_est == 'NULL':
press_est = float('nan')
press_est_method = 'raw'
press_est_uncertainty = 0
else:
press_est_method = 'raw'
press_est_uncertainty = 0
else:
if press_est == 1:
press_est_method = 'estimated'
press_est_uncertainty = 1
else:
press_est_method = 'raw'
press_est_uncertainty = 0
# if isinstance(class_est, str):
# if class_est == 'NULL':
# class_est = float('nan')
# if isinstance(press_est, str):
# if press_est == 'NULL':
# press_est = float('nan')
max_cap_GWh_per_d = shape.record[11]
operator_name = str(shape.record[12])
node_id = ['N_' + str(shape.record[13]), 'N_' + str(shape.record[14])]
if 'N_809066' in node_id and 'N_809063' in node_id:
if node_id[0] == 'N_809066':
node_id[0] = 'N_809076'
else:
node_id[1] = 'N_809076'
if 'N_809066' in node_id and 'N_1000001' in node_id:
if node_id[0] == 'N_809066':
node_id[0] = 'N_809076'
else:
node_id[1] = 'N_809076'
if 'N_809065' in node_id and 'N_809025' in node_id:
if node_id[0] == 'N_809065':
node_id[0] = 'N_809075'
else:
node_id[1] = 'N_809075'
if 'N_809065' in node_id and 'N_1000001' in node_id:
if node_id[0] == 'N_809065':
node_id[0] = 'N_809075'
else:
node_id[1] = 'N_809075'
if 'N_809064' in node_id and 'N_809026' in node_id:
if node_id[0] == 'N_809064':
node_id[0] = 'N_809074'
else:
node_id[1] = 'N_809074'
if 'N_809064' in node_id and 'N_1000001' in node_id:
if node_id[0] == 'N_809064':
node_id[0] = 'N_809074'
else:
node_id[1] = 'N_809074'
country_code = ['DE', 'DE']
if is_virtualPipe == False:
ReturnComponent.append(K_Component.PipeSegments(id = id,
name = name,
lat = lat,
long = long,
country_code = country_code,
node_id = node_id,
source_id = source_id,
param = {'max_pressure_bar': max_pressure_bar,
'is_H_gas' : is_H_gas,
'length' : length,
'diameter_mm' : diameter_mm,
'pipe_class_type': pipe_class_type,
'max_cap_GWh_per_d': max_cap_GWh_per_d,
'operator_name' : operator_name},
method = {'diameter_mm' : diam_est_method,
'pipe_class_type' : class_est_method,
'max_pressure_bar' : press_est_method},
uncertainty = {'diameter_mm': diam_est_uncertainty,
'pipe_class_type' : class_est_uncertainty,
'max_pressure_bar' : press_est_uncertainty},
))
count = count + 1
if count > NumDataSets:
return ReturnComponent
elif DataType in 'Nodes':
inCoord = 'epsg:31468'
outCoord = 'epsg:4326'
FileName_Shape = str(RelDirName / 'nodes_utf8.shp')
# Loading from shape file
Shapes = shapefile.Reader(FileName_Shape, encoding = "utf8")
# Malen der Europa Karte
for shape in Shapes.shapeRecords():
id = 'N_' + shape.record[0]
source_id = [ID_Add + str(shape.record[0])]
name = replaceString(shape.record[1])
operator_name = str(shape.record[2])
is_import = shape.record[3]
is_export = shape.record[4]
H_L_conver = int(shape.record[5])
operator_Z = shape.record[6]
compressor = shape.record[7]
compUnit = shape.record[8]
if 'NULL' in compUnit:
compUnit = 0
elif len(compUnit) == 0:
compUnit = 0
else:
compUnit = float(compUnit)
country_code= shape.record[12]
X_coor = shape.record[13]
Y_coor = shape.record[14]
entsog_nam = str(shape.record[15])
if len(entsog_nam) > 0:
name = entsog_nam
if name == '':
name = 'Ort_' + str(id)
entsog_key = shape.record[16]
if entsog_key == '':
entsog_key = None
is_crossBorder = shape.record[17]
ugs = shape.record[19]
production = shape.record[20]
exact = 1
license = 'open data'
Line = K_Component.PolyLine(lat = Y_coor, long = X_coor)
Line = M_Projection.XY2LatLong(Line, inCoord, outCoord)
lat = Line.lat
long = Line.long
if id == 'N_809066' and country_code == 'AT':
id = 'N_809076'
elif id == 'N_809065' and country_code == 'AT':
id = 'N_809075'
elif id == 'N_809064' and country_code == 'AT':
id = 'N_809074'
ReturnComponent.append(K_Component.Nodes(id = id,
node_id = [id],
name = name,
source_id = source_id,
long = long,
lat = lat,
country_code= country_code,
param = {'exact': exact,
'H_L_conver': H_L_conver,
'operator_Z': operator_Z,
'compressor': compressor,
'comp_units': compUnit,
'entsog_key': entsog_key,
'is_crossBorder': is_crossBorder,
'ugs' : ugs,
'production': production,
'operator_name': operator_name,
'is_import' : is_import,
'is_export' : is_export,
'license' : license}))
count = count + 1
if count > NumDataSets:
return ReturnComponent
elif DataType in 'Storages':
FileName_Shape = str(RelDirName / 'storages_utf8.shp')
# Loading from shape file
Shapes = shapefile.Reader(FileName_Shape, encoding = "utf8")
# Malen der Europa Karte
for shape in Shapes.shapeRecords():
id = 'N_' + shape.record[0]
source_id = [ID_Add + str(shape.record[0])]
name = replaceString(shape.record[1])
operator_name = str(shape.record[2])
entsog_nam = str(shape.record[9])
if len(entsog_nam) > 0:
name = entsog_nam
entsog_key = shape.record[10]
if entsog_key == '':
entsog_key = None
max_cap_pipe2store_GWh_per_d = shape.record[11]
max_cap_store2pipe_GWh_per_d = shape.record[12]
node_id = ['N_' + shape.record[0]]
country_code = shape.record[6]
ReturnComponent.append(K_Component.Storages(id = id,
name = name,
source_id = source_id,
country_code = country_code,
node_id = node_id,
param = {'operator_name': operator_name,
'entsog_key' : entsog_key,
'max_cap_pipe2store_GWh_per_d': max_cap_pipe2store_GWh_per_d,
'max_cap_store2pipe_GWh_per_d': max_cap_store2pipe_GWh_per_d}))
count = count + 1
if count > NumDataSets:
return ReturnComponent
elif DataType in 'Productions':
FileName_Shape = str(RelDirName / 'productions_utf8.shp')
# Loading from shape file
Shapes = shapefile.Reader(FileName_Shape, encoding = "utf8")
# Malen der Europa Karte
for shape in Shapes.shapeRecords():
id = 'N_' + shape.record[0]
source_id = [ID_Add + str(shape.record[0])]
name = replaceString(shape.record[1])
operator_name = str(shape.record[2])
entsog_nam = str(shape.record[9])
if len(entsog_nam) > 0:
name = entsog_nam
entsog_key = shape.record[10]
if entsog_key == '':
entsog_key = None
max_production = shape.record[11]
node_id = ['N_' + shape.record[0]]
country_code = shape.record[6]
ReturnComponent.append(K_Component.Productions(id = id,
name = name,
source_id = source_id,
node_id = node_id,
country_code = country_code,
param = {'entsog_key': entsog_key,
'operator_name': operator_name,
'is_H_gas' : 1,
'max_production_GWh_per_d': max_production}))
count = count + 1
if count > NumDataSets:
return ReturnComponent
return ReturnComponent
def read_component(DataType='', NumDataSets=1e+100, RelDirName=None):
""" Method of reading in Norway not infield pipelines from shape files. **RelDirName**
supplies the relative location of the shape files, whereas **DataType** specifies
which component is to be read in with options 'PipeSegments' and 'Nodes'
\n.. comments:
Input:
DataType String, specifying the component to be read in
(default = '')
NumDataSets: Number, indicating the maximum number of elements to be read in
(default = 1e+100).
RelDirName: string, containing the relative path name of where data will be loaded from
Default = None
Return:
[]
"""
# init variable to return and counter
ReturnComponent = []
count = 0
# start and target
inCoord = 'epsg:4230'
outCoord = 'epsg:4326'
# Path to Shapefile
FileName_Map = os.path.join(RelDirName, 'pipLine.shp')
# Read in shapefile
Shapes = shapefile.Reader(FileName_Map)
if DataType in 'PipeLines':
# go through every pipeline stored in shapefile
for shape in Shapes.shapeRecords():
# only read out gas pipelines
if 'Gas' == shape.record[11]:
# Getting the coordinates of the PipeSegment
parts = sorted(shape.shape.parts)
# Joining X and Y coordinates from Shape.shape.points
vec = shape.shape.points
polyLine = K_Component.PolyLine(lat=[], long=[])
for x, y in vec:
polyLine.long.append(x)
polyLine.lat.append(y)
# check if coordinates exists
if len(polyLine.long) and len(polyLine.lat):
# Converting to LatLong
polyLine = M_Projection.XY2LatLong(polyLine, inCoord,
outCoord)
# Generation of PipeLine
PipeLine = M_Shape.PolyLine2PipeLines(polyLine,
parts,
source=C_Code,
country_code=C_Code)
for ii in range(len(PipeLine)):
PipeLine[ii].id = 'N_' + str(count)
PipeLine[ii].source_id = [C_Code + '_' + str(count)]
PipeLine[ii].name = shape.record[1]
PipeLine[ii].node_id = [
'N_' + str(count * 2), 'N_' + str(count * 2 + 1)
]
PipeLine[ii].param.update({
'lat_mean':
M_MatLab.get_mean(PipeLine[ii].lat)[0]
})
PipeLine[ii].param.update({
'long_mean':
M_MatLab.get_mean(PipeLine[ii].long)[0]
})
PipeLine[ii].param.update({
'diameter_mm':
convInchToMm(shape.record[13])
}) # convert inches to mm
print(convInchToMm(shape.record[13]))
count = count + 1
ReturnComponent.extend(PipeLine)
if count > NumDataSets:
return ReturnComponent
elif DataType in 'Nodes':
# go through every pipeline stored in shapefile
for shape in Shapes.shapeRecords():
# Only read out nodes of gas pipelines
if 'Gas' == shape.record[11]:
# Getting the coordinates of the PipeSegment
parts = sorted(shape.shape.parts)
# Joining X and Y coordinates from Shape.shape.points
vec = shape.shape.points
polyLine = K_Component.PolyLine(lat=[], long=[])
for x, y in vec:
polyLine.long.append(x)
polyLine.lat.append(y)
# check if coordinates exists
if len(polyLine.long) and len(polyLine.lat):
# Converting to LatLong
polyLine = M_Projection.XY2LatLong(polyLine, inCoord,
outCoord)
# Generation of PipeSegments
Segments = M_Shape.PolyLine2PipeSegment(
polyLine, parts, source=C_Code, country_code=C_Code)
# Generation of the Nodes from PipeSegments
# two Nodes per PipeSegment
for seg in Segments:
id = 'N_' + str(len(ReturnComponent))
name = 'N_' + str(len(ReturnComponent))
node_id = [id]
source_id = [C_Code + '_' + str(len(ReturnComponent))]
country_code = C_Code
lat = seg.lat[0]
long = seg.long[0]
ReturnComponent.append(
K_Component.Nodes(id=id,
node_id=node_id,
name=name,
lat=lat,
long=long,
source_id=source_id,
country_code=country_code,
param={}))
id = 'N_' + str(len(ReturnComponent))
name = 'N_' + str(len(ReturnComponent))
node_id = [id]
source_id = [C_Code + '_' + str(len(ReturnComponent))]
country_code = C_Code
lat = seg.lat[1]
long = seg.long[1]
ReturnComponent.append(
K_Component.Nodes(id=id,
node_id=node_id,
name=name,
lat=lat,
long=long,
country_code=country_code,
source_id=source_id,
param={}))
count = count + 1
# Terminate new data if exceeding user requests
if count > NumDataSets:
return ReturnComponent
return ReturnComponent
def getMatch_LatLong_CountryCode(comp_0, comp_1, method = 'inv', thresholdVal = None):
"""Gets the separation between two points in km, and returns as 100-1/distance and other methods.
Method allows to select different measures of distance returned (distance used here in [km]), from:
"inv" (100 / distance),
"power2inv" (100 / (distance^2)),
"loginv" (100 / log(distance), with base e),
"log10inv" (100 / log10(distance), with base 10),
"distance" (distance)
"""
# Initialization
RetVal = 0
# Netz_0 is empty
if comp_0 == '':
RetVal = 0
# Netz_1 is empty
elif comp_1 == '':
RetVal = 0
elif comp_0.long == None:
RetVal = 0
elif comp_1.long == None:
RetVal = 0
elif type(comp_0.lat) == str:
print('ERROR: M_Matching.getComp_LatLong: input type is string. Float expected. comp_0')
RetVal
elif type(comp_1.lat) == str:
print('ERROR: M_Matching.getComp_LatLong: input type is string. Float expected. comp_1')
RetVal = 0
# Both Netze contain components
else:
cc_Netz_0 = comp_0.country_code
cc_Netz_1 = comp_1.country_code
if cc_Netz_0 == cc_Netz_1 or cc_Netz_1 == None or cc_Netz_0 == None:
# Creation of LatLong "vector" from component latlong
latlong_Netz_0 = K_Comp.PolyLine(lat = [comp_0.lat], long = [comp_0.long] ) #M_Netze.get_latlongPairs_Points(comp_0)
thisLatLong = K_Comp.PolyLine(lat = comp_1.lat, long = comp_1.long ) #M_Netze.get_latlongPairs_Points(comp_1)
[pos, minVal] = M_FindPos.find_pos_closestLatLongInList(thisLatLong, latlong_Netz_0)
#minVal = minVal/1000
if minVal == 0.0:
RetVal = 100
elif method == 'inv':
RetVal = min([100 / minVal, 100])
elif method == 'power2inv':
RetVal = 100 / minVal/minVal
elif method == 'log10inv':
RetVal = 100 / math.log(minVal, 10)
elif method == 'loginv':
RetVal = 100 / math.log(minVal)
elif method == 'distance':
RetVal = minVal
elif method == 'distanceThreshold':
if minVal <= thresholdVal:
RetVal = 100
elif method == 'exp':
RetVal = 100 * math.exp(-minVal*1000/thresholdVal)
else:
print('ERROR: M_Matching: get_Comp_LatLong: method not defined.')
else:
return -100000
# Testig if nan, if so then set to zero
if math.isnan(RetVal) :
RetVal = 0
return RetVal
#def replacePipeSegments(Netz_Main, Netz_Fine, nodeDistance = 10000, lengthDistance = 0.2):
# """This function does not do a thing
# """
# # Determine which nodes are the same in both data sets
# [pos_match_Netz_0, pos_add_Netz_0, pos_match_Netz_1, pos_add_Netz_1] = match(
# Netz_Main, Netz_Fine, compName = 'Nodes', threshold = 45, multiSelect = False,
# numFuncs = 1,
# funcs = (
# lambda comp_0, comp_1: getMatch_LatLong_CountryCode(comp_0, comp_1, method = 'inv')
# ))
#
# # Convert Netz_Fine into NetWorkx
# InfoDict = {'Gewichtung': 'Gleich', 'Weight': 'Gleich'}
# [Graph_Fine, MDGraph_Fine] = M_Graph.build_nx(InfoDict, Netz_Fine)
# [Graph_Main, MDGraph_Main] = M_Graph.build_nx(InfoDict, Netz_Main)
#
# for pipe1 in Netz_Main.PipeSegments:
# # Determine length of network 1
# pair = [pipe1.node_id[0], pipe1.node_id[1]]
# length_Main = M_Graph.get_shortest_paths_distances(Graph_Main, pair, edge_weight_name = 'length')
#
# # Determine length of network 2
# #pos = M_FindPos.find_pos_ValInVector(Val, Vector, Type)
# length_Fine = M_Graph.get_shortest_paths_distances(Graph_Fine, pair, edge_weight_name = 'length')
#
# print('M_Matching.replacePipeSegments: this function need checking, currently does nothing')
#
# return Netz_Main
def getMatch_LatLong(comp_0, comp_1, method = 'inv'):
"""Gets the separation between two points in km, and returns as 100-1/distance and other methods.
Method allows to select different measures of distance returned (distance used here in [km]), from:
"inv" (100 / distance),
"power2inv" (100 / (distance^2)),
"loginv" (100 / log(distance), with base e),
"log10inv" (100 / log10(distance), with base 10),
"distance" (distance)
"""
# Initialization
RetVal = 0
# Netz_0 is empty
if comp_0 == '':
return 0
# Netz_1 is empty
elif comp_1 == '':
return 0
elif comp_0.long == None:
return 0
elif comp_1.long == None:
return 0
elif type(comp_0.lat) == str:
print('ERROR: M_Matching.getComp_LatLong: input type is string. Float expected. comp_0')
elif type(comp_1.lat) == str:
print('ERROR: M_Matching.getComp_LatLong: input type is string. Float expected. comp_1')
# Both Netze contain components
else:
# Creation of LatLong "vector" from component latlong
latlong_Netz_0 = K_Comp.PolyLine(lat = [comp_0.lat], long = [comp_0.long] ) #M_Netze.get_latlongPairs_Points(comp_0)
thisLatLong = K_Comp.PolyLine(lat = comp_1.lat, long = comp_1.long ) #M_Netze.get_latlongPairs_Points(comp_1)
[pos, minVal] = M_FindPos.find_pos_closestLatLongInList(thisLatLong, latlong_Netz_0)
minVal = minVal/1000
if minVal == 0.0:
RetVal = 100
elif method == 'inv':
RetVal = min([100 / minVal, 100])
elif method == 'power2inv':
RetVal = 100 / minVal/minVal
elif method == 'log10inv':
RetVal = 100 / math.log(minVal, 10)
elif method == 'loginv':
RetVal = 100 / math.log(minVal)
elif method == 'distance':
RetVal = minVal
else:
print('ERROR: M_Matching: get_Comp_LatLong: method not defined.')
# Testig if nan, if so then set to zero
if math.isnan(RetVal) :
RetVal = 0
return RetVal