def outFilteredData(dataList, cyc):
filDataList = []
for data in dataList:
filData = avgFilter(data, cyc)
filDataList.append(filData)
return filDataList
def outageFn(dataFileName, tFileName, eventFileName, classInd):
# use this function to get any sort of data from outage events
# file objects
eventList = []
with open(eventFileName, 'r') as f:
fileLines = f.read().split('\n')
for line in fileLines[1:]:
if line == '':
continue
eventList.append(line.strip())
dataFileName = open(
dataFileName,
'rb') # 'wb' needed to avoid blank space in between lines
tFileName = open(tFileName, 'rb')
vReader = csv.reader(dataFileName, quoting=csv.QUOTE_NONNUMERIC
) # 'wb' needed to avoid blank space in between lines
tReader = csv.reader(tFileName, quoting=csv.QUOTE_NONNUMERIC)
tme = [row for idx, row in enumerate(tReader) if idx == 0][0]
listOut = []
# get the voltage data (and simulate balanced three phase)
for idx, row in enumerate(vReader):
eventKey = eventList[idx]
eventKeyWords = eventKey.split('/')
currentbus = eventKeyWords[2][1:].strip()
if currentbus in PMUBuses:
# add some noise to the outputs
row = np.array(row) + np.random.normal(
0, std,
len(row)) # normal noise with standard deviation of std
# pass the input through a 6 cycle average filter
row = avgFilter(row, numcyc)
allKeys.append(eventKey)
for i in range(shiftRange):
currentList = []
for bs in range(3): # simulate three phase
for v in row[startind - i:endind - i]:
currentList.append(v)
listOut.append(currentList)
eventoutArray = np.array(listOut)
eventoutTarget = np.array([classInd] * eventoutArray.shape[0])
# close all files
dataFileName.close()
tFileName.close()
return eventoutArray, eventoutTarget
def spike(startTime, endTime, originalSignal, spikeHeight):
#startTime = 1000
#endTime = 1001
# calculate the starting sample and the number of samples needed
distStart = startTime * 30
distEnd = endTime * 30
distSamples = int((distEnd - distStart))
#numSineWaves = 100
numSineWaves = 20 # number to sine waves to add to get the final timeseries during the dip
#dropV = 0.01 # drop in the voltage
#dropVariance = 0.125 # ratio of variance wrt drop
dropVariance = 0.5 # ratio of max variance of the sine waves with respect to drop
frequencyRange = np.linspace(1, 5, 10)
#frequencyRange = np.linspace(1,20,10)
#frequencyRange = np.linspace(1,100,10)
#dist = np.zeros(distSamples)
#dist = dist - dropV
spike = np.zeros(distSamples)
for i in range(numSineWaves):
freq = random.choice(frequencyRange)
Fs = distSamples # sampling frquency
#sample = 50
s = np.arange(distSamples)
y = np.sin(2 * np.pi * freq * s / Fs)
spike += y
#waveRange = spike.max() - spike.min()
spike = spike / spike.max() * spikeHeight
z = np.zeros(originalSignal.shape[0])
z[distStart:distStart + spike.shape[0]] = spike
# apply smoothing on the disturbance and noise
z = np.array(avgFilter(z, 6))
z += np.random.normal(0, 0.001, z.shape[0])
newSignal = originalSignal + z
return newSignal
print 'Organizing all the data into arrays for the classifier'
for idx, row in enumerate(readerV):
eventKey = eventList[idx]
eventKeyWords = eventKey.split('/')
PL = eventKeyWords[0][1:].strip()
faultbus = eventKeyWords[1][1:].strip()
currentbus = eventKeyWords[2][1:].strip()
faulttype = eventKeyWords[3].strip()
phase = eventKeyWords[4].strip()
faultZ = eventKeyWords[5].strip()
condensedKey = 'R{}/F{}/B{}/{}/{}'.format(PL, faultbus, currentbus,
faulttype, faultZ)
# pass the input through a 6 cycle average filter
row = avgFilter(row, numcyc)
# add some noise to the outputs
row = np.array(row) + np.random.normal(
0, std, len(row)) # normal noise with standard deviation of std
# only get the data if a PMU has been randomly placed at that bus
if currentbus in PMUBuses and faultbus != currentbus:
if phase == 'A':
if faulttype == 'ABCG':
for i in range(shiftRange):
condensedKey += 'S{}'.format(i)
list3PHA.append(row[startind - i:endind - i])
condensedEventKey3ph.append(condensedKey)
eventTarget3ph.append(0)
EventDict = {}
SignalDict = {}
for idx, row in enumerate(readerV):
eventKey = eventList[idx]
eventKeyWords = eventKey.split('/')
PL = eventKeyWords[0][1:].strip()
faultbus = eventKeyWords[1][1:].strip()
currentbus = eventKeyWords[2][1:].strip()
faulttype = eventKeyWords[3].strip()
phase = eventKeyWords[4].strip()
faultZ = eventKeyWords[5].strip()
eventID = 'R{}/F{}/{}/{}'.format(PL, faultbus, faulttype, faultZ)
# implement 6 cycle filter and smoothing
row = avgFilter(row, 6)
row = np.array(row) + np.random.normal(
0, std, len(row)) # normal noise with standard deviation of std
if eventID not in EventDict:
EventDict[eventID] = Signals()
EventDict[eventID].SignalDict['B{}/{}'.format(currentbus, phase)] = row
# arrange all the fault event data into rows sequentially
AllEventList = []
targetList = []
for event in EventDict:
s = EventDict[event].SignalDict
# get the event class
eventKeyWords = event.split('/')
faulttype = eventKeyWords[2].strip()
eventTargetSLGA = [] # targets for all the single phase fault data at phase A
eventTargetSLGB = [] # SLG phase B
eventTargetSLGC = [] # SLG phase C
print 'Organizing all the data into arrays for the classifier'
for idx, row in enumerate(readerV):
eventKey = eventList[idx]
eventKeyWords = eventKey.split('/')
PL = eventKeyWords[0][1:].strip()
faultbus = eventKeyWords[1][1:].strip()
currentbus = eventKeyWords[2][1:].strip()
faulttype = eventKeyWords[3].strip()
phase = eventKeyWords[4].strip()
faultZ = eventKeyWords[5].strip()
condensedKey = 'R{}/F{}/B{}/{}/{}'.format(PL, faultbus, currentbus,
faulttype, faultZ)
rowFil = avgFilter(row, 6)
if phase == 'A':
if faulttype == 'ABCG':
list3PHA.append(row[faultonind:faultoffind])
list3PHAFil.append(
rowFil[faultonind:faultoffind]) # 6 cycle filter
#listSteadyA.append(row[:faultonind-1]) # isolate the steady state part
condensedEventKey3ph.append(condensedKey)
steadyKey = condensedKey + '/steady'
#condensedEventKeySteady.append(steadyKey)
if currentbus == faultbus:
eventTarget3ph.append(0) # three phase fault
else:
eventTarget3ph.append(4)
elif faulttype == 'AG':
eventList.append(line.strip())
EventDict = {}
SignalDict = {}
for idx, row in enumerate(readerV):
eventKey = eventList[idx]
eventKeyWords = eventKey.split('/')
PL = eventKeyWords[0][1:].strip()
faultbus = eventKeyWords[1][1:].strip()
currentbus = eventKeyWords[2][1:].strip()
faulttype = eventKeyWords[3].strip()
phase = eventKeyWords[4].strip()
faultZ = eventKeyWords[5].strip()
# pass the input through a 6 cycle average filter
out = avgFilter(row, 6)
# add some noise to the outputs
out = np.array(out) + np.random.normal(
0, 0.01, len(out)) # normal noise with standard deviation of 0.01
# ignore all faults with relatively high impedance
#if float(faultZ) > 0.5e-2:
# continue
eventID = 'R{}/F{}/{}/{}'.format(PL, faultbus, faulttype, faultZ)
if eventID not in EventDict:
EventDict[eventID] = Signals()
EventDict[eventID].SignalDict['B{}/{}'.format(currentbus, phase)] = out
# arrange all the fault event data into rows sequentially
AllEventList = []
precontvolt = genoutSample[0]
genoutSample += (1.0 - precontvolt)
sampleLength = genoutSample.shape[0]
# add noise
genoutSample += np.random.normal(0, 0.001, genoutSample.shape[0])
# add smoothing
#genoutSample = avgFilter(genoutSample,6)
startTime = 4000
startSample = startTime * 30
newV[startSample:startSample + sampleLength] = genoutSample
endSample = startSample + sampleLength
# try to smooth the transition
transitionWindow = newV[startSample - 1000:endSample + 1000]
transitionWindow = np.array(avgFilter(transitionWindow, 100))
transitionWindow += np.random.normal(0, 0.001, transitionWindow.shape[0])
#newV[startSample-1000:endSample+1000] = transitionWindow
newV[endSample:endSample + 1000] = transitionWindow[
-1000:] # only incorporate the transition after the event ends
# plt.plot(t,newV)
# plt.grid()
# plt.ylim(0.2,1.1)
# plt.show()
######
### add line outage data
LineOutDict = {}
lineOutDir = 'G:/My Drive/My PhD research/Running TS3ph/LineOut'
vFileName = '{}/vLineOut.csv'.format(
for idx, row in enumerate(readerT):
tDict[idx] = row
for idx, row in enumerate(readerV):
vDict[idx] = row
croppedV = []
for idx in tDict:
t = tDict[idx]
startInd = min([ind for ind, val in enumerate(t) if val >=startTime])
endInd = startInd + numSamples
v = vDict[idx]
# add noise and smoothing
# pass the input through a 6 cycle average filter
v = avgFilter(v,numcyc)
# add some noise to the outputs
v = np.array(v) + np.random.normal(0,std,len(v)) # normal noise with standard deviation of std
for ts in range(shiftRange):
lst = []
for rep in range(3):
for val in v[startInd-ts:endInd-ts]:
lst.append(val)
croppedV.append(lst)
mStartArray = np.array(croppedV)
"""
for i in range(5):
condensedEventKeySteady = []
#condensedset = set()
eventTarget3ph = [] # targets for all the 3 ph fault data
eventTargetSLGA = [] # targets for all the single phase fault data at phase A
eventTargetSLGB = [] # SLG phase B
eventTargetSLGC = [] # SLG phase C
for idx, row in enumerate(readerV):
eventKey = eventList[idx]
eventKeyWords = eventKey.split('/')
PL = eventKeyWords[0][1:].strip()
faultbus = eventKeyWords[1][1:].strip()
currentbus = eventKeyWords[2][1:].strip()
faulttype = eventKeyWords[3].strip()
phase = eventKeyWords[-1].strip()
condensedKey = 'R{}/F{}/B{}/{}'.format(PL, faultbus, currentbus, faulttype)
rowFil = avgFilter(row, 6)
if phase == 'A':
if faulttype == 'ABCG':
list3PHA.append(row[faultonind:faultoffind])
list3PHAFil.append(
rowFil[faultonind:faultoffind]) # 6 cycle filter
listSteadyA.append(row[:faultonind -
1]) # isolate the steady state part
condensedEventKey3ph.append(condensedKey)
steadyKey = condensedKey + '/steady'
condensedEventKeySteady.append(steadyKey)
if currentbus == faultbus: # 3 phase fault at current bus
eventTarget3ph.append(1)
else: # three phase fault at different bus
eventTarget3ph.append(5)
def outageFn(dataFileName, tFileName, eventFileName, classInd, eventPrefix):
# use this function to get any sort of data from outage events
EventDict = {}
# file objects
eventList = []
with open(eventFileName, 'r') as f:
fileLines = f.read().split('\n')
for line in fileLines[1:]:
if line == '':
continue
eventList.append(line.strip())
dataFileName = open(
dataFileName,
'rb') # 'wb' needed to avoid blank space in between lines
tFileName = open(tFileName, 'rb')
dataReader = csv.reader(
dataFileName, quoting=csv.QUOTE_NONNUMERIC
) # 'wb' needed to avoid blank space in between lines
tReader = csv.reader(tFileName, quoting=csv.QUOTE_NONNUMERIC)
tme = [row for idx, row in enumerate(tReader) if idx == 0][0]
# make a dictionary where each key corresponds to all measurements available for the event
for idx, row in enumerate(dataReader):
eventKey = eventList[idx]
eventKeyWords = eventKey.split('/')
PL = eventKeyWords[0][1:].strip()
stuffout = eventKeyWords[1][1:].strip()
currentbus = eventKeyWords[2][1:].strip()
if currentbus in PMUBuses:
# add some noise to the outputs
row = np.array(row) + np.random.normal(
0, std,
len(row)) # normal noise with standard deviation of std
# pass the input through a 6 cycle average filter
row = avgFilter(row, numcyc)
eventID = 'R{}/{}{}'.format(PL, eventPrefix, stuffout)
if eventID not in EventDict:
#eventTracker.append(eventID)
#eventSet.add(eventID)
EventDict[eventID] = []
for bs in range(3): # simulate three phase
for v in row[startind:endind]:
#lineoutvdata.append(v)
EventDict[eventID].append(v)
# now make an array out of the dictionary keys
EventList = []
for event in EventDict:
EventList.append(EventDict[event])
eventTracker.append(event)
EventArray = np.array(EventList)
targetArray = np.array([classInd] * (EventArray.shape[0]))
return EventArray, targetArray
frequencyRange = np.linspace(1, 5, 10)
distSamples = 1000
dist = np.zeros(distSamples)
dist = dist - suddendrop
wave = np.zeros(distSamples)
for i in range(numSineWaves):
freq = random.choice(frequencyRange)
Fs = distSamples # sampling frquency
#sample = 50
s = np.arange(distSamples)
y = np.sin(2 * np.pi * freq * s / Fs)
wave += y
waveRange = wave.max() - wave.min()
wave = wave / waveRange * suddendrop * dropVariance
dist += wave
# plt.plot(signal/signal.max()*0.2)
# plt.grid()
# plt.show()
z[5000:5000 + dist.shape[0]] = dist
a = x + z
a = np.array(avgFilter(a, 100))
a += np.random.normal(0, 0.001, a.shape[0])
plt.plot(a)
plt.ylim(0.75, 1.1)
plt.grid()
plt.show()
