def plot_overlay(plots):
fig = plt.figure()
ax = fig.add_subplot(111)
plt.xlabel('Time')
plt.ylabel('Service Demand')
ax.set_title('Signal Overlay');
accumulated_signal = None
for plot in plots:
trace = load_trace(plot)
signal = []
for element in trace.elements:
signal.append(element.value)
if accumulated_signal == None:
accumulated_signal = signal
else:
for i in range(0, len(signal)):
accumulated_signal[i] += signal[i]
plot_signal(ax, signal)
plot_signal(ax, accumulated_signal)
# Save
fname = 'aggregated' + '.png'
fname = fname.replace('/', '_')
#fig.savefig('../../target/%s' % (fname))
# Show the plot in a window
plt.show()
def extract_signal(elements):
signal = []
for element in elements:
signal.append(element.value)
return signal
def main(*argv):
tr = {}
### create the pulse template and save it to the return
(template, python_template) = getTemplate()
tr['template'] = python_template
pulseLength = len(python_template) #get the length from the template
# get a noise pulse
# save it to the return
random.seed()
noisepulse = getNoisePulse(pulseLength)
wn = []
for i in range(pulseLength):
wn.append(noisepulse[i])
tr['noise'] = wn
#create the signal by adding the template to the noise
#save it to the return
signal = createSignal(pulseLength, python_template, noisepulse, 10000.)
signal2 = createSignal(pulseLength, python_template, noisepulse, 20000.)
sig = []
for i in range(pulseLength):
sig.append(signal[i])
tr['signal'] = sig
def dekoderManchester(clk, manchester, samples):
signal = []
for i in range(len(clk) - 1):
if (clk[i] == 1 and clk[i + 1] == 0):
signal.append(manchester[i])
signal.append(manchester[-1])
return signal
def _synchronize(self):
for name_global_spr in [NAME_GLOBAL_SPR, 'spr_integral']:
try:
with open(
self.folder + name_global_spr + self.file[-2:] +
'.tsv', 'r') as spr:
contents = spr.readlines()
time = []
signal = []
for line in contents[:-1]:
line_split = line.split('\t')
time.append(float(line_split[0]))
signal.append(float(line_split[1]))
beginning = list(self.graphs['spr_signal'][:2])
for i in range(len(signal)):
if signal[i:i + 2] == beginning:
self.zero_time = time[i]
break
elif i == len(signal) - 2:
self.zero_time = 0
return
except FileNotFoundError:
self.zero_time = 0
def zeroPadding(signal, length_of_padding):
#apply zero padding in the request length
for x in range(length_of_padding):
signal.append(0)
return signal
def construct(self):
self.setup_axes(animate=True)
signal = []
for i, k in zip(range(0, 5), [1, 2, 2, 1, 3]):
line = Line(self.coords_to_point(i, 0),
self.coords_to_point(i, k),
buff=0.05)
point = Dot(self.coords_to_point(i, k))
line_point = VGroup(line, point)
signal.append(line_point)
signals = VGroup(*signal)
self.play(ShowCreation(signals))
self.wait(1)
self.play(
ApplyMethod(signals.shift,
-5 * np.array([self.space_unit_to_x, 0, 0])))
unit = Arrow(self.graph_origin, self.coords_to_point(0, 1), buff=0.05)
self.play(ShowCreation(unit))
responses = []
for n in range(0, 5):
res = []
for i, k in zip(range(3 + n, 8 + n), [2, 3, 3, 1, 5]):
line = Line(self.coords_to_point(i, 0),
self.coords_to_point(i, k),
buff=0.05)
point = Dot(self.coords_to_point(i, k))
line_point = VGroup(line, point)
line_point.set_color(RED)
res.append(line_point)
response = VGroup(*res)
responses.append(response)
independent_res = VGroup(*responses)
for i in range(0, 5):
self.play(
ApplyMethod(signals[i].shift,
(5 - i) * np.array([self.space_unit_to_x, 0, 0])))
self.play(ShowCreation(responses[i]))
self.play(FadeOut(signals[i]))
self.wait(0.5)
self.play(FadeOut(unit))
total_response = []
for i, k in zip(range(3, 12), [2, 5, 5, 6, 11, 9, 6, 5]):
line = Line(self.coords_to_point(i, 0),
self.coords_to_point(i, k),
buff=0.05)
point = Dot(self.coords_to_point(i, k))
line_point = VGroup(line, point)
line_point.set_color(RED)
total_response.append(line_point)
total_responses = VGroup(*total_response)
self.play(Transform(independent_res, total_responses))
self.wait(2)
def get_signal(filename: str):
signal_file = open(filename, 'r')
signal = []
for line in signal_file:
signal.append(int(line))
signal_file.close()
return signal
def construct(self):
self.setup_axes(animate=True)
signal = []
for i, k in zip(range(0, 5), [1, 2, 2, 1, 3]):
line = Line(self.coords_to_point(i, 0),
self.coords_to_point(i, k),
buff=0.05)
point = Dot(self.coords_to_point(i, k))
line_point = VGroup(line, point)
signal.append(line_point)
signals = VGroup(*signal)
# signal2=[]
# for i,k in zip(range(0,5),[3,1,2,2,1]):
# line=Line(self.coords_to_point(i,0),self.coords_to_point(i,k),buff=0.05)
# point=Dot(self.coords_to_point(i,k))
# line_point=VGroup(line,point)
# signal2.append(line_point)
# signals2=VGroup(*signal2)
# signals2.shift(-5*np.array([self.space_unit_to_x, 0, 0]))
self.play(ShowCreation(signals))
self.wait(1)
self.play(
ApplyMethod(signal[4].shift,
-9 * np.array([self.space_unit_to_x, 0, 0])),
ApplyMethod(signal[3].shift,
-7 * np.array([self.space_unit_to_x, 0, 0])),
ApplyMethod(signal[2].shift,
-5 * np.array([self.space_unit_to_x, 0, 0])),
ApplyMethod(signal[1].shift,
-3 * np.array([self.space_unit_to_x, 0, 0])),
ApplyMethod(signal[0].shift,
-1 * np.array([self.space_unit_to_x, 0, 0])),
)
unit = Arrow(self.graph_origin, self.coords_to_point(0, 1), buff=0.05)
self.play(ShowCreation(unit))
total_response = []
for i, k in zip(range(3, 12), [2, 5, 5, 6, 11, 9, 6, 5]):
line = Line(self.coords_to_point(i, 0),
self.coords_to_point(i, k),
buff=0.05)
point = Dot(self.coords_to_point(i, k))
line_point = VGroup(line, point)
line_point.set_color(RED)
total_response.append(line_point)
total_responses = VGroup(*total_response)
self.play(
ApplyMethod(signals.shift,
5 * np.array([self.space_unit_to_x, 0, 0])),
ShowCreation(total_responses))
self.play(FadeOut(signals), FadeOut(unit))
self.wait(2)
def signal_extraction(path):
signal = []
with open(path, 'rt') as csvfile:
spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|')
for row in spamreader:
signal.append(', '.join(row))
return signal
def training_tempo(filename, sr=22050):
signal = []
with open(filename, mode='r') as file:
datareader = csv.reader(file)
next(datareader)
for row in datareader:
tempo, interval = int(row[0]), int(row[1])
signal.append((tempo, interval))
signal = np.array(signal)
return signal
def equalizer_10band(data, fs, gains):
signal = []
for i in range(len(gains)):
signal.append(
bandpass_filter(data,
lowcut_frequencies[i],
highcut_frequencies[i],
fs,
order=3) * 10**(gains[i] / 20))
return sum(signal)
def sprawdzacz(signalData, w):
low = 120
high = 6000
signalData = np.array(signalData) # to numpy array
if (len(signalData.shape) > 1): # if array 2D then make it 1D
signalData = [s[0] for s in signalData]
signal = []
if w * 3 < len(signalData):
for i in range(w, w * 3):
signal.append(signalData[i])
else:
signal = signalData
signalfft = fft(signal)
signalfft = abs(signalfft)
signal = []
freqs = range(int(len(signalfft) / 2))
for i in freqs:
signal.append(signalfft[i])
if i < low or i > high:
signal[i] = 0
output = []
result = signal.copy()
output.append(signal)
for i in range(1, 8):
output.append(scipy.signal.decimate(
signal, i)) # downsampling the signal by applying filter
for j in range(len(output[i])):
result[j] = result[j] * output[i][
j] # apply filtered signal to destination array
for i in range(len(result)):
if result[i] < 1:
result[i] = 0
# plt.subplot(211)
# p1 = plt.plot(freqs, signal, '-')
# plt.yscale('log')
# plt.subplot(212)
# p2 = plt.plot(freqs, result, '-')
# plt.yscale('log')
# plt.show()
print(freqs[argmax(result, 0)])
if freqs[argmax(result, 0)] > 350:
return ("K")
else:
return ("M")
def _read_signal_file(
filepaths: List[str],
signal_unit: Quantity = None) -> IrregularlySampledSignal:
# things are a bit complicated here as the signal is not necessarily covering the whole experiment!
try:
# get the continous signal file
signal_file = [
file for file in filepaths
if "continuous" in os.path.basename(file).lower()
][0]
times = []
signal = []
# open the file for reading
with open(signal_file, "r") as file:
# and create a reader
reader = csv.reader(file, delimiter=",")
# this try/catch handles the exception that is raised by "next" if the reader reached the file ending
try:
while True:
# read time and signal rows
time_row = np.array([float(val) for val in next(reader)])
signal_row = np.array([float(val) for val in next(reader)])
assert len(time_row) == len(signal_row)
times.append(time_row)
signal.append(signal_row)
except StopIteration:
pass
# concatenate our list of arrays
times = np.concatenate(times) * second
signal = np.concatenate(signal)
assert len(times) == len(signal)
if not signal_unit is None:
signal = Quantity(signal, signal_unit)
else:
signal = Quantity(signal, "dimensionless")
result = IrregularlySampledSignal(times=times,
signal=signal,
name="Irregularly Sampled Signal",
file_origin=signal_file)
channel_id = f"{TypeID.RAW_DATA.value}.0"
result.annotate(id=channel_id, type_id=TypeID.RAW_DATA.value)
return result
# something might go wrong as we perform some IO operations here
except Exception as ex:
traceback.print_exc()
def wavelet_decomposition(self,signals):
import scipy.signal
signal = []
noise = []
snr = []
for freq in xrange(1,75):
dec = numpy.array([scipy.signal.fftconvolve(s-numpy.mean(s),self.gabor_wavelet(freq,s.sampling_rate.rescale(qt.Hz).magnitude),mode='same') for s in signals])
l = len(dec[0])
l1 = len(signals[0])
if l > l1:
dec = [d[int((l-l1)/2):int((l-l1)/2)+l1] for d in dec]
m = numpy.mean(dec,axis=0)
signal.append(numpy.absolute(m))
noise.append(numpy.mean(numpy.absolute(dec - m),axis=0))
snr.append(numpy.divide(signal[-1], noise[-1]))
return (numpy.array(signal),numpy.array(noise),numpy.array(snr))
def wavelet_decomposition(self,signals):
import scipy.signal
signal = []
noise = []
snr = []
for freq in xrange(1,75):
dec = numpy.array([scipy.signal.fftconvolve(s-numpy.mean(s),self.gabor_wavelet(freq,s.sampling_rate.rescale(qt.Hz).magnitude),mode='same') for s in signals])
l = len(dec[0])
l1 = len(signals[0])
if l > l1:
dec = [d[int((l-l1)/2):int((l-l1)/2)+l1] for d in dec]
m = numpy.mean(dec,axis=0)
signal.append(numpy.absolute(m))
noise.append(numpy.mean(numpy.absolute(dec - m),axis=0))
snr.append(numpy.divide(signal[-1], noise[-1]))
return (numpy.array(signal),numpy.array(noise),numpy.array(snr))
def _load_spr(self):
try:
with open(
self.folder + self.file.replace(NAME_RAW, NAME_LOCAL_SPR) +
'.tsv', 'r') as spr:
contents = spr.readlines()
time = []
signal = []
for line in contents[:]:
line_split = line.split('\t')
time.append(float(line_split[0]))
signal.append(float(line_split[1]))
return time, np.array(signal)
except FileNotFoundError:
self.print('SPR file not found. Diseable ploting of SPR. ')
return None, None
def predictToGenerateThreshold(self, model):
predicted = model.predict(self.X_test)
z = []
start = 0
for i in range(start, len(predicted) + start):
z.append(
abs(predicted[i][0]) + abs(predicted[i][1]) +
abs(predicted[i][2]) + abs(predicted[i][3]) +
abs(predicted[i][4]) + abs(predicted[i][5]))
z = np.array(z)
signal = []
for i in range(start, len(predicted) + start):
signal.append(
abs(self.y_test[i][0]) + abs(self.y_test[i][1]) +
abs(self.y_test[i][2]) + abs(self.y_test[i][3]) +
abs(self.y_test[i][4]) + abs(self.y_test[i][5]))
signal = np.array(signal)
mse = ((signal - z)**2)
# generate threshold
max = 0
min = 0
for i in range(0, len(mse)):
if (mse[i] > max):
max = mse[i]
if (mse[i] < min):
min = mse[i]
self.max = max
self.min = min
print("threshold", min, max)
def move_signal(signals, shifts, frequencies=None, freq_offset=1):
if len(shifts) != len(signals):
_logger.error("1st Dimension missmatch between signal and shift-list")
return None
else:
if frequencies is None:
frequencies = [None] * len(shifts)
for i in xrange(len(shifts)):
frequencies[i] = (np.array(signals[i][1:]) -
np.array(signals[i][:-1])).mean()
for i, s in enumerate(map(int, shifts)):
if s != 0:
f = frequencies[(i + freq_offset) % len(signals)]
if s > 0:
signal = list(signals[i][s:])
for i in xrange(s):
signal.append(signal[-1] + f)
else:
signal = signals[i][:s]
for i in xrange(-s):
signal.insert(0, signal[0] - f)
signals[i][:] = np.array(signal)
for i, s in enumerate([s - int(s) for s in shifts]):
if s != 0:
signals[i][:] = np.array(
[sign + s * frequencies[i] for sign in signals[i]])
return signals
def gencurve(self, n_windows):
#TODO numpy array instead of list
signal = []
label = np.empty(n_windows)
signal.append(self.generate_random_sample_const()
) # signal always starts with zeros
label[0] = 0
for i in range(1, n_windows):
newSignal, label[i] = self.__data_generation()
newSignal += signal[-1][
-1] #add last Value to create continous signal
signal.append(newSignal)
#Conversion to numpy arraay
longfunction = np.empty(0)
for i in range(len(signal)):
values = np.asarray(signal[i])
longfunction = np.concatenate([longfunction, values])
time = np.arange(0, longfunction.shape[0] * self.resolution,
self.resolution)
time_series = np.empty([2, longfunction.shape[0]])
time_series[0] = time
time_series[1] = longfunction
return time_series, label
def my_square():
time = np.arange(0, 10.1, 0.1)
signal = []
for i in time:
if i < 4:
signal.append(0)
elif 4 < i < 8:
signal.append(1)
else:
signal.append(0)
plt.plot(time, signal, linewidth = 3)
plt.show()
return signal
def insert_zeros(self, intervals): intervals.sort() # garante os intervalos ordenados signal = [] index = 0 for time in numpy.nditer(self.time): try: # garante que nao vai dar out of index if time < intervals[0][0]: signal.append(self.signal[index]) if intervals[0][0] <= time <= intervals[0][1]: signal.append(0.0) elif time > intervals[0][1]: intervals.pop(0) except IndexError: signal.append(self.signal[index]) index += 1 self.signal = numpy.array(signal, dtype=float)
def isnontrivialmagnitudebytime(self, iteration, iterationsize, timescale,
trainperiod, deltat, confidence, gobacksec,
sensor, epctag):
nontrivialmagnitude = True
timekey = iteration * iterationsize
if timekey < 0:
return nontrivialmagnitude
if timekey in self.nontrivialmagnitudes:
return self.nontrivialmagnitudes[timekey]
# capture the first epc96 tag seen so that we only process those; must be done before we do constant delta t or any resampling, because this removes the string values from the query... set epctag prior to this point to override with a specific tag
if epctag is None:
rows = sensor.df.query('relative_timestamp >= ' +
str(iteration * timescale * iterationsize -
gobacksec * timescale) +
' and relative_timestamp < ' +
str(iteration * timescale * iterationsize))
if len(rows) <= 0:
return nontrivialmagnitude
epctag = rows['epc96'][0]
rows = sensor.df.query('relative_timestamp >= ' +
str(iteration * timescale * iterationsize -
gobacksec * timescale) +
' and relative_timestamp < ' +
str(iteration * timescale * iterationsize) +
' and epc96 == \"' + epctag + '\"')
if len(rows) <= 0:
return nontrivialmagnitude
rows = sensor.constantdeltat(rows,
deltat=str(int(deltat * timescale)) + 'U')
signal = []
for i, row in rows.iterrows():
signal.append(float(row['rssi_from_mean']))
signal = sensor.interpnan(signal)
powers = []
signal = np.asarray(signal, dtype='float64')
freqs, times, Sxx = scipy.signal.spectrogram(signal,
fs=1.0 / deltat,
nperseg=len(signal),
noverlap=0,
detrend='linear')
for i in range(len(times)):
for j in range(len(Sxx)):
powers.append(Sxx[j][i])
power = np.mean(powers)
if iteration * iterationsize < trainperiod:
self.meanpowers.append(power)
else:
t, p = ttest(power, np.mean(self.meanpowers),
np.var(self.meanpowers), len(self.meanpowers))
if p < 1 - confidence and power < np.mean(
self.meanpowers
): # passes t test, and remove instances in which the sample is smaller than the typical respirating mean (abnormally deep breath is not a cessation anomaly)
nontrivialmagnitude = False
self.nontrivialmagnitudes[timekey] = nontrivialmagnitude
return nontrivialmagnitude
def make_gaussian_random_signal(length, mean=0, sd=1000):
signal = []
for i in range(length):
signal.append(np.random.normal())
return Signal(signal)
def make_uniform_random_signal(length, minimum=-32768, maximum=32768):
signal = []
for i in range(length):
signal.append((random.random() * (maximum-minimum)) + minimum)
return Signal(signal)
return y
fps = 30 #było 30
signal = butter_highpass_filter(lines, 0.2, fps)
print(signal[0:20])
for i in range(0, len(signal)):
time.append(signal[i][0])
values.append(signal[i][1])
signal = []
for i in range(len(time)):
signal.append([time[i], values[i]])
print(signal[0:20])
signal = np.array(signal)
print(signal[0:20])
values = []
for i in range(len(signal)):
values.append(signal[i][1])
data = np.array(values)
def dataset_making(sig):
signal = []
dataset = []
raw = []
ann = wfdb.rdann('INCART/' + sig, 'atr')
sig, fields = wfdb.rdsamp('INCART/' + sig, channels=[5])
for g in sig:
raw.append(g[0])
samplerate = ann.fs
signew = preprocessing(raw, samplerate)
cd = []
for w in range(len(ann.sample)):
types = ann.symbol[w]
array = [ann.sample[w], types]
cd.append(array)
for k in range(len(signew)):
sgt = signew[k]
temps = [k, sgt]
signal.append(temps)
A = cd
B = signal
B_Dict = {b[0]: b for b in B}
array_new = [[B_Dict.get(a[0])[0],
B_Dict.get(a[0])[1], a[1]] for a in A if B_Dict.get(a[0])]
for j in range(len(array_new)):
for k in range(len(array_new[j])):
amplitude = array_new[j][1]
d1 = array_new[j][0]
if (j == 0 or j != (len(array_new) - 1)):
d2 = array_new[j + 1][0]
distance = d2 - d1
RR = (distance / samplerate)
HR = int(60 / RR)
else:
RR = 0
HR = 0
if (j == 0):
toward = array_new[j + 1][1]
toback = 0
if (j == (len(array_new) - 1)):
toward = 0
toback = array_new[j - 1][1]
else:
toward = array_new[j + 1][1]
toback = array_new[j - 1][1]
class_data = (array_new[j][2])
# temp = [amplitude, toback, toward, RR, HR, class_data]
temp = [amplitude, toback, toward, RR, HR, class_data]
dataset.append(temp)
dtn = []
amp = []
bk = []
fr = []
rrt = []
hr = []
cld = []
hrb = []
hra = []
for h in range(len(dataset)):
amp.append(dataset[h][0])
bk.append(dataset[h][1])
fr.append(dataset[h][2])
rrt.append(dataset[h][3])
hr.append(dataset[h][4])
cld.append(dataset[h][5])
for g in range(len(hr)):
if (g == (len(hr) - 1)):
hrb.append(hr[g - 1])
hra.append(0)
elif (g == 0):
hra.append(hr[g + 1])
hrb.append(0)
else:
hra.append(hr[g + 1])
hrb.append(hr[g - 1])
for c in range(len(hr) - 1):
c += 1
typenew = [
amp[c], bk[c], fr[c], rrt[c], hrb[c], hr[c], hra[c],
dtypes(cld[c])
]
dtn.append(typenew)
return dtn
kRedisc = 1.2
N1 = 500
N2 = kRedisc * N1
periodNumber = 1
f1 = N1 / periodNumber
f2 = N2 / periodNumber
tSignal = np.linspace(-3 * np.pi + periodNumber * 2 * np.pi,
-np.pi + periodNumber * 2 * np.pi, N1)
signal = []
signalRedics = []
for i in range(N1):
signal.append(
np.sin(-np.pi + 2 * np.pi * i / N1 + np.random.random_sample() - 1))
signalRedics, tSignalRedisc = resample(signal, tSignal, f1, f2)
plt.figure('Rediscretisation')
plt.subplot(311)
plt.title('signal')
plt.plot(tSignal, signal, 'bo', markersize=2)
plt.plot(tSignal, signal, 'k', linewidth=0.5)
plt.subplot(312)
plt.title('Rediscretisation of signal')
plt.plot(tSignalRedisc, signalRedics, 'ro', markersize=1)
plt.plot(tSignalRedisc, signalRedics, 'k', linewidth=0.5)
plt.subplot(313)
def signal_for_indices(channel, raw_data, indices):
signal = []
channel_data = raw_data[:, channel]
for i in indices:
signal.append(channel_data[i])
return signal
def predictForFrame(self, data):
result = []
#print(data.shape)
for index in range(0, len(data) - sequence_length):
result.append(data[index:index + sequence_length])
result = np.array(result)
result = self.z_norm(result)
X_test = result[:, :-1]
y_test = result[:, -1]
self.X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 6))
if (FFT):
print("start FFT")
X_test = self.X_test
for i in range(0, len(X_test)):
outer = X_test[i, :, :]
for j in range(0, outer.shape[1]):
fft = (np.fft.fft(outer[:, [j]]))
outer[:, [j]] = fft
X_test[i] = outer
self.X_test = X_test
print("finished FFT ", X_test.shape)
predicted = model.predict(self.X_test)
#predicted = np.reshape(predicted, (predicted.size,))
z = []
start = 0
for i in range(0, len(predicted)):
z.append(
abs(predicted[i][0]) + abs(predicted[i][1]) +
abs(predicted[i][2]) + abs(predicted[i][3]) +
abs(predicted[i][4]) + abs(predicted[i][5]))
z = np.array(z)
signal = []
for i in range(0, len(predicted)):
signal.append(
abs(y_test[i][0][0]) + abs(y_test[i][0][1]) +
abs(y_test[i][0][2]) + abs(y_test[i][0][3]) +
abs(y_test[i][0][4]) + abs(y_test[i][0][5]))
mse = ((signal - z)**2)
print(self.anomaly)
# generate threshold
max = 0
min = 0
for i in range(0, len(mse)):
if (mse[i] > max):
max = mse[i]
if (mse[i] < min):
min = mse[i]
# alarm if out of threshold
if (max > self.max or min < self.min):
if not self.anomaly:
self.anomaly = True
self.error = self.error + 1
else:
if self.anomaly:
self.anomaly = False
print(str(self.error) + 'Error =', max, ' Threshold =', self.max)
with open("error_" + output_name + ".txt", 'a') as file:
file.write(str(max) + '\n')
def run_network(model=None, data=None):
global_start_time = time.time()
if data is None:
print('Loading data... ')
# train on first 700 samples and test on next 300 samples (has anomaly)
X_train, y_train, X_test, y_test = get_split_prep_data(
0, 1000, 0, 2000)
else:
X_train, y_train, X_test, y_test = data
print('\nData Loaded. Compiling...\n')
if model is None:
model = build_model()
try:
print("Training...")
model.fit(X_train,
y_train,
batch_size=batch_size,
nb_epoch=epochs,
validation_split=0.05)
print("Predicting...")
predicttime_start = time.time() - global_start_time
predicted = model.predict(X_test)
predicttime_end = time.time() - global_start_time
pduration = predicttime_end - predicttime_start
print("Predict duration", pduration)
print(predicted.shape)
z = []
#######################################################################
start = 0
for i in range(start, len(predicted) + start):
#.append(math.atan2(-predicted [i][1], predicted [i] [0])) #yaw-angle
#z.append(math.atan2(-predicted [i][5], predicted [i] [8]))
#z.append(math.atan2(predicted [i][2], math.sqrt(predicted [i] [8]*predicted [i] [8]+ predicted [i] [5]*predicted [i] [5])))
z.append(
abs(predicted[i][0]) + abs(predicted[i][1]) +
abs(predicted[i][2]) + abs(predicted[i][3]) +
abs(predicted[i][4]) + abs(predicted[i][5]))
z = np.array(z)
signal = []
for i in range(start, len(predicted) + start):
#signal.append(math.atan2(-y_test [i][1], y_test [i] [0]))
#signal.append(math.atan2(-y_test [i][5], y_test [i] [8]))
#signal.append(math.atan2(y_test [i][2], math.sqrt(y_test [i] [8]*y_test [i] [8]+ y_test [i] [5]*y_test [i] [5])))
signal.append(
abs(y_test[i][0]) + abs(y_test[i][1]) + abs(y_test[i][2]) +
abs(y_test[i][3]) + abs(y_test[i][4]) + abs(y_test[i][5]))
# for i in range (0, 400):
# test1.append(y_test[i][2])
# for i in range (0, 400):
# test1.append(predicted[i][0])
#
mse = []
signal = np.array(signal)
print("Reshaping predicted")
predicted = np.reshape(predicted, (predicted.size, ))
except KeyboardInterrupt:
print("prediction exception")
print('Training duration (s) : ', time.time() - global_start_time)
return model, y_test, 0
try:
plt.figure(1)
plt.subplot(311)
plt.title("Actual Test Signal w/Anomalies")
plt.plot(signal, 'b')
#plt.plot(test1, 'b')
plt.subplot(312)
plt.title("Predicted Signal")
plt.plot(z, 'g')
#plt.plot(test2, 'b')
plt.subplot(313)
plt.title("Squared Error")
mse = ((signal - z)**2)
# for i in range (0, len(predicted)):
# mse.append((abs(predicted [i] [0]) - abs(y_test[i][0])+abs(predicted [i] [1]) - abs(y_test[i][1])+abs(predicted [i] [2]) - abs(y_test[i][2])+abs(predicted [i] [3]) - abs(y_test[i][3])+abs(predicted [i] [4]) - abs(y_test[i][4])+abs(predicted [i] [5]) - abs(y_test[i][5])+abs(predicted [i] [6]) - abs(y_test[i][6])+abs(predicted [i] [7]) - abs(y_test[i][7])+abs(predicted [i] [8]) - abs(y_test[i][8])) **2)
mse = normalize(mse)
# print ("mse-length", len(mse))
# #Anomaly Counting
# anomaly_counter =0
# i=0
# while i < len(mse)-1:
# if mse [i] >= 0.5:
# anomaly_counter += 1
# while mse [i] >= 0.45:
# i += 1
# i += 1
#
#
# print ("Anomalies detected:", anomaly_counter)
mse = mse - mse.mean()
plt.plot(mse, 'r')
warnings = []
for i in range(0, len(mse)):
if abs(mse[i]) > 0.1:
line = warnings.append(i)
line.set_alpha(0.5)
for xc in warnings:
plt.axvline(x=xc)
# x = np.arange(len(mse)+1)
# heights = [1] * len(mse)
#
#
#
# cmap = plt.cm.rainbow
# norm = matplotlib.colors.Normalize(vmin=0, vmax=1)
#
# fig, ax = plt.subplots()
# ax.bar( x, heights, width=1.0, color=cmap(norm(mse)))
# ax.set_xticks([])
# ax.set_yticks([])
# ax.set_ylim([0,10])
#
#
# sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm)
# sm.set_array([])
# fig.colorbar(sm)
plt.show()
except Exception as e:
print("plotting exception")
print(str(e))
print('Training duration (s) : ', time.time() - global_start_time)
return model, y_test, predicted
def main():
#generate the random white Gaussian noise
e = whiteNoise()
#generate the signal for t = 0,.....,N-1
signal = []
N = 64 # the initial points in time domain
for t in range(0, N):
signal.append(y(t, e))
#calculate the psd plot by applying fft to the sequence of the signal
psd = np.fft.fft(signal)
#make the frequency array
fxx = fp(len(signal))
print len(signal)
#calculate psd
print 'PSD without padding:', (np.real(sum(psd) / len(psd)))
signal1 = zeroPadding(signal, N)
psd1 = np.fft.fft(signal1)
print len(signal1)
#calculate psd
print 'PSD with padding N:', (np.real(sum(psd1) / len(psd1)))
signal2 = zeroPadding(signal, 3 * N)
psd2 = np.fft.fft(signal2)
print len(signal2)
#calculate psd
print 'PSD with padding 3N:', (np.real(sum(psd2) / len(psd2)))
signal3 = zeroPadding(signal, 5 * N)
psd3 = np.fft.fft(signal3)
print len(signal3)
#calculate psd
print 'PSD with padding 5N:', (np.real(sum(psd3) / len(psd3)))
signal4 = zeroPadding(signal, 7 * N)
psd4 = np.fft.fft(signal4)
print len(signal4)
#calculate psd
print 'PSD with padding 7N:', (np.real(sum(psd4) / len(psd4)))
#plots
fig1 = plt.figure(1, figsize=(12, 6))
plt.plot(e, color='blue')
plt.xlabel('Time (s)')
plt.ylabel('Amplitude')
plt.title('white Gaussian noise e(t)')
fig1.savefig('white_noise.png')
fig2 = plt.figure(2, figsize=(12, 6))
plt.plot(signal, color='blue')
plt.xlabel('Time (s)')
plt.ylabel('y(t)')
plt.title('Signal with padding 7N')
#fig2.savefig('signal_padding_7N.png')
fig3 = plt.figure(3, figsize=(12, 6))
plt.plot(fxx, psd, color='blue')
plt.xlabel('Frequency (Hz)')
plt.ylabel('PSD')
plt.title('Periodogram of y(t) without padding')
#fig3.savefig('periodoram_without_padding.png')
#because psd is symmetric, plot only the 0-pi part because pi to 2*pi is the same
fig4 = plt.figure(4, figsize=(12, 6))
plt.plot(fxx[:int(len(signal) / 2)],
psd[:int(len(signal) / 2)],
color='blue')
plt.xlabel('Frequency (Hz)')
plt.ylabel('PSD')
plt.title('Periodogram of y(t) with padding 7N')
#fig4.savefig('periodoram_wit_padding_7N_half.png')
plt.show()
def main(*args):
tr = {}
whitenoise = std.vector("double")()
template = std.vector("double")()
python_template = []
for i in range(8196):
whitenoise.push_back(random.gauss(0,1))
template.push_back(getgauss(1./(50*sqrt(2*math.pi)),4096., 50., float(i))) #the template needs to have a maximum amplitude of 1
#template.push_back(getgauss(10.,4096., 50., float(i))) #the template needs to have a maximum amplitude of 1
python_template.append(getgauss(1./(50*sqrt(2*math.pi)),4096., 50., float(i)))
#python_template.append(getgauss(10.,4096., 50., float(i)))
signal = std.vector("double")()
for i in range(8196):
signal.push_back( 50*sqrt(2*math.pi)*1.*python_template[i-2000] + whitenoise[i])
#signal.push_back( python_template[i-2000] + whitenoise[i])
filter = KOptimalFilter()
r2hc = KRealToHalfComplexDFT()
r2hc.SetInputPulse(whitenoise)
print 'real to half complex white noise', r2hc.RunProcess()
hcp = KHalfComplexPower()
hcp.SetInputPulse(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
print 'power white noise', hcp.RunProcess()
wp = []
for i in range(hcp.GetOutputPulseSize()):
wp.append(hcp.GetOutputPulse()[i])
tr['noisepower'] = wp
aveNoisePower = np.mean(wp)
avewhitepower = std.vector("double")()
wavepower= []
for i in range(len(wp)):
avewhitepower.push_back(wp[0])
wavepower.append(aveNoisePower)
tr['avenoisepower'] = wavepower
#use ave white noise power
filter.SetNoiseSpectrum(avewhitepower)
#or use the white noise power at this instance
#filter.SetNoiseSpectrum(hcp.GetOutputPulse(), hcp.GetOutputPulseSize())
r2hc.SetInputPulse(template)
print 'real to half complex template', r2hc.RunProcess()
hcp.SetInputPulse(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
print 'power template', hcp.RunProcess()
tmppower = []
for i in range(hcp.GetOutputPulseSize()):
tmppower.append(hcp.GetOutputPulse()[i])
tr['templatepower'] = tmppower
filter.SetTemplateDFT(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
r2hc.SetInputPulse(signal)
print 'real to half complex signal', r2hc.RunProcess()
filter.SetInputPulse(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
print 'building filter', filter.BuildFilter()
print 'applying filter', filter.RunProcess()
output = []
for i in range(filter.GetOutputPulseSize()):
output.append(filter.GetOutputPulse()[i])
tr['amplitude'] = output
wn = []
for i in range(8196):
wn.append(whitenoise[i])
tr['noise'] = wn
tmp = []
for i in range(8196):
tmp.append(template[i])
tr['template'] = tmp
sig = []
for i in range(8196):
sig.append(signal[i])
tr['signal'] = sig
optfil = []
for i in range(filter.GetOptimalFilterSize()):
optfil.append(filter.GetOptimalFilter()[i])
tr['optfilter'] = optfil
hcp.SetInputPulse(filter.GetOptimalFilter(), filter.GetOptimalFilterSize())
print 'get optimal filter power', hcp.RunProcess()
optfilpower = []
for i in range(hcp.GetOutputPulseSize()):
optfilpower.append(hcp.GetOutputPulse()[i])
tr['optfilpower'] = optfilpower
return tr
def make_gaussian_random_signal(length, mean=0, sd=1000):
signal = []
for i in range(length):
signal.append(np.random.normal())
return Signal(signal)
def make_uniform_random_signal(length, minimum=-32768, maximum=32768):
signal = []
for i in range(length):
signal.append((random.random() * (maximum - minimum)) + minimum)
return Signal(signal)
def main(*args):
tr = {}
pulseLength = 8196
### create the pulse template and save it to the return
(template, python_template) = getTemplate(pulseLength)
tr['template'] = python_template
# get a noise pulse
# save it to the return
noisepulse = getNoisePulse(pulseLength)
wn = []
for i in range(pulseLength):
wn.append(noisepulse[i])
tr['noise'] = wn
#create the signal by adding the template to the noise
#save it to the return
signal = createSignal(pulseLength, python_template, noisepulse, 1.)
sig = []
for i in range(pulseLength):
sig.append(signal[i])
tr['signal'] = sig
### calculate the power of the noise pulse for this instance and save it for the return
r2hc = KRealToHalfComplexDFT()
r2hc.SetInputPulse(noisepulse)
print 'real to half complex noisepulse', r2hc.RunProcess()
hcp = KHalfComplexPower()
hcp.SetInputPulse(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
print 'power noise', hcp.RunProcess()
#save it for the return
wp = []
for i in range(hcp.GetOutputPulseSize()):
wp.append(hcp.GetOutputPulse()[i])
tr['noisepower'] = wp
#######
filter = KOptimalFilter()
#### get the average noise power in order to pass it to the filter
avePower = getAverageNoisePower(pulseLength)
filter.SetNoiseSpectrum(avePower)
#save it for the return
ap= []
for i in range(len(wp)):
ap.append(avePower[i])
tr['avenoisepower'] = ap
#### calculate the fourier transform and the power of the pulse template.
#pass the DFT of the transform to the filter and then save the template power in a
#python arry for the return
r2hc.SetInputPulse(template)
print 'real to half complex template', r2hc.RunProcess()
filter.SetTemplateDFT(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
hcp.SetInputPulse(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
print 'power template', hcp.RunProcess()
tmppower = []
for i in range(hcp.GetOutputPulseSize()):
tmppower.append(hcp.GetOutputPulse()[i])
tr['templatepower'] = tmppower
# now calculate the fourier transform of the event signal and pass it to the filter
r2hc.SetInputPulse(signal)
print 'real to half complex signal', r2hc.RunProcess()
filter.SetInputPulse(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
#build the filter and run
print 'building filter', filter.BuildFilter()
print 'applying filter', filter.RunProcess()
#save the output pulse amplitude estimates to the return
output = []
for i in range(filter.GetOutputPulseSize()):
output.append(filter.GetOutputPulse()[i])
tr['amplitude'] = output
#save the optimal filter for the return
optfil = []
for i in range(filter.GetOptimalFilterSize()):
optfil.append(filter.GetOptimalFilter()[i])
tr['optfilter'] = optfil
#save the power of the optimal filter for the return
hcp.SetInputPulse(filter.GetOptimalFilter(), filter.GetOptimalFilterSize())
print 'get optimal filter power', hcp.RunProcess()
optfilpower = []
for i in range(hcp.GetOutputPulseSize()):
optfilpower.append(hcp.GetOutputPulse()[i])
tr['optfilpower'] = optfilpower
return tr
def main(*args):
tr = {}
tr['nyquist'] = 50000 #50 kHz is the nyquist frequncy since we have a sample rate of 100 kHz
### create the pulse template and save it to the return
(template, python_template) = getTemplate()
tr['template'] = python_template
pulseLength = len(python_template) #get the length from the template
# get a noise pulse
# save it to the return
random.seed()
noisepulse = getNoisePulse(pulseLength)
wn = []
for i in range(pulseLength):
wn.append(noisepulse[i])
tr['noise'] = wn
#create the signal by adding the template to the noise
#save it to the return
signal = createSignal(pulseLength, python_template, noisepulse, 5000.)
signal2 = createSignal(pulseLength, python_template, noisepulse, 10000.)
sig = []
for i in range(pulseLength):
sig.append(signal[i])
tr['signal'] = sig
### calculate the power of the noise pulse for this instance and save it for the return
r2hc = KRealToHalfComplexDFT()
r2hc.SetInputPulse(noisepulse)
print 'real to half complex noisepulse', r2hc.RunProcess()
hcp = KHalfComplexPower()
hcp.SetInputPulse(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
print 'power noise', hcp.RunProcess()
#save it for the return
wp = []
for i in range(hcp.GetOutputPulseSize()):
wp.append(hcp.GetOutputPulse()[i])
tr['noisepower'] = wp
#######
filter = KOptimalFilter()
#### get the average noise power in order to pass it to the filter
avePower = getAverageNoisePower(pulseLength)
## use the same noise as added to pulse
#avePower = std.vector("double")()
#for i in range(len(wp)):
# avePower.push_back(wp[i])
###
filter.SetNoiseSpectrum(avePower)
#save it for the return
ap= []
for i in range(len(wp)):
ap.append(avePower[i])
tr['avenoisepower'] = ap
#### calculate the fourier transform and the power of the pulse template.
#pass the DFT of the transform to the filter and then save the template power in a
#python arry for the return
r2hc.SetInputPulse(template)
print 'real to half complex template', r2hc.RunProcess()
filter.SetTemplateDFT(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
hcp.SetInputPulse(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
print 'power template', hcp.RunProcess()
tmppower = []
for i in range(hcp.GetOutputPulseSize()):
tmppower.append(hcp.GetOutputPulse()[i])
tr['templatepower'] = tmppower
# now calculate the fourier transform of the event signal and pass it to the filter
r2hc.SetInputPulse(signal)
print 'real to half complex signal', r2hc.RunProcess()
filter.SetInputPulse(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
#build the filter and run
print 'building filter', filter.BuildFilter()
print 'applying filter', filter.RunProcess()
#save the output pulse amplitude estimates to the return
output = []
for i in range(filter.GetOutputPulseSize()):
output.append(filter.GetOutputPulse()[i])
tr['amplitude'] = output
#save the optimal filter for the return
optfil = []
for i in range(filter.GetOptimalFilterSize()):
optfil.append(filter.GetOptimalFilter()[i])
tr['optfilter'] = optfil
#save the power of the optimal filter for the return
hcp.SetInputPulse(filter.GetOptimalFilter(), filter.GetOptimalFilterSize())
print 'get optimal filter power', hcp.RunProcess()
optfilpower = []
for i in range(hcp.GetOutputPulseSize()):
optfilpower.append(hcp.GetOutputPulse()[i])
tr['optfilpower'] = optfilpower
# now calculate the fourier transform of the event signal and pass it to the filter
r2hc.SetInputPulse(signal2)
print 'real to half complex signal', r2hc.RunProcess()
filter.SetInputPulse(r2hc.GetOutputPulse(), r2hc.GetOutputPulseSize())
print 'output pulse', filter.GetOutputPulse(), filter.GetOutputPulseSize()
#build the filter and run
print 'building filter', filter.BuildFilter()
print 'applying filter', filter.RunProcess()
print 'output pulse', filter.GetOutputPulse()
#save the output pulse amplitude estimates to the return
output2 = []
for i in range(filter.GetOutputPulseSize()):
output2.append(filter.GetOutputPulse()[i])
tr['amplitude2'] = output2
return tr
