def dataGenerate(self, occupancy, lock):
algorithm = Algorithms()
generator = Signal()
const = self.const
methods = const['methods']
iterations = const['iterations']
b = const['b']
e = const['e']
h = const['h']
H = const['H']
A = const['A']
B = const['B']
fill = const['fill']
fillAd, fillAe, fillCd, fillCe = fill
halfA = len(fillAd)
matrix = const['matrix']
window = const['window']
totalSamples = const['totalSamples']
signalT, signalN = generator.signalGenerator(totalSamples, b, fillAd, fillAe, matrix, exp_mean=occupancy)
signalTf, signalNf = generator.signalGenerator(totalSamples, b, fillAd, fillAe, matrix, exp_mean=occupancy)
nnzST = np.count_nonzero(signalT)
nzST = len(signalT) - nnzST
pike = int(np.max(signalN) + 1)
for method in methods:
if 'FIR' in method:
signalA = algorithm.FIR(26, signalNf, signalTf, signalN)
data = generator.roc(signalA, signalT)
data = data.rename(columns={s: 'FIR:' + str(occupancy) + ':' + s for s in list(data.columns.values)})
with lock:
self.addData(data)
elif 'MF' in method:
signalMf, data = algorithm.MatchedFw_roc(signalN, h, totalSamples, b, e, fill, signalT)
data = data.rename(columns={s: 'MF:' + str(occupancy) + ':' + s for s in list(data.columns.values)})
with lock:
self.addData(data)
elif 'MP' in method:
threshold, pdr, far = 0, 5, 5
res = []
while (float('{:.2f}'.format(pdr)) > 0.01) or (float('{:.2f}'.format(far)) > 0.01):
faA, pdA = 0, 0
signalA = np.zeros(window * totalSamples)
for i in range(totalSamples):
step = (i * window)
pace = step + window
if (e > 6):
pace = pace - (e - 6)
signalTw = np.concatenate((fillCd, signalT[step:pace], fillCe))
signalSw = np.concatenate((fillCd, signalN[step:pace], fillCe))
step += halfA
pace = step + b
if 'LS-OMP' in method:
x, fa, pd = algorithm.LS_OMP_roc(threshold, signalSw, signalTw, b, H)
elif 'OMP' in method:
x, fa, pd = algorithm.OMP_roc(threshold, signalSw, signalTw, b, H)
elif 'MP' in method:
x, fa, pd = algorithm.MP_roc(threshold, signalSw, signalTw, b, H)
signalA[step:pace] = x
faA += fa
pdA += pd
far = (faA / nzST)
pdr = (pdA / nnzST)
tmp = '%d,%.6f,%.6f,%.6f' % (threshold, generator.rms(signalA - signalT), far, pdr)
if threshold < pike:
threshold += 1
else:
if threshold == pike:
threshold = math.ceil(pike / 100) * 100
else:
threshold += 100
res.append([float(s) for s in tmp.split(',')])
data = panda.DataFrame(res, columns=[method + ':' + str(occupancy) + ':threshold',
method + ':' + str(occupancy) + ':RMS',
method + ':' + str(occupancy) + ':FA',
method + ':' + str(occupancy) + ':DP'])
with lock:
self.addData(data)
else:
signalA = np.zeros(window * totalSamples)
mi = 0.25
lam = 0.0
for i in range(totalSamples):
step = (i * window)
pace = step + window
if (e > 6):
pace = pace - (e - 6)
signalS = np.concatenate((fillCd, signalN[step:pace], fillCe))
step += halfA
pace = step + b
xAll = signalS[3:b + 3]
Hs = H.T.dot(signalS)
if 'i' in method:
x = B.dot(signalS)
else:
x = xAll
for it in range(iterations):
if 'GD' in method:
x = algorithm.GD(x, Hs, A, mi)
elif 'SSF' in method:
x = algorithm.SSF(x, Hs, A, mi, lam)
elif 'PCD' in method:
x = algorithm.PCD(x, Hs, A, mi, lam)
elif 'TAS' in method:
x = algorithm.TAS(x, Hs, A, mi, lam)
signalA[step:pace] = x
data = generator.roc(signalA, signalT)
data = data.rename(columns={s: method + ':' + str(occupancy) + ':' + s for s in list(data.columns.values)})
with lock:
self.addData(data)
def rodar(patterns, radical, sinais, occupancy, lock, metodos, load=None):
u = Utiliters()
info = u.setup_logger('information', radical + 'info.log')
startedI = datetime.now()
print('Started ROC generate for occupancy %d at %s' %
(occupancy, startedI.strftime("%H:%M:%S %d/%m/%Y")))
info.info('Started ROC generate for occupancy %d' % (occupancy))
algo = Algorithms()
gerador = Signal()
matrizes = Matrizes()
matrix = matrizes.matrix()
samples = 1820
for pattern in patterns:
if load is None:
data = panda.DataFrame([])
nome = radical + pattern + '_' + str(occupancy) + '.csv'
else:
nome = load + pattern + '_' + str(occupancy) + '.csv'
data = panda.read_csv(nome)
bunch = pattern.rsplit('b', 1)
empty = bunch[1].rsplit('e', 1)
b = int(bunch[0])
e = int(empty[0])
bwindow = b + 6
window = b + e
halfA = e - int(math.ceil(e / 2))
halfCd = int(math.ceil((bwindow - window) / 2))
halfCe = int(bwindow - window - halfCd)
if halfCd > 0:
fillCd = np.zeros(halfCd)
else:
fillCd = np.arange(0)
if halfCe > 0:
fillCe = np.zeros(halfCe)
else:
fillCe = np.arange(0)
fillAd = np.zeros(halfA)
fillAe = np.zeros(e - halfA)
fill = [fillAd, fillAe, fillCd, fillCe]
H, A, B = matrizes.generate(b)
h = matrix[0:7, 5]
try:
signalT = np.genfromtxt(sinais + 'signalT_' + pattern + '_' +
str(occupancy) + '.csv',
delimiter=',')
signalN = np.genfromtxt(sinais + 'signalN_' + pattern + '_' +
str(occupancy) + '.csv',
delimiter=',')
signalTf = np.genfromtxt(sinais + 'fir/signalT_' + pattern + '_' +
str(occupancy) + '.csv',
delimiter=',')
signalNf = np.genfromtxt(sinais + 'fir/signalN_' + pattern + '_' +
str(occupancy) + '.csv',
delimiter=',')
except:
print('Error get saved signals')
signalT, signalN, signalTf, signalNf = u.sgen(
pattern, samples, b, fillAd, fillAe, matrix, sinais)
const = {
'iterations': 331,
'occupancy': occupancy,
'pattern': pattern,
'signalT': signalT,
'signalN': signalN
}
opt = {'samples': samples}
nnzST = np.count_nonzero(signalT)
nzST = len(signalT) - nnzST
pike = int(np.max(signalN) + 1)
if ('FDIP' in metodos) or ('MF' in metodos):
started = datetime.now()
print('Started ROC generate for old with occupancy %d at %s' %
(occupancy, started.strftime("%H:%M:%S %d/%m/%Y")))
info.info('Started ROC generate for old with occupancy %d' %
(occupancy))
metodo = 'FDIP'
if metodo in metodos:
const['metodo'] = metodo
constf = const
constf['signalNf'] = signalNf
constf['signalTf'] = signalTf
roc = gerador.roc(algo.getRMSfloat(constf)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'MF'
if metodo in metodos:
signalMf, roc = algo.MatchedFw_roc(signalN, h, samples, b, e,
fill, signalT)
roc = roc.rename(
columns={
s: 'MF:' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
ended = datetime.now()
print(
'Ended ROC generate for old with occupancy %d at %s after %s'
% (occupancy, ended.strftime("%H:%M:%S %d/%m/%Y"),
u.totalTime(started)))
info.info('Ended ROC generate for old with occupancy %d after %s' %
(occupancy, u.totalTime(started)))
if (' MF ' in metodos) or ('OMP ' in metodos) or ('LS-OMP' in metodos):
started = datetime.now()
print('Started ROC generate for Greedy with occupancy %d at %s' %
(occupancy, started.strftime("%H:%M:%S %d/%m/%Y")))
info.info('Started ROC generate for Greedy with occupancy %d' %
(occupancy))
metodo = ' MP '
if metodo in metodos:
threshold, pdr, far = 0, 5, 5
res = []
while (float('{:.2f}'.format(pdr)) > 0.01) or (float(
'{:.2f}'.format(far)) > 0.01):
faA, pdA = 0, 0
signalA = np.zeros(window * samples)
for i in range(samples):
step = (i * window)
paso = step + window
if (e > 6):
paso = paso - (e - 6)
signalTw = np.concatenate(
(fillCd, signalT[step:paso], fillCe))
signalSw = np.concatenate(
(fillCd, signalN[step:paso], fillCe))
step += halfA
paso = step + b
x, fa, pd = algo.MP_roc(threshold, signalSw, signalTw,
b, H)
signalA[step:paso] = x
faA += fa
pdA += pd
far = (faA / nzST)
pdr = (pdA / nnzST)
tmp = '%d,%.6f,%.6f,%.6f' % (
threshold, gerador.rms(signalA - signalT), far, pdr)
if threshold < pike:
threshold += 1
else:
if threshold == pike:
threshold = math.ceil(pike / 100) * 100
else:
threshold += 100
res.append([float(s) for s in tmp.split(',')])
roc = panda.DataFrame(
res,
columns=[
'MP:' + str(occupancy) + ':threshold',
'MP:' + str(occupancy) + ':RMS',
'MP:' + str(occupancy) + ':FA',
'MP:' + str(occupancy) + ':DP'
])
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'OMP '
if metodo in metodos:
threshold, pdr, far = 0, 5, 5
res = []
while (float('{:.2f}'.format(pdr)) > 0.01) or (float(
'{:.2f}'.format(far)) > 0.01):
faA, pdA = 0, 0
signalA = np.zeros(window * samples)
for i in range(samples):
step = (i * window)
paso = step + window
if (e > 6):
paso = paso - (e - 6)
signalTw = np.concatenate(
(fillCd, signalT[step:paso], fillCe))
signalSw = np.concatenate(
(fillCd, signalN[step:paso], fillCe))
step += halfA
paso = step + b
x, fa, pd = algo.OMP_roc(threshold, signalSw, signalTw,
b, H)
signalA[step:paso] = x
faA += fa
pdA += pd
far = (faA / nzST)
pdr = (pdA / nnzST)
tmp = '%d,%.6f,%.6f,%.6f' % (
threshold, gerador.rms(signalA - signalT), far, pdr)
if (threshold < pike):
threshold += 1
else:
if threshold == pike:
threshold = math.ceil(pike / 100) * 100
else:
threshold += 100
res.append([float(s) for s in tmp.split(',')])
roc = panda.DataFrame(
res,
columns=[
'OMP:' + str(occupancy) + ':threshold',
'OMP:' + str(occupancy) + ':RMS',
'OMP:' + str(occupancy) + ':FA',
'OMP:' + str(occupancy) + ':DP'
])
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'LS-OMP'
if metodo in metodos:
threshold, pdr, far = 0, 5, 5
res = []
while (float('{:.2f}'.format(pdr)) > 0.01) or (float(
'{:.2f}'.format(far)) > 0.01):
faA, pdA = 0, 0
signalA = np.zeros(window * samples)
for i in range(samples):
step = (i * window)
paso = step + window
if (e > 6):
paso = paso - (e - 6)
signalTw = np.concatenate(
(fillCd, signalT[step:paso], fillCe))
signalSw = np.concatenate(
(fillCd, signalN[step:paso], fillCe))
step += halfA
paso = step + b
x, fa, pd = algo.LS_OMP_roc(threshold, signalSw,
signalTw, b, H)
signalA[step:paso] = x
faA += fa
pdA += pd
far = (faA / nzST)
pdr = (pdA / nnzST)
tmp = '%d,%.6f,%.6f,%.6f' % (
threshold, gerador.rms(signalA - signalT), far, pdr)
if (threshold < pike):
threshold += 1
else:
if threshold == pike:
threshold = math.ceil(pike / 100) * 100
else:
threshold += 100
res.append([float(s) for s in tmp.split(',')])
roc = panda.DataFrame(
res,
columns=[
'LS-OMP:' + str(occupancy) + ':threshold',
'LS-OMP:' + str(occupancy) + ':RMS',
'LS-OMP:' + str(occupancy) + ':FA',
'LS-OMP:' + str(occupancy) + ':DP'
])
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
ended = datetime.now()
print(
'Ended ROC generate for Greedy with occupancy %d at %s after %s'
% (occupancy, ended.strftime("%H:%M:%S %d/%m/%Y"),
u.totalTime(started)))
info.info(
'Ended ROC generate for Greedy with occupancy %d after %s' %
(occupancy, u.totalTime(started)))
started = datetime.now()
print('Started ROC generate for Shrinkage with occupancy %d at %s' %
(occupancy, started.strftime("%H:%M:%S %d/%m/%Y")))
info.info('Started ROC generate for Shrinkage with occupancy %d' %
(occupancy))
metodo = 'DS'
if metodo in metodos:
const['metodo'] = metodo
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'GD '
if metodo in metodos:
metodo = metodo.strip()
const['metodo'] = metodo
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'GDi'
if metodo in metodos:
metodo = metodo.strip()
const['metodo'] = metodo
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'SSF '
if metodo in metodos:
metodo = metodo.strip()
const['metodo'] = metodo
opt['mi'] = .25
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'SSFi'
if metodo in metodos:
metodo = metodo.strip()
const['metodo'] = metodo
opt['mi'] = .25
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'SSFls '
if metodo in metodos:
metodo = metodo.strip()
const['metodo'] = metodo
opt['mi'] = .25
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'SSFlsi'
if metodo in metodos:
metodo = metodo.strip()
const['metodo'] = metodo
opt['mi'] = .25
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'SSFlsc '
if metodo in metodos:
metodo = metodo.strip()
const['metodo'] = metodo
opt['mi'] = .25
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'SSFlsci'
if metodo in metodos:
metodo = metodo.strip()
const['metodo'] = metodo
opt['mi'] = .25
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'PCD '
if metodo in metodos:
metodo = metodo.strip()
const['metodo'] = metodo
opt['mi'] = math.inf
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
metodo = 'PCDi'
if metodo in metodos:
metodo = metodo.strip()
const['metodo'] = metodo
opt['mi'] = math.inf
roc = gerador.roc(algo.getRMSfloat(const, opt)['signal'], signalT)
roc = roc.rename(
columns={
s: metodo + ':' + str(occupancy) + ':' + s
for s in list(roc.columns.values)
})
data = panda.concat([data, roc], axis=1, sort=False)
with lock:
data.to_csv(nome, index=False)
ended = datetime.now()
print(
'Ended ROC generate for Shrinkage with occupancy %d at %s after %s'
% (occupancy, ended.strftime("%H:%M:%S %d/%m/%Y"),
u.totalTime(started)))
info.info(
'Ended ROC generate for Shrinkage with occupancy %d after %s' %
(occupancy, u.totalTime(started)))
ended = datetime.now()
print('Ended ROC generate for occupancy %d at %s after %s' %
(occupancy, ended.strftime("%H:%M:%S %d/%m/%Y"),
u.totalTime(startedI)))
info.info('Ended ROC generate for occupancy %d after %s' %
(occupancy, u.totalTime(startedI)))
# signalT, signalN = gerador.signalGenerator(totalSamples, b, fillAd, fillAe, matrix)
# signalTf, signalNf = gerador.signalGenerator(totalSamples, b, fillAd, fillAe, matrix)
# np.savetxt(path + 'ttsignalT.csv', signalT, delimiter=',')
# np.savetxt(path + 'ttsignalN.csv', signalN, delimiter=',')
# np.savetxt(path + 'fir/ttsignalT.csv', signalTf, delimiter=',')
# np.savetxt(path + 'fir/ttsignalN.csv', signalNf, delimiter=',')
signalT = np.genfromtxt(path + 'vosignalT_8b4e_30.csv', delimiter=',')
signalN = np.genfromtxt(path + 'vosignalN_8b4e_30.csv', delimiter=',')
signalTf = np.genfromtxt(path + 'fir/vosignalT_8b4e_30.csv', delimiter=',')
signalNf = np.genfromtxt(path + 'fir/vosignalN_8b4e_30.csv', delimiter=',')
print('RMS without Filter =', gerador.rms(signalN - signalT))
graphTese(signalT, signalN, signalN, 'Noise', 'v_sinal_obtido')
signalMfw, roc = algo.MatchedFw_roc(signalN, h, totalSamples, b, e, fill,
signalT)
m_rms = np.nanmin(roc['RMS'])
t_rms = roc.loc[roc['RMS'] == m_rms]['threshold'].tolist()[0]
signalMf = algo.MatchedFw(signalN, h, t_rms, totalSamples, b, e, fill)
print('RMS of Matched Filter =', gerador.rms(signalMf - signalT))
# graphTese(signalT, signalN, signalMf, 'Noise', 'Matched Filter')
graphTese(signalT, signalMf, signalMf, 'Matched Filter',
'v_matched_filter')
signalF = algo.FIR(26, signalNf, signalTf, signalN)
print('RMS of Fir Filter =', gerador.rms(signalF - signalT))
# graphTese(signalT, signalMf, signalF, 'Matched Filter', 'FIR order 26')
graphTese(signalT, signalF, signalF, 'Filter FDIP order 26',
'v_fdip_order_26')
signalM = np.zeros(window * totalSamples)