def MFCC_test():
p_ = preprocess.PreProcess(['MFCC'])
s_ = p_.forward(x)
x_hat = p_.inverse(s_)
print(x_hat)
import pdb
pdb.set_trace()
print(torch.max(x_hat[0, 0:1000] - x[0, 0:1000]))
def get_spectrogram():
nem = "17"
fs, x = get_audio_clean(nem)
fs, x_adv = get_audio_adv(nem)
x_noise = x - x_adv
T = preprocess.PreProcess(['spectrogram', 'insert_data_dim', 'mag2db'])
T.stft_n_fft = 128
z = T.forward(np.expand_dims(x, axis=0))
z_adv = T.forward(np.expand_dims(x_adv, axis=0))
z_noise = T.forward(x_noise)
min_clip = -40
z = np.clip(z, min_clip, 10000)
z_adv = np.clip(z_adv, min_clip, 10000)
z_noise = np.clip(z_noise, min_clip, 10000)
minima = np.array([z.min(), z_adv.min(), z_noise.min()]).min()
z_squ = np.squeeze(z) - minima
z_adv_squ = np.squeeze(z_adv) - minima
z_noise_squ = np.squeeze(z_noise) - minima
maxima = np.array([z_squ.max(), z_adv_squ.max(), z_noise_squ.max()]).max()
#fig=plt.figure(figsize=(8, 8))
fig = plt.figure()
rows = 3
import pdb
pdb.set_trace()
# clean
fig.add_subplot(rows, 1, 1)
#plt.imshow(z_squ/maxima, origin="lower",vmin=0, vmax=1)
plt.specgram(x, Fs=fs)
# adv
fig.add_subplot(rows, 1, 2)
#plt.imshow(z_adv_squ/maxima, origin="lower",vmin=0, vmax=1)
plt.specgram(x_adv, Fs=fs)
# noise
fig.add_subplot(rows, 1, 3)
#plt.imshow((z_squ/maxima - z_adv_squ/maxima) * 1/(z_squ/maxima - z_adv_squ/maxima).max(), origin="lower",vmin=0, vmax=1)
#plt.imshow(z_noise_squ/maxima, origin="lower",vmin=0, vmax=1)
plt.specgram(x_noise, Fs=fs)
plt.show()
fig_z = plt.figure()
plt.imshow(z, origin="lower", vmin=0, vmax=maxima)
plt.savefig("spectrogram_x.pdf")
fig_z_adv = plt.figure()
plt.imshow(z_adv, origin="lower", vmin=0, vmax=maxima)
plt.savefig("spectrogram_x_adv.pdf")
fig_z_noise = plt.figure()
plt.imshow(z_noise, origin="lower", vmin=0, vmax=maxima)
plt.savefig("spectrogram_x_noise.pdf")
def main():
preprocessor = preprocess.PreProcess(window_size)
device = torch.device("cuda:" +
str(dev) if torch.cuda.is_available() else "cpu")
model = ABHUE()
model = model.to(device)
model.load_state_dict(torch.load(savePath))
model.eval()
batch_datas = []
label_datas = []
data_idx = 0
batch_count = 0
for data_file in os.listdir(data_dir):
for data in preprocessor.parseData(os.path.join(data_dir, data_file)):
if not preprocessor.create_sliding_window(data):
continue
data_input, data_label = preprocessor.tensorfy()
output = model(data_input)
batch_datas.append((data_idx, output.detach()))
label_datas.append((data_idx, data_label))
num_speakers = CheckNumOfSpeakers(label_datas)
if data_idx + 1 >= batch_size or num_speakers >= max_speakers:
km = Kmeans(k=num_speakers, size=200)
km.run(batch_datas)
score = bestLabels(km.clusters, label_datas, num_speakers)
# round(score / (data_idx + 1), 10)
print(
'[{}] Inference Score: {} \t Batch Size: {} \t Speakers: {}'
.format((batch_count + 1), score, data_idx + 1,
num_speakers)) #
batch_count += 1
data_idx = 0
batch_datas = []
label_datas = []
else:
data_idx += 1
print("")
break
'%Y-%m-%d %H:%M:%S.%f')
cur.execute(
"INSERT INTO pontos2 (lat, lng, instante, rota, posicao, velocidade,viaje, matricula_id) VALUES (%s, %s, %s, %s, %s, %s,%s, %s)",
(all_data.lat.values[i], all_data.lng.values[i], time,
all_data.Ruta.values[i], all_data.Posicion.values[i],
all_data.Velocidad.values[i], all_data.Viaje.values[i],
all_data.matricula.values[i]))
conn.commit()
if __name__ == "__main__":
conn = ps.connect("dbname=urbanmobility user=postgres password=242124")
cur = conn.cursor()
print('Criando banco de dados...')
cur.execute(
"CREATE TABLE linha2 (matricula varchar PRIMARY KEY, unidade numeric, nome varchar, estado numeric,linha numeric );"
)
cur.execute(
"CREATE TABLE pontos2 (id serial PRIMARY KEY, lat numeric, lng numeric, instante timestamp, rota numeric, posicao numeric, velocidade numeric,viaje numeric, matricula_id varchar );"
)
conn.commit()
data_frame = pd.read_csv('011017_011117.csv', sep=';')
print('Fazendo o cleaning dos dados...')
new_data = prep.PreProcess().clean_data(data_frame)
print('Convertendo coordenadas para latitude e longitude')
lat_lng = prep.PreProcess().coordinates_to_latlng(new_data)
print('Salvando os dados no banco de dados....')
save_in_db(lat_lng)
import pandas as pd
import preprocess as prep
from tqdm import tqdm
import numpy as np
import gmplot.gmplot
traffic_stops_data = pd.read_csv('semaforos.csv', sep=';')
data = pd.read_csv('data-100000-bus-stop-mapped.csv',delimiter=',', float_precision=None)
# data['label'] = ['in_route' for i in range(data.shape[0])]
prepro = prep.PreProcess()
for idx, row in tqdm(data.iterrows()):
if row.velocidade < 5:
for idx2,stop in traffic_stops_data.iterrows():
print('indice:{0}'.format(idx2))
print('values:{0}'.format(stop.Latitude, stop.Longitude))
if idx2 != 292:
dist = prepro.distance_in_meters([row.lat,row.lng], [stop.Latitude,stop.Longitude])
if dist < 30 and row.label != 'bus_stop':
data.loc[idx,'label'] = 'traffic_light'
data.to_csv('data-100000-traffic-light.csv')
elif run_raw and len(args.clean) != 0:
logger.error(
"--raw is set to True and --clean recieved 1 input. To run both set --clean with 0 input"
)
sys.exit()
else:
logger.error("--clean args requires 0 or 1 csv")
sys.exit()
logger.debug(f"Running --raw: {run_raw}")
logger.debug(f"Running --clean: {run_clean}")
logger.debug(f"Running --raw and --clean: {run_raw_and_clean}")
if run_raw:
if not run_raw_and_clean:
p = preprocess.PreProcess(args.raw)
p.combine_df().to_csv('Preprocess.csv', index=False)
print('Preprocess complete. Preprocess.csv file saved')
else:
print('Two step process, preprocess and feature engineer, will run')
if run_clean:
f = feature_eng.FeatEng(args.clean)
f.feateng_update().to_csv('Clean_data.csv', index=False)
print('Feature engineer complete. Clean_data.csv file saved')
if run_raw_and_clean:
p = preprocess.PreProcess(args.raw)
# Code for debugging purposes
# result = pd.read_csv('Train_Preprocess.csv',parse_dates=['ClaimStartDt','ClaimEndDt','AdmissionDt','DOD','DischargeDt'])
args = parser.parse_args()
assert args.prefix, "provide prefix"
filenums = map(int, args.filenums.split(','))
codewords_fname = '%s-%s' % (args.prefix, 'acc_beam_codewords.h5')
if args.skipcw:
codeword_datasets = {
'yag': ('yag_codeword1', 'yag_codeword2'),
'vcc': ('vcc_codeword1', 'vcc_codeword2')
}
else:
assert args.alg and args.alg in preprocess.ALGS.keys(
), "provide algorithm, one of %r, see preprocess.py" % preprocess.ALGS.keys(
)
preprocess = preprocess.PreProcess(alg=args.alg, final_shape=(224, 224))
accdata = beamdata.BeamData(preprocess=preprocess,
datadir='data',
subbkg=args.subbkg,
filenums=filenums,
prefix=args.prefix,
force=args.force,
nn=args.nn)
accdata.loadall(reload=False)
if args.viewbkg:
accdata.viewbkg()
if args.view or args.save > 0:
def spectrogram_test():
p = preprocess.PreProcess(['spectrogram'])
s = p.forward(x)
x_hat = p.inverse(s)
print(np.max(x_hat - x))
def stft_test():
p = preprocess.PreProcess(['stft'])
s = p.forward(x)
x_hat = p.inverse(s)
print(np.max(x_hat - x))
import numpy as np
import masking
import torch
import preprocess
t_x = torch.ones((3, 2, 15, 17))
t_y = torch.ones((3))
t_p = preprocess.PreProcess(['None'])
def differential_evolution(x,
y,
pre,
N_dimensions_value,
N_perturbations,
N_iterations,
N_population,
targeted=False,
x_min=-1,
x_max=1,
train=False):
def evolve(p_pos, p_val, F=0.5):
#p_pos = [B, N, K, 3]
c_pos = np.copy(p_pos)
c_val = np.copy(p_val)
# pick out random individuals from population
idxs = np.random.choice(
I, size=(I, 3)) # use same random process for whole batch
x_pos = p_pos[:, idxs, :, :]
x_val = p_val[:, idxs, :, :]
def normalize(text):
preprocessed = preprocess.PreProcess(text)
preprocessed.normalize_letter()
preprocessed.remove_punctuation()
return preprocessed
def Preprocessing_Initialization(self, glove_data=None):
self.preprocessor = preprocess.PreProcess(self.window_size,
dev=self.device,
glove_data=glove_data)