Пример #1
0
    def MFCC_test():
        p_ = preprocess.PreProcess(['MFCC'])
        s_ = p_.forward(x)

        x_hat = p_.inverse(s_)
        print(x_hat)
        import pdb
        pdb.set_trace()
        print(torch.max(x_hat[0, 0:1000] - x[0, 0:1000]))
Пример #2
0
def get_spectrogram():
    nem = "17"
    fs, x = get_audio_clean(nem)
    fs, x_adv = get_audio_adv(nem)
    x_noise = x - x_adv

    T = preprocess.PreProcess(['spectrogram', 'insert_data_dim', 'mag2db'])
    T.stft_n_fft = 128

    z = T.forward(np.expand_dims(x, axis=0))
    z_adv = T.forward(np.expand_dims(x_adv, axis=0))
    z_noise = T.forward(x_noise)

    min_clip = -40
    z = np.clip(z, min_clip, 10000)
    z_adv = np.clip(z_adv, min_clip, 10000)
    z_noise = np.clip(z_noise, min_clip, 10000)

    minima = np.array([z.min(), z_adv.min(), z_noise.min()]).min()

    z_squ = np.squeeze(z) - minima
    z_adv_squ = np.squeeze(z_adv) - minima
    z_noise_squ = np.squeeze(z_noise) - minima

    maxima = np.array([z_squ.max(), z_adv_squ.max(), z_noise_squ.max()]).max()

    #fig=plt.figure(figsize=(8, 8))
    fig = plt.figure()
    rows = 3
    import pdb
    pdb.set_trace()

    # clean
    fig.add_subplot(rows, 1, 1)
    #plt.imshow(z_squ/maxima, origin="lower",vmin=0, vmax=1)
    plt.specgram(x, Fs=fs)
    # adv
    fig.add_subplot(rows, 1, 2)
    #plt.imshow(z_adv_squ/maxima, origin="lower",vmin=0, vmax=1)
    plt.specgram(x_adv, Fs=fs)
    # noise
    fig.add_subplot(rows, 1, 3)
    #plt.imshow((z_squ/maxima - z_adv_squ/maxima) * 1/(z_squ/maxima - z_adv_squ/maxima).max(), origin="lower",vmin=0, vmax=1)
    #plt.imshow(z_noise_squ/maxima, origin="lower",vmin=0, vmax=1)
    plt.specgram(x_noise, Fs=fs)
    plt.show()

    fig_z = plt.figure()
    plt.imshow(z, origin="lower", vmin=0, vmax=maxima)
    plt.savefig("spectrogram_x.pdf")
    fig_z_adv = plt.figure()
    plt.imshow(z_adv, origin="lower", vmin=0, vmax=maxima)
    plt.savefig("spectrogram_x_adv.pdf")
    fig_z_noise = plt.figure()
    plt.imshow(z_noise, origin="lower", vmin=0, vmax=maxima)
    plt.savefig("spectrogram_x_noise.pdf")
Пример #3
0
def main():
    preprocessor = preprocess.PreProcess(window_size)

    device = torch.device("cuda:" +
                          str(dev) if torch.cuda.is_available() else "cpu")
    model = ABHUE()
    model = model.to(device)
    model.load_state_dict(torch.load(savePath))
    model.eval()

    batch_datas = []
    label_datas = []
    data_idx = 0
    batch_count = 0
    for data_file in os.listdir(data_dir):
        for data in preprocessor.parseData(os.path.join(data_dir, data_file)):
            if not preprocessor.create_sliding_window(data):
                continue

            data_input, data_label = preprocessor.tensorfy()

            output = model(data_input)

            batch_datas.append((data_idx, output.detach()))
            label_datas.append((data_idx, data_label))

            num_speakers = CheckNumOfSpeakers(label_datas)
            if data_idx + 1 >= batch_size or num_speakers >= max_speakers:
                km = Kmeans(k=num_speakers, size=200)
                km.run(batch_datas)
                score = bestLabels(km.clusters, label_datas, num_speakers)
                # round(score / (data_idx + 1), 10)
                print(
                    '[{}] Inference Score: {} \t Batch Size: {} \t Speakers: {}'
                    .format((batch_count + 1), score, data_idx + 1,
                            num_speakers))  #
                batch_count += 1
                data_idx = 0
                batch_datas = []
                label_datas = []
            else:
                data_idx += 1
        print("")
        break
Пример #4
0
                                        '%Y-%m-%d %H:%M:%S.%f')
            cur.execute(
                "INSERT INTO pontos2 (lat, lng, instante, rota, posicao, velocidade,viaje, matricula_id) VALUES (%s, %s, %s, %s, %s, %s,%s, %s)",
                (all_data.lat.values[i], all_data.lng.values[i], time,
                 all_data.Ruta.values[i], all_data.Posicion.values[i],
                 all_data.Velocidad.values[i], all_data.Viaje.values[i],
                 all_data.matricula.values[i]))
            conn.commit()


if __name__ == "__main__":

    conn = ps.connect("dbname=urbanmobility user=postgres password=242124")
    cur = conn.cursor()
    print('Criando banco de dados...')
    cur.execute(
        "CREATE TABLE linha2 (matricula varchar PRIMARY KEY, unidade numeric, nome varchar, estado numeric,linha numeric );"
    )
    cur.execute(
        "CREATE TABLE pontos2 (id serial PRIMARY KEY, lat numeric, lng numeric, instante timestamp, rota numeric, posicao numeric, velocidade numeric,viaje numeric, matricula_id varchar );"
    )
    conn.commit()

    data_frame = pd.read_csv('011017_011117.csv', sep=';')
    print('Fazendo o cleaning dos dados...')
    new_data = prep.PreProcess().clean_data(data_frame)
    print('Convertendo coordenadas para latitude e longitude')
    lat_lng = prep.PreProcess().coordinates_to_latlng(new_data)
    print('Salvando os dados no banco de dados....')
    save_in_db(lat_lng)
Пример #5
0
import pandas as pd
import preprocess as prep
from tqdm import tqdm
import numpy as np
import gmplot.gmplot

traffic_stops_data = pd.read_csv('semaforos.csv', sep=';')

data = pd.read_csv('data-100000-bus-stop-mapped.csv',delimiter=',', float_precision=None)

# data['label'] = ['in_route' for i in range(data.shape[0])]

prepro = prep.PreProcess()

for idx, row in tqdm(data.iterrows()):
    if row.velocidade < 5:
        for idx2,stop in traffic_stops_data.iterrows():
            print('indice:{0}'.format(idx2))
            print('values:{0}'.format(stop.Latitude, stop.Longitude))
            if idx2 != 292:
                dist = prepro.distance_in_meters([row.lat,row.lng], [stop.Latitude,stop.Longitude])
                if dist < 30 and row.label != 'bus_stop':
                    data.loc[idx,'label'] = 'traffic_light'

data.to_csv('data-100000-traffic-light.csv')
Пример #6
0
    elif run_raw and len(args.clean) != 0:
        logger.error(
            "--raw is set to True and --clean recieved 1 input. To run both set --clean with 0 input"
        )
        sys.exit()
    else:
        logger.error("--clean args requires 0 or 1 csv")
        sys.exit()

logger.debug(f"Running --raw: {run_raw}")
logger.debug(f"Running --clean: {run_clean}")
logger.debug(f"Running --raw and --clean: {run_raw_and_clean}")

if run_raw:
    if not run_raw_and_clean:
        p = preprocess.PreProcess(args.raw)
        p.combine_df().to_csv('Preprocess.csv', index=False)
        print('Preprocess complete. Preprocess.csv file saved')
    else:
        print('Two step process, preprocess and feature engineer, will run')

if run_clean:
    f = feature_eng.FeatEng(args.clean)
    f.feateng_update().to_csv('Clean_data.csv', index=False)
    print('Feature engineer complete. Clean_data.csv file saved')

if run_raw_and_clean:
    p = preprocess.PreProcess(args.raw)

    # Code for debugging purposes
    # result = pd.read_csv('Train_Preprocess.csv',parse_dates=['ClaimStartDt','ClaimEndDt','AdmissionDt','DOD','DischargeDt'])
Пример #7
0
args = parser.parse_args()
assert args.prefix, "provide prefix"
filenums = map(int, args.filenums.split(','))

codewords_fname = '%s-%s' % (args.prefix, 'acc_beam_codewords.h5')
if args.skipcw:
    codeword_datasets = {
        'yag': ('yag_codeword1', 'yag_codeword2'),
        'vcc': ('vcc_codeword1', 'vcc_codeword2')
    }
else:
    assert args.alg and args.alg in preprocess.ALGS.keys(
    ), "provide algorithm, one of %r, see preprocess.py" % preprocess.ALGS.keys(
    )
    preprocess = preprocess.PreProcess(alg=args.alg, final_shape=(224, 224))

    accdata = beamdata.BeamData(preprocess=preprocess,
                                datadir='data',
                                subbkg=args.subbkg,
                                filenums=filenums,
                                prefix=args.prefix,
                                force=args.force,
                                nn=args.nn)

    accdata.loadall(reload=False)

    if args.viewbkg:
        accdata.viewbkg()

    if args.view or args.save > 0:
Пример #8
0
 def spectrogram_test():
     p = preprocess.PreProcess(['spectrogram'])
     s = p.forward(x)
     x_hat = p.inverse(s)
     print(np.max(x_hat - x))
Пример #9
0
 def stft_test():
     p = preprocess.PreProcess(['stft'])
     s = p.forward(x)
     x_hat = p.inverse(s)
     print(np.max(x_hat - x))
import numpy as np
import masking
import torch
import preprocess

t_x = torch.ones((3, 2, 15, 17))
t_y = torch.ones((3))
t_p = preprocess.PreProcess(['None'])


def differential_evolution(x,
                           y,
                           pre,
                           N_dimensions_value,
                           N_perturbations,
                           N_iterations,
                           N_population,
                           targeted=False,
                           x_min=-1,
                           x_max=1,
                           train=False):
    def evolve(p_pos, p_val, F=0.5):
        #p_pos = [B, N, K, 3]
        c_pos = np.copy(p_pos)
        c_val = np.copy(p_val)

        # pick out random individuals from population
        idxs = np.random.choice(
            I, size=(I, 3))  # use same random process for whole batch
        x_pos = p_pos[:, idxs, :, :]
        x_val = p_val[:, idxs, :, :]
Пример #11
0
def normalize(text):
    preprocessed = preprocess.PreProcess(text)
    preprocessed.normalize_letter()
    preprocessed.remove_punctuation()
    return preprocessed
Пример #12
0
 def Preprocessing_Initialization(self, glove_data=None):
     self.preprocessor = preprocess.PreProcess(self.window_size,
                                               dev=self.device,
                                               glove_data=glove_data)