示例#1
0
def print_evaluate2(PA, kappa, **optional):  #prints out results
    #set parameters
    if 'path' in optional:
        path = optional['path']
    else:
        path = AD1.set_path()
    #calculate num_bats
    list_bats, _ = AD1.set_batscolor(Templates=path +
                                     '/Templates_regular')  #bat species
    num_bats, num_total = AD1.set_numbats(
        list_bats,
        Templates=path + '/Templates_regular')  #number of each bat species
    num_bats_dml, num_total_dml = AD1.set_numbats(list_bats,
                                                  Templates=path +
                                                  '/Templates_dml')
    num_bats_eval, num_total_eval = AD1.set_numbats(list_bats,
                                                    Templates=path +
                                                    '/Templates_eval')
    total_bats = num_bats + num_bats_dml + num_bats_eval
    #print out
    print('Species: Amount: PA: Cohen-kappa:')
    for i in range(len(list_bats)):
        print(list_bats[i] + '     ' + str(total_bats[i]) + '       ' +
              str(round(PA[i], 2)) + '   ' + str(round(kappa[i], 2)))

    return ()
def loading_init(**optional):
    regions_temp, rectangles_temp = AD1.read_templates(**optional)
    list_bats, colors_bat = AD1.set_batscolor(**optional)
    num_bats, num_total = AD1.set_numbats(list_bats, **optional)
    freq_bats, freq_range_bats, freq_peakT_bats, freq_peakF_bats = AD1.set_batfreq(
        rectangles_temp, regions_temp, list_bats, num_bats)
    return (freq_bats, freq_range_bats, freq_peakT_bats, freq_peakF_bats,
            list_bats, colors_bat, num_bats, num_total, regions_temp,
            rectangles_temp)
def print_features(**optional):
    list_bats, colors_bat = AD1.set_batscolor(**optional)
    num_bats, num_total = AD1.set_numbats(list_bats, **optional)
    a = 6
    print('Frequency: 0-' + str(a))
    for i in range(len(list_bats)):
        a += 1
        print(list_bats[i] + ': ' + str(a) + '-' + str(a + num_bats[i]))
        a += num_bats[i] - 1
    return ()
示例#4
0
def KNN_calc(X_final, Y_final, D, **optional):
    if 'path' in optional:
        path = optional['path']
    else:
        path = AD1.set_path()
    if 'K' in optional:
        m = optional['K']
    else:  #defaut to 3 neighbors
        m = 3
    list_bats, _ = AD1.set_batscolor(Templates=path +
                                     '/Templates_regular')  #length needed
    dist_mat = []  #distances, empty list because it needs to be sorted later
    matches = np.zeros((X_final.shape[1], ))  #saves the matches, temporarily
    match_scores = np.zeros((X_final.shape[1], ))  #final score individual
    final_scores = np.zeros((len(list_bats), ))  #final score type (PA)
    #temp variables
    count_i = 0
    temp_1 = 0
    temp_2 = 0
    for i in range(X_final.shape[1]):  #go through all datapoints
        dist_mat = []  #reset the list so it is empty again
        for j in range(X_final.shape[1]):  #check with each other datapoint
            dist_mat.append(
                np.sum(np.dot((X_final[:, i] - X_final[:, j])**2,
                              D)))  #distance between point j and i
        #take the lowest m distances, save them in matches temporarily
        for k in range(m):
            #sort ascending, take m lowest values
            index_dummy = [dist_mat.index(x) for x in sorted(dist_mat)
                           ]  #sorted indexes from low to high
            #indexes start at 1 because index 0 is the point i (matches with itself, distance 0)
            matches[k] = Y_final[index_dummy[k + 1]]
        #calculate score for point i
        count_i = 0  #start at zero
        for l in range(m):
            if Y_final[i] == matches[l]:
                count_i += 1  #one match
        match_scores[i] = count_i / m  #total percentage of matches

    #convert scores to take average per type
    for n1 in range(len(list_bats)):
        temp_1 = 0
        temp_2 = 0
        for n2 in range(X_final.shape[1]):  #go through all datapoints
            if Y_final[n2] == n1:  #match
                temp_1 += 1  #number of matches
                temp_2 += match_scores[n2]  #score
        if temp_1 == 0:
            final_scores[n1] = 0
        else:
            final_scores[n1] = temp_2 / temp_1  #average score
    return (final_scores, match_scores)
def create_template(
        file_name, timestep, region_num, bat_name,
        **optional):  #creates three templates (image, rectangle and array)
    if 'Templates' in optional:
        path = optional['Templates']
    else:
        path = AD1.set_path()
    path_original = path  #needed for make_folders
    if 'template_type' in optional:
        if optional['template_type'] == 'regular':
            path += '/Templates_regular'  #add this to the pathway
        elif optional['template_type'] == 'dml':
            path += '/Templates_dml'  #add this to the pathway
        elif optional['template_type'] == 'evaluate':
            path += '/Templates_eval'  #add this to the pathway
        else:  #default to regular
            path += '/Templates_regular'  #add this to the pathway
            print(
                'Warning: template type is not valid. \'regular\' was selected instead'
            )
    else:  #default to regular
        path += '/Templates_regular'  #add this to the pathway
        print('Warning: template type is not given. \'regular\' was selected.')

    #extract regions and rectangles
    list_bats, _ = AD1.set_batscolor()
    num_bats, _ = AD1.set_numbats(list_bats, **optional)
    rectangles, regions, _ = AD2.spect_loop(file_name, **optional)
    #create folders if need be
    path_regular = path_original + '/Templates_regular'
    path_dml = path_original + '/Templates_dml'
    path_eval = path_original + '/Templates_eval'
    AD1.make_folders(path_regular, list_bats + [bat_name])
    AD1.make_folders(path_dml, list_bats + [bat_name])
    AD1.make_folders(path_eval, list_bats + [bat_name])
    #make hash-code
    hash_code = hash(str(regions[int(timestep)][region_num]))
    #save image
    path_image = path + '/Templates_images/' + bat_name + '/' + str(
        hash_code) + '.png'
    plt.imshow(regions[int(timestep)][region_num], origin='lower')
    plt.savefig(path_image)
    plt.close()
    #build correct pathways
    path_array = path + '/Templates_arrays/' + bat_name + '/' + str(
        hash_code) + '.npy'
    path_rect = path + '/Templates_rect/' + bat_name + '/' + str(
        hash_code) + '.npy'
    #save the data
    np.save(path_array, regions[int(timestep)][region_num])
    np.save(path_rect, rectangles[int(timestep)][:, region_num])
    #print out the hash-codes to the user
    print('hash code:', hash_code)
    return ()
def spect_plot(samples, sample_rate, **optional):
    #Makes a spectrogram, data normalised to the range [0-1]
    #Change parameters of spectrogram (window, resolution)
    para = AD1.set_parameters()
    if 'min_spec_freq' in optional:
        min_spec_freq = optional['min_spec_freq']
    else:
        min_spec_freq = para[4]
    if 'max_spec_freq' in optional:
        max_spec_freq = optional['max_spec_freq']
    else:
        max_spec_freq = para[5]
    frequencies, times, spectrogram = signal.spectrogram(samples,
                                                         sample_rate,
                                                         window=('hamming'),
                                                         nfft=1024)
    dummy = (spectrogram - spectrogram.min()) / (spectrogram.max() -
                                                 spectrogram.min())
    dummy = np.array(np.round(dummy * 256),
                     dtype=np.uint8)  #Convert to grayscale
    if 'nosub' in optional:  #no subtraction
        if optional['nosub']:  #if True
            spectro = dummy[min_spec_freq:max_spec_freq, :]
        else:  #subtraction
            spectro = AD2.substraction(dummy[min_spec_freq:max_spec_freq, :])
    else:  #subtraction
        spectro = AD2.substraction(dummy[min_spec_freq:max_spec_freq, :])
    return (spectro)
def set_batscolor(**optional):  #dictionary linking bats to colors
    if 'Templates' in optional:
        path = optional['Templates']
    else:
        path = AD1.set_path()
        path += '/Templates_regular'  #add this to the pathway to get the correct folder
    colors_bat = {}
    list_bats = sorted(os.listdir(path + '/Templates_arrays'))
    colors = ("#ff0000", "#008000", "#0000ff", "#a52a2a", "#ee82ee", "#f0f8ff",
              "#faebd7", "#f0ffff", "#006400", "#ffa500", "#ffff00", "#40e0d0",
              "#4b0082", "#ff00ff", "#ffd700", "#ff0000", "#008000", "#0000ff",
              "#a52a2a", "#ee82ee", "#f0f8ff", "#faebd7", "#f0ffff", "#006400",
              "#ffa500", "#ffff00", "#40e0d0", "#4b0082", "#ff00ff", "#ffd700",
              "#ff0000", "#008000", "#0000ff", "#a52a2a", "#ee82ee", "#f0f8ff",
              "#faebd7", "#f0ffff", "#006400", "#ffa500", "#ffff00", "#40e0d0",
              "#4b0082", "#ff00ff", "#ffd700", "#ff0000", "#008000", "#0000ff",
              "#a52a2a", "#ee82ee", "#f0f8ff", "#faebd7", "#f0ffff", "#006400",
              "#ffa500", "#ffff00", "#40e0d0", "#4b0082", "#ff00ff", "#ffd700",
              "#ff0000", "#008000", "#0000ff", "#a52a2a", "#ee82ee", "#f0f8ff",
              "#faebd7", "#f0ffff", "#006400", "#ffa500", "#ffff00", "#40e0d0",
              "#4b0082", "#ff00ff", "#ffd700", "#ff0000", "#008000", "#0000ff",
              "#a52a2a", "#ee82ee", "#f0f8ff", "#faebd7", "#f0ffff", "#006400",
              "#ffa500", "#ffff00", "#40e0d0", "#4b0082", "#ff00ff", "#ffd700")
    for i in range(len(list_bats)):
        colors_bat[list_bats[i]] = colors[i]
    return (list_bats, colors_bat)
def spect(file_name, **optional):
    para = AD1.set_parameters()
    if 'spect_window' in optional:
        spect_window = optional['spect_window']
    else:
        spect_window = para[6]
    if 'spect_window_overlap' in optional:
        spect_window_overlap = optional['spect_window_overlap']
    else:
        spect_window_overlap = para[7]
    #Read information from audio file
    if 'sr' in optional:  #used for metadata
        samples, sample_rate = librosa.load(file_name,
                                            sr=optional['sr'],
                                            mono=False)
        samples = samples * 32768  #converting factor
        samples = np.transpose(samples)
    else:
        sample_rate, samples = scipy.io.wavfile.read(file_name, mmap=False)
    if 'channel' in optional:
        if optional['channel'] == 'l':
            samples = samples[:, 0]
        elif optional['channel'] == 'r':
            samples = samples[:, 1]
    N = len(samples)  #number of samples
    t = np.linspace(0, N / sample_rate, num=N)
    #time_array
    total_time = N / sample_rate
    steps = math.floor(
        (1000 * total_time) /
        (spect_window - spect_window_overlap))  #number of windows
    return (sample_rate, samples, t, total_time, steps)
def import_dml(dml, **optional):
    if 'path' in optional:
        path = optional['path']
    else:
        path = AD1.set_path()
    D = np.load(path + '/' + dml + '.npy')
    return (D)
def set_numbats(list_bats, **optional):  #sets the number of templates per bat
    if 'Templates' in optional:
        path = optional['Templates']
    else:
        path = AD1.set_path()
        path += '/Templates_regular'  #add this to the pathway to get the correct folder
    AD1.make_folders(path, list_bats)
    full_path = ''  #will be overwritten every time
    num_bats = np.zeros((len(list_bats), ), dtype=np.uint8)
    for i in range(len(list_bats)):
        path2 = list_bats[i]
        full_path = path + '/Templates_arrays/' + path2
        files_list = sorted(
            os.listdir(full_path))  #list all templates in folder
        num_bats[i] = len(files_list)  #number of templates
    num_total = sum(num_bats)
    return (num_bats, num_total)
def import_map(map_name, **optional):
    if 'path' in optional:
        path = optional['path']
    else:
        path = AD1.set_path()
    net = np.load(path + '/' + map_name + '.npy')
    raw_data = np.load(path + '/' + map_name + '_data.npy')
    return (net, raw_data)
示例#12
0
def print_evaluate(**optional):  #prints out number of templates
    #set parameters
    if 'path' in optional:
        path = optional['path']
    else:
        path = AD1.set_path()
    #calculate num_bats
    list_bats, _ = AD1.set_batscolor(Templates=path +
                                     '/Templates_regular')  #bat species
    num_bats, num_total = AD1.set_numbats(
        list_bats,
        Templates=path + '/Templates_regular')  #number of each bat species
    num_bats_dml, num_total_dml = AD1.set_numbats(list_bats,
                                                  Templates=path +
                                                  '/Templates_dml')
    num_bats_eval, num_total_eval = AD1.set_numbats(list_bats,
                                                    Templates=path +
                                                    '/Templates_eval')
    #print out
    print('Regular templates: ')
    for i in range(len(list_bats)):
        print(list_bats[i] + ': ' + str(num_bats[i]))
    print('Total: ' + str(num_total))

    print(' ')  #empty line
    print('dml templates: ')
    for i in range(len(list_bats)):
        print(list_bats[i] + ': ' + str(num_bats_dml[i]))
    print('Total: ' + str(num_total_dml))

    print(' ')  #empty line
    print('Evaluation templates: ')
    for i in range(len(list_bats)):
        print(list_bats[i] + ': ' + str(num_bats_eval[i]))
    print('Total: ' + str(num_total_eval))

    print(' ')  #empty line
    print('Total templates: ')
    for i in range(len(list_bats)):
        print(list_bats[i] + ': ' +
              str(num_bats[i] + num_bats_dml[i] + num_bats_eval[i]))
    print('Total: ' + str(num_total + num_total_dml + num_total_eval))

    return ()
示例#13
0
def SOM(raw_data, network_dim, n_iter, init_learning_rate, normalise_data,
        normalise_by_column, **optional):
    if 'DML' in optional:
        D = optional['DML']
    else:
        D = np.identity(raw_data.shape[0])
    m = raw_data.shape[0]
    n = raw_data.shape[1]
    # initial neighbourhood radius
    init_radius = max(network_dim[0], network_dim[1]) / 2
    # radius decay parameter
    time_constant = n_iter / np.log(init_radius)
    if normalise_data:
        if normalise_by_column:
            # normalise along each column
            col_maxes = raw_data.max(axis=0)
            data = raw_data / col_maxes[np.newaxis, :]
        else:
            # normalise entire dataset
            data = raw_data / data.max()
    else:
        data = raw_data
    # setup random weights between 0 and 1
    # weight matrix needs to be one m-dimensional vector for each neuron in the SOM
    net = np.random.random((network_dim[0], network_dim[1], m))
    for i in range(n_iter):
        # select a training example at random
        t = data[:, np.random.randint(0, n)].reshape(np.array([m, 1]))
        # find its Best Matching Unit
        bmu, bmu_idx = AD4.find_bmu(t, net, m, D)
        # decay the SOM parameters
        r = AD4.decay_radius(init_radius, i, time_constant)
        l = AD4.decay_learning_rate(init_learning_rate, i, n_iter)
        # now we know the BMU, update its weight vector to move closer to input
        # and move its neighbours in 2-D space closer
        # by a factor proportional to their 2-D distance from the BMU
        for x in range(net.shape[0]):
            for y in range(net.shape[1]):
                w = net[x, y, :].reshape(m, 1)
                # get the 2-D distance (again, not the actual Euclidean distance)
                w_dist = np.sum((np.array([x, y]) - bmu_idx)**2)
                # if the distance is within the current neighbourhood radius
                if w_dist <= r**2:
                    # calculate the degree of influence (based on the 2-D distance)
                    influence = AD4.calculate_influence(w_dist, r)
                    # now update the neuron's weight using the formula:
                    # new w = old w + (learning rate * influence * delta)
                    # where delta = input vector (t) - old w
                    new_w = w + (l * influence * (t - w))
                    # commit the new weight
                    net[x, y, :] = new_w.reshape(1, m)
    if 'export' in optional:
        path = AD1.set_path()
        np.save(path + '/' + optional['export'] + '.npy', net)
        np.save(path + '/' + optional['export'] + '_data.npy', raw_data)
    return (net)
示例#14
0
def calc_total_bats(**optional):
    if 'path' in optional:
        path = optional['path']
    else:
        path = AD1.set_path()
    #calculate num_bats
    list_bats, _ = AD1.set_batscolor(Templates=path +
                                     '/Templates_regular')  #bat species
    num_bats, num_total = AD1.set_numbats(
        list_bats,
        Templates=path + '/Templates_regular')  #number of each bat species
    num_bats_dml, num_total_dml = AD1.set_numbats(list_bats,
                                                  Templates=path +
                                                  '/Templates_dml')
    num_bats_eval, num_total_eval = AD1.set_numbats(list_bats,
                                                    Templates=path +
                                                    '/Templates_eval')
    total_bats = num_bats + num_bats_dml + num_bats_eval
    return (total_bats)
示例#15
0
def make_list(list_files, **optional):
    #Optional arguments
    #Audio data
    if 'Audio_data' in optional:  #different Audio_data folder
        path = optional['Audio_data']
    else:
        path = AD1.set_path()  #current working directory
        path += '/Audio_data'  #add Audio_data
    #folder and subfolder
    if 'folder' in optional:  #read all files in folder
        if optional['folder'] != 'None':  #True
            if 'subfolders' in optional:  #subfolders within Audio_data
                if optional['subfolders']:  #true, subfolders
                    folders_list = sorted(
                        os.listdir(path + '/' + optional['folder']))
                    list_files2 = list()  #empty list
                    for k in range(len(folders_list)):  #amend list
                        list_files2 += sorted(
                            os.listdir(path + '/' + optional['folder'] + '/' +
                                       folders_list[k]))
                else:  #false, no subfolders
                    list_files2 = sorted(
                        os.listdir(path + '/' + optional['folder']))
            else:  #argument not present, assume no subfolders
                list_files2 = sorted(
                    os.listdir(path + '/' + optional['folder']))
        else:  #folder equals 'None', fall back on list_files
            list_files2 = list_files
    else:  #folder argument not present, fall back on list_files
        list_files2 = list_files
    #full and subfolders, overwrites previous code
    if 'full' in optional:  #read all files
        if optional['full']:  #True
            if 'subfolders' in optional:  #subfolders within Audio_data
                if optional['subfolders']:  #true, subfolders
                    folders_list = sorted(os.listdir(path))
                    list_files2 = list()  #empty list
                    for k in range(len(folders_list)):  #amend list
                        list_files2 += sorted(
                            os.listdir(path + '/' + folders_list[k]))
                else:  #false, no subfolders
                    list_files2 = sorted(os.listdir(path))
            else:  #argument not present, assume no subfolders
                list_files2 = sorted(os.listdir(path))
        else:  #full equals False, fall back on list_files
            list_files2 = list_files
    #Delete files that aren't .wav
    count = 0
    for i in range(len(list_files2)):
        if list_files2[i - count][-4:] != '.WAV' and list_files2[
                i - count][-4:] != '.wav':  #no wav file
            del list_files2[i - count]  #delete files that aren't audio
            count += 1  #len changes, take this into account
    return (list_files2)
示例#16
0
def calc_matching(full_name, **optional):
    if 'dim1' in optional and 'dim2' in optional:
        network_dim = (optional['dim1'], optional['dim2'])
    else:
        para = AD1.set_parameters()
        network_dim = para[9]
    M = np.zeros((network_dim[0], network_dim[1]), dtype=np.uint8)
    for i in range(network_dim[0]):
        for j in range(network_dim[1]):
            M[i, j] = len(full_name[i][j])
    return (M)
示例#17
0
def fit_dml(**optional):
    #set parameters
    if 'Templates' in optional:
        path = optional['path']
    else:
        path = AD1.set_path()
        path += '/Templates_regular'
    if 'data_X' in optional and 'data_Y' in optional:  #data given
        X = optional['data_X']
        Y = optional['data_Y']
    else:  #read in data
        list_bats, _ = AD1.set_batscolor(Templates=path)  #bat species
        num_bats, num_total = AD1.set_numbats(
            list_bats, Templates=path)  #number of each bat species
        num_bats_dml, num_total_dml = AD1.set_numbats(list_bats,
                                                      Templates=path +
                                                      '/Templates_dml')
        #read normal templates
        regions, rectangles = AD1.read_templates(Templates=path)
        #read dml_templates
        regions2, rectangles2 = AD1.read_templates(Templates=path +
                                                   '/Templates_dml')
        #set variables
        templates = regions.copy()
        #combine both rectangles and regions
        for k in range(num_total_dml):
            rectangles[num_total + k] = rectangles2[k]
            regions[num_total + k] = regions2[k]

        #calculate features
        features = AD3.calc_features2(rectangles, regions, templates,
                                      list_bats, num_total)
        X = np.transpose(features)
        Y = np.zeros((X.shape[0], ), dtype=np.uint8)
        #Fill in Y matrix
        count = 0
        #go through the templates
        for i in range(len(list_bats)):  #i corresponds to a bat species
            for j in range(num_bats[i]):  #number of this type of bat
                Y[count] = i
                count += 1
        #same thing, but for templates_dml
        for i in range(len(list_bats)):  #i corresponds to a bat species
            for j in range(num_bats_dml[i]):  #number of this type of bat
                Y[count] = i
                count += 1
    #fit model
    model = dml.dmlmj.DMLMJ()
    model.fit(X, Y)
    D = model.transformer()
    #Export D-matrix
    if 'export' in optional:
        path = AD1.set_path()
        np.save(path + '/' + optional['export'] + '.npy', D)
    return (D)
def read_templates(**optional):
    if 'Templates' in optional:
        path = optional['Templates']
    else:
        path = AD1.set_path()
        path += '/Templates_regular'  #add this to the pathway to get the correct folder
    full_path = ''  #string will be constructed every step
    full_path_rec = ''
    list_bats, _ = AD1.set_batscolor(**optional)
    num_bats, _ = AD1.set_numbats(list_bats, **optional)
    regions = {}
    rectangles = {}
    count = 0
    for i in range(len(list_bats)):
        list_files_arrays = sorted(
            os.listdir(path + '/Templates_arrays/' +
                       list_bats[i]))  #list all files in folder
        list_files_rect = sorted(
            os.listdir(path + '/Templates_rect/' + list_bats[i]))
        count1 = 0
        count2 = 0
        for k in range(len(list_files_arrays)):
            if list_files_arrays[k - count1][-4:] != '.npy':
                del list_files_arrays[
                    k - count1]  #remove files that aren't npy extensions
                count1 += 1  #len of list changes, take this into account
        for k in range(len(list_files_rect)):  #repeat for rectangles
            if list_files_rect[k - count2][-4:] != '.npy':
                del list_files_rect[
                    k - count2]  #remove files that aren't npy extensions
                count2 += 1  #len of list changes, take this into account
        for j in range(num_bats[i]):  #go through the files one by one
            full_path = path + '/Templates_arrays/' + list_bats[
                i] + '/' + list_files_arrays[j]
            full_path_rec = path + '/Templates_rect/' + list_bats[
                i] + '/' + list_files_rect[j]
            regions[count] = np.load(full_path)
            rectangles[count] = np.load(full_path_rec)
            count += 1
    return (regions, rectangles)
示例#19
0
def fit_SOM(list_files, **optional):
    #set parameters
    para = AD1.set_parameters()
    if 'dim1' in optional and 'dim2' in optional:
        network_dim = (optional['dim1'], optional['dim2'])
    else:
        network_dim = para[9]
    if 'n_iter' in optional:
        n_iter = optional['n_iter']
    else:
        n_iter = para[10]
    if 'init_learning_rate' in optional:
        init_learning_rate = optional['init_learning_rate']
    else:
        init_learning_rate = para[11]
    if 'normalise_data' in optional:
        normalise_data = optional['normalise_data']
    else:
        normalise_data = para[12]
    if 'normalise_by_column' in optional:
        normalise_by_column = optional['normalise_by_column']
    else:
        normalise_by_column = para[13]
    #data already present, override list_files
    if 'features' in optional:
        raw_data = optional['features']
    else:
        list_files2 = AD4.make_list(
            list_files, **optional)  #create a list of files to be analyses
        #first file
        rectangles1, regions1, spectros1 = AD2.spect_loop(
            list_files2[0], **optional)
        num_reg = AD3.calc_num_regions(regions1)
        freq_bats, freq_range_bats, freq_peakT_bats, freq_peakF_bats, list_bats, colors_bat, num_bats, num_total, regions_temp, rectangles_temp = AD1.loading_init(
            **optional)
        features1, _, _ = AD3.calc_features(rectangles1, regions1,
                                            regions_temp, num_reg, list_bats,
                                            num_total)
        raw_data = np.zeros((features1.shape[0], 0))
        raw_data = np.concatenate((raw_data, features1), axis=1)
        #other files
        for i in range(1, len(list_files2)):
            rectangles1, regions1, spectros1 = AD2.spect_loop(
                list_files2[i], **optional)
            num_reg = AD3.calc_num_regions(regions1)
            features1, features_key1, features_freq1 = AD3.calc_features(
                rectangles1, regions1, regions_temp, num_reg, list_bats,
                num_total)
            raw_data = np.concatenate((raw_data, features1), axis=1)
    net = AD4.SOM(raw_data, network_dim, n_iter, init_learning_rate,
                  normalise_data, normalise_by_column, **optional)
    return (net, raw_data)
示例#20
0
def calc_maxc(full_names, **optional):
    if 'dim1' in optional and 'dim2' in optional:
        network_dim = (optional['dim1'], optional['dim2'])
    else:
        para = AD1.set_parameters()
        network_dim = para[9]
    max_c = 0
    for i in range(network_dim[0]):
        for j in range(network_dim[1]):
            temp_c = len(full_names[i][j])
            if temp_c > max_c:
                max_c = temp_c
    return (max_c)
示例#21
0
def calc_net_features(net, **optional):
    if 'dim1' in optional and 'dim2' in optional:
        network_dim = (optional['dim1'], optional['dim2'])
    else:
        para = AD1.set_parameters()
        network_dim = para[9]
    net_features = np.zeros((net.shape[2], network_dim[0] * network_dim[1]))
    count = 0
    for i in range(network_dim[0]):
        for j in range(network_dim[1]):
            net_features[:, count] = net[i, j, :]
            count += 1
    return (net_features)
def overload(rectangles, regions, **optional):
    para = AD1.set_parameters()
    if 'max_roi' in optional:
        max_roi = optional['max_roi']
    else:
        max_roi = para[2]
    rectangles2 = rectangles.copy()  #copy dictionaries
    regions2 = regions.copy()
    for i, j in rectangles.items():  #iterate over all items
        if len(rectangles[i][0, :]) > max_roi:
            rectangles2.pop(i)
            regions2.pop(i)
    return (rectangles2, regions2)
示例#23
0
def calc_center(region, time, min_freq, max_freq, rectangle):
    #convert to pixels
    para = AD1.set_parameters()
    #keep in mind a spectrogram starts counting at a specific frequency
    min_spec_freq = para[4]
    time_pixels = int(time / 0.32)
    freq_pixels = int((max_freq - min_freq) / 0.375)
    region_center = np.zeros((freq_pixels, time_pixels), dtype=np.uint8)
    #location of start and end frequency region above min_freq
    starting_height = (rectangle[1] * 0.375) - min_freq + (
        min_spec_freq * 0.375)  #in kHz
    #in pixels
    start_h = math.ceil(starting_height /
                        0.375)  #start height in pixels (round up)
    end_h = start_h + region.shape[0]  #ending height in pixels
    #time where region ends
    end_time = region.shape[1]  #in pixels
    #check boundaries

    #starting height
    if starting_height > 0:  #start of region is above bounding box, fits inside
        start_j = start_h
        offset_j = 0
    else:  #start of region is below bounding box, start filling region_center at zero
        start_j = 0
        offset_j = int(-starting_height /
                       0.375)  #amount of pixels below cutoff
    #ending height
    if end_h < freq_pixels:  #end of region is below bounding box, fits inside
        end_j = end_h
    else:  #end of region is above bounding box, stop filling at freq_pixels (end of bounding box)
        end_j = freq_pixels
    start_i = 0  #always start filling time at zero
    #ending time
    if end_time < time_pixels:  #end of region is to the left of bounding box, fits inside
        end_i = end_time
    else:  #end of region is to the right of bounding box, stop filling at end of bounding box
        end_i = time_pixels

    #coordinates in region
    count_j = 0
    count_i = 0
    #fill in array
    for i in range(start_i, end_i):  #column
        for j in range(start_j, end_j):  #row
            region_center[j, i] = region[offset_j + count_j, count_i]
            count_j += 1
        count_j = 0  #next column starts
        count_i += 1

    return (region_center)
示例#24
0
def calc_PE(**optional):
    if 'path' in optional:
        path = optional['path']
    else:
        path = AD1.set_path()
    #set num_bats
    list_bats, _ = AD1.set_batscolor(Templates=path +
                                     '/Templates_regular')  #bat species
    num_bats, num_total = AD1.set_numbats(
        list_bats,
        Templates=path + '/Templates_regular')  #number of each bat species
    num_bats_dml, num_total_dml = AD1.set_numbats(list_bats,
                                                  Templates=path +
                                                  '/Templates_dml')
    num_bats_eval, num_total_eval = AD1.set_numbats(list_bats,
                                                    Templates=path +
                                                    '/Templates_eval')
    #initialise kappa
    PE = np.zeros((len(list_bats), ))
    total_dummy = num_total + num_total_dml + num_total_eval
    for i in range(len(list_bats)):
        PE[i] = (num_bats[i] + num_bats_dml[i] +
                 num_bats_eval[i]) / total_dummy
    return (PE)
def spect_loop(file_name, **optional):
    para = AD1.set_parameters()
    if 'X' in optional:
        X = optional['X']
    else:
        X = para[0]
    if 'kern' in optional:
        kern = optional['kern']
    else:
        kern = para[1]
    if 'spect_window' in optional:
        spect_window = optional['spect_window']
    else:
        spect_window = para[6]
    if 'spect_window_overlap' in optional:
        spect_window_overlap = optional['spect_window_overlap']
    else:
        spect_window_overlap = para[7]
    #change number of steps so it matches the window and overlap
    #change algorithm so the spectrograms aren't fixed at 100 ms, but rather at the number of actual steps
    #templates are already defined so the algorthitm becomes obsolete
    rectangles = {}
    regions = {}
    spectros = {}
    sample_rate, samples, _, _, steps = AD2.spect('Audio_data/' + file_name,
                                                  **optional)
    if 'exp_factor' in optional:
        sample_rate = optional['exp_factor'] * sample_rate
        steps = math.floor(steps / optional['exp_factor'])
    start_index = 0
    stop_index = int(spect_window_overlap * len(samples) /
                     (steps * spect_window))
    for i in range(steps):
        samples_dummy = samples[start_index:stop_index]
        start_index = stop_index
        stop_index += int(spect_window_overlap * len(samples) /
                          (steps * spect_window))
        spectros[i] = AD2.spect_plot(samples_dummy, sample_rate, **optional)
        ctrs, dummy_flag = AD2.ROI(spectros[i], kern, X)
        if dummy_flag:
            rectangles[i], regions[i] = AD2.ROI2(ctrs, spectros[i])
            rectangles, regions = AD2.check_overlap(rectangles, regions,
                                                    spectros, i, spect_window,
                                                    spect_window_overlap)
    rectangles2, regions2 = AD2.overload(rectangles, regions, **optional)
    regions3 = AD2.rescale_region(regions2)
    return (rectangles2, regions3, spectros)
def set_parameters():
    #read in the file
    path = AD1.set_path()
    f = open(path + '/parameters.txt', 'r')
    a = f.readlines()
    words = [None] * len(a)
    i = 0  #counter
    for line in a:
        words[i] = line.split(';')  #save text in 'words' array
        i += 1
    #Spectrogram and ROI
    binary_thresh = int(words[1][0])
    spec_min = int(words[2][0])
    spec_max = int(words[3][0])
    spect_window = int(words[4][0])
    spect_window_overlap = int(words[5][0])
    max_roi = int(words[6][0])
    kern = int(words[7][0])  #window for roi
    #SOM
    network_dim1 = int(words[9][0])
    network_dim2 = int(words[10][0])
    n_iter = int(words[11][0])
    init_learning_rate = float(words[12][0])
    context_window = int(words[13][0])
    context_window_freq_dummy = int(words[14][0])
    #Hierarchical clustering
    thresh = float(words[16][0])
    w_1 = float(words[17][0])
    w_2 = float(words[18][0])
    w_3 = float(words[19][0])
    w_4 = float(words[20][0])
    #Convert parameters
    w_impor = (w_1, w_2, w_3, w_4)
    X = int(binary_thresh * 255 / 100)  #Threshold for noise binary image
    min_spec_freq = int(spec_min / 0.375)  #freq to pixels
    max_spec_freq = int(spec_max / 0.375)  #freq to pixels
    network_dim = (network_dim1, network_dim2)
    context_window_freq = int(context_window_freq_dummy / 0.375)
    #Extra parameters for SOM
    normalise_data = False
    normalise_by_column = False
    fig_size = (10, 10)
    #Put everything in one variable
    para=(X, kern, max_roi, thresh, min_spec_freq, max_spec_freq, spect_window, spect_window_overlap, w_impor, \
          network_dim, n_iter, init_learning_rate, normalise_data, normalise_by_column, context_window, context_window_freq, fig_size, \
          spec_min, spec_max, binary_thresh)
    return (para)
def show_region(rectangles, spectros, i, **optional):
    para = AD1.set_parameters()
    if 'min_spec_freq' in optional:
        min_spec_freq = optional['min_spec_freq']
    else:
        min_spec_freq = para[4]
    spec_min = int(min_spec_freq / 0.375)
    if 'max_spec_freq' in optional:
        max_spec_freq = optional['max_spec_freq']
    else:
        max_spec_freq = para[5]
    spec_max = int(max_spec_freq / 0.375)
    if 'spect_window' in optional:
        t_max = optional['spect_window']
    else:
        t_max = para[6]
    f, ax1 = plt.subplots()
    ax1.imshow(spectros[i], origin='lower', aspect='auto')
    dummy = rectangles[i].shape
    for j in range(dummy[1]):
        rect = patches.Rectangle((rectangles[i][0, j], rectangles[i][1, j]),
                                 rectangles[i][2, j],
                                 rectangles[i][3, j],
                                 linewidth=1,
                                 edgecolor='r',
                                 facecolor='none')
        #Add the patch to the Axes
        ax1.add_patch(rect)
    plt.title(i)
    labels_X = [item.get_text() for item in ax1.get_xticklabels()]
    labels_Y = [item.get_text() for item in ax1.get_yticklabels()]
    labels_X[1] = 0
    labels_X[-2] = t_max
    for i in range(1, len(labels_Y) - 1):
        labels_Y[i] = int((spec_max - spec_min) * (i - 1) /
                          (len(labels_Y) - 3) + spec_min)
    ax1.set_xticklabels(labels_X)
    ax1.set_yticklabels(labels_Y)
    plt.xlabel('Time (ms)')
    plt.ylabel('Frequency (kHz)')
    plt.show()
    if 'export' in optional:
        f.savefig(optional['export'] + '.png', format='png', dpi=1200)
    plt.close()
    return ()
示例#28
0
def calc_context_spec(spectros, k, temp_key, **optional):
    para = AD1.set_parameters()
    if 'spect_window' in optional:
        spect_window = optional['spect_window']
    else:
        spect_window = para[6]
    if 'spect_window_overlap' in optional:
        spect_window_overlap = optional['spect_window_overlap']
    else:
        spect_window_overlap = para[7]
    if 'context_window' in optional:
        context_window = optional['context_window']
    else:
        context_window = para[14]
    overlap_factor = (1 - spect_window_overlap / spect_window
                      )  #corrects for overlap between windows
    max_key = len(spectros[k]) - 1
    extra_time = 0  #extra time points to correct rectangle
    context_spec = spectros[k][temp_key[0]]
    steps = context_spec.shape[1]
    #left
    for i in range(1, context_window + 1):
        if (temp_key[0] - i) >= 0:  #window the left exists
            context_spec = np.concatenate(
                (spectros[k][temp_key[0] - i][:,
                                              0:int(steps * overlap_factor)],
                 context_spec),
                axis=1)
            extra_time += int(
                len(spectros[k][temp_key[0] - i][0, :]) * overlap_factor)
    #right
    for i in range(1, context_window + 1):
        if (temp_key[0] + i) <= max_key:  #window to the right exists
            context_spec = np.concatenate(
                (context_spec,
                 spectros[k][temp_key[0] + i][:,
                                              int(steps *
                                                  (1 - overlap_factor)):]),
                axis=1)
    return (context_spec, extra_time)
示例#29
0
def evaluation_SOM(**optional):
    #set parameters
    if 'path' in optional:
        path = optional['path']
    else:
        path = AD1.set_path()
    if 'dim1' in optional and 'dim2' in optional:
        network_dim = (optional['dim1'], optional['dim2'])
    else:
        para = AD1.set_parameters()
        network_dim = para[9]

    #calculate num_bats
    list_bats, _ = AD1.set_batscolor(Templates=path +
                                     '/Templates_regular')  #bat species
    num_bats, num_total = AD1.set_numbats(
        list_bats,
        Templates=path + '/Templates_regular')  #number of each bat species
    num_bats_dml, num_total_dml = AD1.set_numbats(list_bats,
                                                  Templates=path +
                                                  '/Templates_dml')
    num_bats_eval, num_total_eval = AD1.set_numbats(list_bats,
                                                    Templates=path +
                                                    '/Templates_eval')

    #read normal templates
    regions, rectangles = AD1.read_templates(Templates=path +
                                             '/Templates_regular')
    #read dml_templates
    regions2, rectangles2 = AD1.read_templates(Templates=path +
                                               '/Templates_dml')
    #read eval_templates
    regions3, rectangles3 = AD1.read_templates(Templates=path +
                                               '/Templates_eval')
    #set templates
    if 'templates_features' in optional:
        templates, _ = AD1.read_templates(
            Templates=optional['templates_features'] + '/Templates_regular')
    else:
        templates = regions.copy()
    #combine rectangles and regions
    for k in range(num_total_dml):
        rectangles[num_total + k] = rectangles2[k]
        regions[num_total + k] = regions2[k]

    print('Initialisation complete (1/3)')

    #calculate features for DML
    X = AD3.calc_features2(rectangles, regions, templates, list_bats,
                           num_total)
    Y = np.zeros((X.shape[1], ), dtype=np.uint8)
    #Fill in Y matrix
    count = 0
    #go through the templates
    for i in range(len(list_bats)):  #i corresponds to a bat species
        for j in range(num_bats[i]):  #number of this type of bat
            Y[count] = i
            count += 1
    #same thing, but for templates_dml
    for i in range(len(list_bats)):  #i corresponds to a bat species
        for j in range(num_bats_dml[i]):  #number of this type of bat
            Y[count] = i
            count += 1

    #calculate dml
    if 'DML' in optional:
        D = optional['dml']
    else:
        D = AD4.fit_dml(data_X=np.transpose(X), data_Y=Y)

    print('DML complete (2/3)')

    #add the final part to the data (eval)
    for k in range(num_total_eval):
        rectangles[num_total + num_total_dml + k] = rectangles3[k]
        regions[num_total + num_total_dml + k] = regions3[k]

    #final data
    X_final = AD3.calc_features2(rectangles, regions, templates, list_bats,
                                 num_total)
    Y_final = np.zeros((X_final.shape[1], ), dtype=np.uint8)
    #Fill in Y matrix
    Y_final[0:count] = Y  #we already have this data
    #add the eval templates
    for i in range(len(list_bats)):  #i corresponds to a bat species
        for j in range(num_bats_eval[i]):  #number of this type of bat
            Y_final[count] = i
            count += 1

    #make a SOM
    if 'SOM' in optional:
        net = optional['SOM']
    else:
        if 'Full' in optional and optional[
                'Full']:  #if Full_flag is present and set to true
            list_files = 0
            net, raw_data = AD4.fit_SOM(list_files,
                                        full=optional['full'],
                                        dim1=network_dim[0],
                                        dim2=network_dim[1],
                                        DML=D)
        elif 'List_files' in optional:  #fit on list of files
            Full_flag = False
            net, raw_data = AD4.fit_SOM(optional['List_files'],
                                        full=Full_flag,
                                        dim1=network_dim[0],
                                        dim2=network_dim[1],
                                        DML=D)
        else:  #fit on templates only
            list_files = 0  #is overwritten by features
            Full_flag = False  #overwritten by features
            if 'fit_eval' in optional and optional[
                    'eval_SOM']:  #present and true, use eval data to fit SOM
                #use X_final
                net, raw_data = AD4.fit_SOM(list_files,
                                            full=Full_flag,
                                            dim1=network_dim[0],
                                            dim2=network_dim[1],
                                            DML=D,
                                            features=X_final)
            else:  #exclude eval data to fit SOM
                #use X
                net, raw_data = AD4.fit_SOM(list_files,
                                            full=Full_flag,
                                            dim1=network_dim[0],
                                            dim2=network_dim[1],
                                            DML=D,
                                            features=X)

    print('SOM complete (3/3)')

    #make a plot
    if 'Plot_Flag' in optional:
        if optional['Plot_Flag']:  #if set to true
            m = X_final.shape[0]

            f, ax1 = plt.subplots()

            count = np.zeros((len(list_bats), ), dtype=np.uint8)
            col_list = [
                'ro', 'g>', 'b*', 'cv', 'm^', 'kd', 'r<', 'g1', 'b2', 'c3',
                'm4', 'k8', 'rs', 'gp', 'ch', 'm,', 'ro', 'g>', 'b*', 'cv',
                'm^', 'kd', 'r<', 'g1', 'b2', 'c3', 'm4', 'k8', 'rs', 'gp',
                'ch', 'm,', 'ro', 'g>', 'b*', 'cv', 'm^', 'kd', 'r<', 'g1',
                'b2', 'c3', 'm4', 'k8', 'rs', 'gp', 'ch', 'm,', 'ro', 'g>',
                'b*', 'cv', 'm^', 'kd', 'r<', 'g1', 'b2', 'c3', 'm4', 'k8',
                'rs', 'gp', 'ch', 'm,'
            ]
            for i in range(X_final.shape[1]):
                for j in range(len(list_bats)):
                    #take the datapoint and the bmu
                    t = X_final[:, i].reshape(np.array([m, 1]))
                    bmu, bmu_idx = AD4.find_bmu(t, net, m, D)
                    if Y_final[i] == j:  #class j
                        col = col_list[j]
                        if count[j] == 0:  #first hit
                            ax1.plot(bmu_idx[0],
                                     bmu_idx[1],
                                     col,
                                     label=list_bats[j])
                            count[j] += 1  #only do this once
                        else:  #plot without label
                            ax1.plot(bmu_idx[0], bmu_idx[1], col)
            #legend
            handles, labels = ax1.get_legend_handles_labels()
            plt.legend(handles, labels)
            plt.xlabel('Index 1 SOM')
            plt.ylabel('Index 2 SOM')
            #optional
            if 'title' in optional:
                plt.title(optional['title'])
            if 'export' in optional:
                plt.savefig(optional['export'])
            plt.show()
            plt.close()
    return (X_final, Y_final, net, D)
示例#30
0
def plot_region_neuron(full_region, full_rectangle, full_spectro, full_name,
                       dim1, dim2, point, max_time, min_freq, max_freq,
                       context, FI, fig_size, **optional):
    if not point in full_rectangle[dim1][dim2]:  #reached the end of the matches
        plot_flag = False
    else:
        plot_flag = True
        region_center = AD4.calc_center(full_region[dim1][dim2][point],
                                        max_time, min_freq, max_freq,
                                        full_rectangle[dim1][dim2][point])
    #parameters
    if 'context_window_freq' in optional:
        context_window_freq = optional['context_window_freq']
    else:
        para = AD1.set_parameters()
        context_window_freq = para[15]

    #size
    fig_size = (fig_size, fig_size)

    #create plot
    if context == True and FI == True:
        f, (ax1, ax2,
            ax3) = plt.subplots(1,
                                3,
                                figsize=fig_size,
                                gridspec_kw={'width_ratios': [1, 1, 1]})
    if context == True and FI == False:
        f, (ax1, ax2) = plt.subplots(1,
                                     2,
                                     figsize=fig_size,
                                     gridspec_kw={'width_ratios': [1, 1]})
    if context == False and FI == True:
        f, (ax2, ax3) = plt.subplots(1,
                                     2,
                                     figsize=fig_size,
                                     gridspec_kw={'width_ratios': [1, 1]})
    if context == False and FI == False:
        f, (ax2) = plt.subplots(1, 1, figsize=fig_size)

    #Middle image, always on

    if plot_flag:
        ax2.imshow(region_center, origin='lower', aspect='equal')
        #set up the axis
        plt.draw()  #sets up the ticks
        #set labels
        labels_Y = [item.get_text()
                    for item in ax2.get_yticklabels()]  #original labels
        labels_y = list()  #new labels
        labels_y.append(labels_Y[0])
        for i in range(1, len(labels_Y)):
            labels_y.append(
                str(round(float((float(labels_Y[i]) * 0.375) + min_freq), 2)))
        labels_X = [item.get_text()
                    for item in ax2.get_xticklabels()]  #original labels
        labels_x = list()
        labels_x.append(labels_X[0])
        for i in range(1, len(labels_X)):
            labels_x.append(str(round(float(float(labels_X[i]) / 2.34375),
                                      2)))  #convert to ms
            ax2.set_xticklabels(labels_x)
            ax2.set_yticklabels(labels_y)
            ax2.set_xlabel('Time (ms)')
            ax2.set_ylabel('Frequency (kHz)')
    else:  #show empty image
        ax2.imshow(np.zeros((1, 1)))

    #Left image (full spectro), only of context is on

    if context == 1 and plot_flag:
        freq1_index = full_rectangle[dim1][dim2][point][1] - context_window_freq
        if freq1_index < 0:
            freq1_index = 0
        freq2_index = full_rectangle[dim1][dim2][point][1] + full_rectangle[
            dim1][dim2][point][3] + context_window_freq
        if freq2_index > full_spectro[dim1][dim2][point].shape[0]:
            freq2_index = full_spectro[dim1][dim2][point].shape[0]
        #image
        ax1.imshow(full_spectro[dim1][dim2][point][freq1_index:freq2_index],
                   origin='lower',
                   aspect='auto')
        #rectangle
        rect = patches.Rectangle(
            (full_rectangle[dim1][dim2][point][0],
             full_rectangle[dim1][dim2][point][1] - freq1_index),
            full_rectangle[dim1][dim2][point][2],
            full_rectangle[dim1][dim2][point][3],
            linewidth=1,
            edgecolor='r',
            facecolor='none')
        # Add the patch to the Axes
        ax1.add_patch(rect)
        plt.draw()  #sets up the ticks
        #set labels
        labels_Y = [item.get_text()
                    for item in ax1.get_yticklabels()]  #original labels
        labels_y = list()  #new labels
        labels_y.append(labels_Y[0])
        for i in range(1, len(labels_Y)):
            labels_y.append(
                str(float((float(labels_Y[i]) + freq1_index) * 0.375)))
        labels_X = [item.get_text() for item in ax1.get_xticklabels()]
        labels_x = list()
        labels_x.append(labels_X[0])
        for i in range(1, len(labels_X)):
            labels_x.append(str(int(int(labels_X[i]) /
                                    2.34375)))  #convert to ms
        ax1.set_xticklabels(labels_x)
        ax1.set_yticklabels(labels_y)
        ax1.set_xlabel('Time (ms)')
        ax1.set_ylabel('Frequency (kHz)')

    #Right image, if FI is on

    if FI == 1 and plot_flag:
        freq1_index = full_rectangle[dim1][dim2][point][1] - context_window_freq
        if freq1_index < 0:
            freq1_index = 0
        freq2_index = full_rectangle[dim1][dim2][point][1] + full_rectangle[
            dim1][dim2][point][3] + context_window_freq
        if freq2_index > full_spectro[dim1][dim2][point].shape[0]:
            freq2_index = full_spectro[dim1][dim2][point].shape[0]
        #image
        FI_matrix = AD4.calc_FI_matrix(full_region[dim1][dim2][point])
        ax3.imshow(FI_matrix, origin='lower', aspect='auto')
        plt.draw()
        #labels
        labels_X = [item.get_text()
                    for item in ax3.get_xticklabels()]  #original labels
        labels_x = list()  #new labels
        labels_x.append(labels_X[0])
        for i in range(1, len(labels_X)):
            labels_x.append(
                str(
                    round((float(labels_X[i]) + freq1_index +
                           context_window_freq) * 0.375, 2)))
        ax3.set_xticklabels(labels_x)
        ax3.set_xlabel('Frequency (kHz)')
        ax3.set_ylabel('Intensity')
    #title plot
    if plot_flag:
        ax2.set_title(full_name[dim1][dim2][point])
    else:
        ax2.set_title("No more matches. Change sliders.")
    #show plot
    plt.show()
    plt.close()
    return ()