コード例 #1
0
def function3():
	print('This is function 2 calling function 1 and saving the result')
	import numpy as np

	result 			=	submodule1.function1()

	np.save('results/some_result',result)
コード例 #2
0
ファイル: io.py プロジェクト: drajguru/wyrm
def save(dat, filename):
    """Save a ``Data`` object into a NumPy .npy file.


    Parameters
    ----------
    dat : Data
        `Data` object
    filename : str
        Filename of the file to save to. If the filename does not end
        with ``.npy``, the ``.npy`` extension will be automatically
        appended.


    See Also
    --------
    :func:`load`


    Examples
    --------

    >>> io.save(dat, 'foo.npy')
    >>> dat2 = io.load('foo.npy')

    """
    np.save(filename, dat)
コード例 #3
0
ファイル: modularity.py プロジェクト: Lx37/dmgraphanalysis
def export_mod_mask_file(Louvain_mod_file,Louvain_node_file,coords_file,mask_file):

    import numpy as np
    import nibabel as nib
    import os
    
    from dmgraphanalysis.utils_cor import return_mod_mask_corres,read_Louvain_corres_nodes,read_mod_file

    print 'Loading node_corres'
    
    node_corres = read_Louvain_corres_nodes(Louvain_node_file)
    
    print node_corres
    print node_corres.shape
    
    print 'Loading coords'
    
    coords = np.array(np.loadtxt(coords_file),dtype = int)
    
    print coords.shape
    
    print 'Loading mask parameters'
    
    mask = nib.load(mask_file)
    
    data_mask_shape = mask.get_data().shape
    
    mask_header = mask.get_header().copy()
    
    mask_affine = np.copy(mask.get_affine())
    
    print "Loading community belonging file" + Louvain_mod_file

    community_vect = read_mod_file(Louvain_mod_file)
    
    #print community_vect
    print community_vect.shape
    
    print "transforming to nii file"
    mod_mask_data = return_mod_mask_corres(community_vect,node_corres,coords,data_mask_shape)

    #print mod_mask_data
    print mod_mask_data.shape


    print "saving npy file"

    mod_mask_file = os.path.abspath("mod_mask_data.npy")

    np.save(mod_mask_file ,np.array(mod_mask_data,dtype = int))
    
    print "saving nii file"

    mod_mask_file = os.path.abspath("mod_mask_data.nii")

    nib.save(nib.Nifti1Image(np.array(mod_mask_data,dtype = int),mask_affine,mask_header),mod_mask_file)

    print "returning"

    return mod_mask_file
コード例 #4
0
def segment_request(request):
    max_iteration=int(request[:,0].max()//optimal_interval)
    for i in range(max_iteration+1):
        temp=request[np.logical_and(request[:,0]<=optimal_interval*(i+1),request[:,0]>=(optimal_interval*i+1))]
        temp[:,0]=temp[:,0]-optimal_interval*i
        np.save('experiment_%d/new_passenger_%d.npy'% (experiment,i),temp)
    return max_iteration
コード例 #5
0
ファイル: modularity.py プロジェクト: Lx37/dmgraphanalysis
def compute_signif_conf_Z_list(cor_mat_file,conf_cor_mat_file,coords_file):       
        
    import rpy,os
    import nibabel as nib
    import numpy as np
    
    from dmgraphanalysis.utils_cor import export_List_net_from_list,export_Louvain_net_from_list
    from dmgraphanalysis.utils_cor import return_signif_conf_net_list
    from dmgraphanalysis.utils_plot import plot_cormat
    
    print "loading cor_mat_file"
    
    cor_mat = np.load(cor_mat_file)
    
    print "loading conf_cor_mat_file"
    
    conf_cor_mat = np.load(conf_cor_mat_file)
    
    print 'load coords'
    
    coords = np.array(np.loadtxt(coords_file),dtype = int)
    
    print "computing net_list by thresholding conf_cor_mat based on distance and net_threshold"
    
    net_list,binary_signif_matrix = return_signif_conf_net_list(cor_mat,conf_cor_mat)
    
    print binary_signif_matrix.shape
    
    print "saving binary_signif_matrix"
    
    binary_signif_matrix_file = os.path.abspath('binary_signif_matrix.npy')
    
    np.save(binary_signif_matrix_file,binary_signif_matrix)
    
    print "plotting binary_signif_matrix"
    
    plot_binary_signif_matrix_file = os.path.abspath('binary_signif_matrix.eps')
    
    plot_cormat(plot_binary_signif_matrix_file,binary_signif_matrix,list_labels = [])
    
    ## Z correl_mat as list of edges
    
    print "saving net_list as list of edges"
    
    net_List_file = os.path.abspath('net_List_signif_conf.txt')
    
    export_List_net_from_list(net_List_file,net_list)
    
    ### Z correl_mat as Louvain format
    
    print "saving net_list as Louvain format"
    
    net_Louvain_file = os.path.abspath('net_Louvain_signif_conf.txt')
    
    export_Louvain_net_from_list(net_Louvain_file,net_list,coords)
    
    #net_List_file = ''
    #net_Louvain_file = ''
    
    return net_List_file, net_Louvain_file
コード例 #6
0
ファイル: readh5.py プロジェクト: shub1905/project_apollo
def split_data(min_timbre=100, timbre_width=12, min_songs=4, data_file='mfcc'):
    ArtistMapping, ArtistIdMapping, data = generate_data(
        min_timbre=min_timbre, timbre_width=timbre_width, min_songs=min_songs)
    train = numpy.zeros((0, min_timbre * timbre_width + 1))
    validation = numpy.zeros((0, min_timbre * timbre_width + 1))
    test = numpy.zeros((0, min_timbre * timbre_width + 1))

    print 'Splitting data...'
    for artist_id in ArtistIdMapping:
        indices = ArtistIdMapping[artist_id][1]

        valid_data = data[indices[0]].reshape(1, -1)
        validation = numpy.vstack((validation, valid_data))

        test_indx = 1 + max(int((len(indices) - 1) * .3), 1)

        for i in range(1, test_indx):
            test_data = data[indices[i]].reshape(1, -1)
            test = numpy.vstack((test, test_data))

        for i in range(test_indx, len(indices)):
            train_data = data[indices[i]].reshape(1, -1)
            train = numpy.vstack((train, train_data))

    print 'Saving Data...'

    numpy.save('data/' + data_file + '_songs_{}'.format(min_songs) + '_test', test, allow_pickle=True)
    numpy.save('data/' + data_file + '_songs_{}'.format(min_songs) + '_train', train, allow_pickle=True)
    numpy.save('data/' + data_file + '_songs_{}'.format(min_songs) + '_valid', validation, allow_pickle=True)
    numpy.save('data/' + data_file + '_dict_songs_{}'.format(min_songs), ArtistMapping, allow_pickle=True)
    numpy.save('data/' + data_file + '_dictId_songs_{}'.format(min_songs), ArtistIdMapping, allow_pickle=True)
    return ArtistMapping, ArtistIdMapping, train, validation, test
コード例 #7
0
def main():

    for i in list(range(4))[::-1]:
        print(i+1)
        time.sleep(1)


    paused = False
    while(True):

        if not paused:
            # 800x600 windowed mode
            screen = grab_screen(region=(0,40,800,640))
            last_time = time.time()
            screen = cv2.cvtColor(screen, cv2.COLOR_BGR2GRAY)
            screen = cv2.resize(screen, (160,120))
            # resize to something a bit more acceptable for a CNN
            keys = key_check()
            output = keys_to_output(keys)
            training_data.append([screen,output])
            
            if len(training_data) % 1000 == 0:
                print(len(training_data))
                np.save(file_name,training_data)

        keys = key_check()
        if 'T' in keys:
            if paused:
                paused = False
                print('unpaused!')
                time.sleep(1)
            else:
                print('Pausing!')
                paused = True
                time.sleep(1)
コード例 #8
0
ファイル: matustats.py プロジェクト: simkovic/matustools
def pcaNIPALS(K=5,tol=1e-4,verbose=False):
    ''' Reference:
            Section 2.2 in Andrecut, M. (2009).
            Parallel GPU implementation of iterative PCA algorithms.
            Journal of Computational Biology, 16(11), 1593-1599.
        TODO - replace custom linear algebra (e.g. XmeanCenter) with
        numpy algebra
            
    '''
    if verbose: print 'Mean centering columns'
    XmeanCenter(1)
    latent=[]
    for k in range(K):
        lam0=0;lam1=np.inf
        T=np.matrix(XgetColumn(k))
        if verbose: print 'Computing PC ',k
        h=0
        while abs(lam1-lam0)>tol and h<100:
            P=Xleftmult(T,True)
            P=P/np.linalg.norm(P)
            T=Xleftmult(P)
            lam0=lam1
            lam1=np.linalg.norm(T)
            if verbose: print '\t Iteration '+str(h)+', Convergence =', abs(lam1-lam0)
            h+=1
        latent.append(lam1)
        XminusOuterProduct(T,P)
        #np.save(inpath+'T%02d'%k,T)
        np.save(inpath+'coeffT%d'%k,P.T)
    np.save(inpath+'latent',latent)
コード例 #9
0
    def setUp(self):
        """Generate files"""

        self.ary = numpy.random.randint(0, 6500, size=99).reshape(11, 9).astype("uint16")
        self.fn = os.path.join(UtilsTest.tempdir, "numpy.npy")
        self.fn2 = os.path.join(UtilsTest.tempdir, "numpy2.npy")
        numpy.save(self.fn, self.ary)
コード例 #10
0
def relax_system(mesh):

    sim = Sim(mesh, chi=1e-3, name='relax', driver='llbar_full')

    sim.driver.set_tols(rtol=1e-7, atol=1e-7)
    sim.Ms = 8.0e5
    sim.driver.alpha = 0.1
    sim.beta = 0
    sim.driver.gamma = 2.211e5

    sim.set_m((1, 0.25, 0.1))
    # sim.set_m(np.load('m0.npy'))

    A = 1.3e-11
    exch = UniformExchange(A=A)
    sim.add(exch)

    mT = 795.7747154594767
    zeeman = Zeeman([-100 * mT, 4.3 * mT, 0], name='H')
    sim.add(zeeman, save_field=True)

    demag = Demag()
    sim.add(demag)

    ONE_DEGREE_PER_NS = 17453292.52

    sim.relax(dt=1e-12, stopping_dmdt=0.01,
              max_steps=5000, save_m_steps=100, save_vtk_steps=50)

    np.save('m0.npy', sim.spin)
コード例 #11
0
def plotForce():
    figure(size=3,aspect=0.5)
    subplot(1,2,1)
    from EvalTraj import plotFF
    plotFF(vp=351,t=28,f=900,cm=0.6,foffset=8)
    subplot_annotate()
    
    subplot(1,2,2)
    for i in [1,2,3,4]:
        R=np.squeeze(np.load('Rdpse%d.npy'%i))
        R=stats.nanmedian(R,axis=2)[:,1:,:]
        dps=np.linspace(-1,1,201)[1:]
        plt.plot(dps,R[:,:,2].mean(0));
    plt.legend([0,0.1,0.2,0.3],loc=3) 
    i=2
    R=np.squeeze(np.load('Rdpse%d.npy'%i))
    R=stats.nanmedian(R,axis=2)[:,1:,:]
    mn=np.argmin(R,axis=1)
    y=np.random.randn(mn.shape[0])*0.00002+0.0438
    plt.plot(np.sort(dps[mn[:,2]]),y,'+',mew=1,ms=6,mec=[ 0.39  ,  0.76,  0.64])
    plt.xlabel('Displacement of Force Origin')
    plt.ylabel('Average Net Force Magnitude')
    hh=dps[mn[:,2]]
    err=np.std(hh)/np.sqrt(hh.shape[0])*stats.t.ppf(0.975,hh.shape[0])
    err2=np.std(hh)/np.sqrt(hh.shape[0])*stats.t.ppf(0.75,hh.shape[0])
    m=np.mean(hh)
    print m, m-err,m+err
    np.save('force',[m, m-err,m+err,m-err2,m+err2])
    plt.xlim([-0.5,0.5])
    plt.ylim([0.0435,0.046])
    plt.grid(b=True,axis='x')
    subplot_annotate()
コード例 #12
0
ファイル: models.py プロジェクト: davidrs06/AMICO
    def generate( self, out_path, aux, idx_in, idx_out ):
        scheme_high = amico.lut.create_high_resolution_scheme( self.scheme, b_scale = 1 )
        protocolHR = self.scheme2noddi( scheme_high )

        nATOMS = len(self.IC_ODs)*len(self.IC_VFs) + 1
        progress = ProgressBar( n=nATOMS, prefix="   ", erase=True )

        # Coupled contributions
        IC_KAPPAs = 1 / np.tan(self.IC_ODs*np.pi/2)
        for kappa in IC_KAPPAs:
            signal_ic = self.synth_meas_watson_SH_cyl_neuman_PGSE( np.array([self.dPar*1E-6, 0, kappa]), protocolHR['grad_dirs'], np.squeeze(protocolHR['gradient_strength']), np.squeeze(protocolHR['delta']), np.squeeze(protocolHR['smalldel']), np.array([0,0,1]), 0 )

            for v_ic in self.IC_VFs:
                dPerp = self.dPar*1E-6 * (1 - v_ic)
                signal_ec = self.synth_meas_watson_hindered_diffusion_PGSE( np.array([self.dPar*1E-6, dPerp, kappa]), protocolHR['grad_dirs'], np.squeeze(protocolHR['gradient_strength']), np.squeeze(protocolHR['delta']), np.squeeze(protocolHR['smalldel']), np.array([0,0,1]) )

                signal = v_ic*signal_ic + (1-v_ic)*signal_ec
                lm = amico.lut.rotate_kernel( signal, aux, idx_in, idx_out, False )
                np.save( pjoin( out_path, 'A_%03d.npy'%progress.i) , lm )
                progress.update()

        # Isotropic
        signal = self.synth_meas_iso_GPD( self.dIso*1E-6, protocolHR)
        lm = amico.lut.rotate_kernel( signal, aux, idx_in, idx_out, True )
        np.save( pjoin( out_path, 'A_%03d.npy'%progress.i) , lm )
        progress.update()
コード例 #13
0
def vectorize_and_aggregate(in_data_file_list, mask_file, matrix_name, parcellation_path, fwhm, use_diagonal,
                            use_fishers_z, df_file, df_col_names):
    import os, pickle
    import numpy as np
    from LeiCA_LIFE.learning.prepare_data_utils import vectorize_ss

    # get an example of the data:
    #save_template: template file; for behav: col names
    vectorized_data, data_type, masker, save_template = vectorize_ss(in_data_file_list[0], mask_file, matrix_name,
                                                                     parcellation_path, fwhm, use_diagonal,
                                                                     use_fishers_z, df_file,
                                                                     df_col_names)
    vectorized_data = np.zeros((len(in_data_file_list), vectorized_data.shape[1]))
    vectorized_data.fill(np.nan)

    for i, in_data_file_ss in enumerate(in_data_file_list):
        vectorized_data[i, :], _, _, _ = vectorize_ss(in_data_file_ss, mask_file, matrix_name, parcellation_path, fwhm,
                                                      use_diagonal, use_fishers_z, df_file, df_col_names)

    vectorized_aggregated_file = os.path.abspath('vectorized_aggregated_data.npy')
    np.save(vectorized_aggregated_file, vectorized_data)

    unimodal_backprojection_info = {'data_type': data_type,
                                    'masker': masker,
                                    'save_template': save_template
                                    }
    unimodal_backprojection_info_file = os.path.abspath('unimodal_backprojection_info.pkl')
    pickle.dump(unimodal_backprojection_info, open(unimodal_backprojection_info_file, 'w'))
    return vectorized_aggregated_file, unimodal_backprojection_info_file
コード例 #14
0
ファイル: features.py プロジェクト: fcchou/CS229-project
 def labels(self):
     '''
     Decide the labels
     2 for unsecure Jos, 1 for secure Jos, 0 for others
     '''
     #TODO consider labeling different authors for multiclass assignment
     fname = 'features/labels'
     if self.use_saved_features and path.isfile(fname):
         return np.load(fname)
     with open(self._get_path('work_list/Josquin_secure.txt')) as f:
         secure_jos = set(f.read().splitlines())
     labels = []
     for work in self._works:
         label = 0
         if 'Ock' in work:
             label = -1
         if 'Jos' in work:
             if work in secure_jos:
                 label = 1
             else:
                 label = 2
         labels.append(label)
     labels = np.array(labels, dtype=int)
     if self.save_data:
         np.save(fname, labels)
     return labels
コード例 #15
0
ファイル: features.py プロジェクト: fcchou/CS229-project
    def __init__(
            self, save_data=True,
            use_saved_features=True, use_saved_npz=True):
        'Init by getting all the works.'
        self._self_dir = path.abspath(path.dirname(__file__))
        self.use_saved_features = use_saved_features
        self.use_saved_npz = use_saved_npz
        self.save_data = save_data
        self.npz_data = None
        self._vectorizer = DictVectorizer()

        if save_data and not path.isdir('features'):
            os.mkdir('features')

        work_fname = 'features/all_works.npy'
        if use_saved_features and path.isfile(work_fname):
            self._works = np.load(work_fname)
        else:
            works = []
            with open(self._get_path('work_list/AllWorks.txt')) as f:
                for line in f:
                    works.append(line.split('-')[0])
            self._works = np.array(works)
            if self.save_data:
                np.save('features/all_works', self.works)
コード例 #16
0
ファイル: ben_run.py プロジェクト: nickrodd/NPTFWorking2
def save_medians(b_inst,save_path):
    medians = b_inst.medians
    poiss_comp_numbers = b_inst.poiss_comp_numbers

    save_vec = [ [key, medians[ poiss_comp_numbers[key] ] ] for key in poiss_comp_numbers.keys() ]

    np.save(save_path,save_vec)
コード例 #17
0
def sample_lnprob(weight_index):
    import emcee

    ndim = 4
    nwalkers = 8 * ndim
    print("using {} walkers".format(nwalkers))
    p0 = np.vstack((np.random.uniform(-0.5, 2, size=(1, nwalkers)),
                    np.random.uniform(50, 300, size=(1, nwalkers)),
                    np.random.uniform(0.2, 1.5, size=(1, nwalkers)),
                    np.random.uniform(0.2, 1.5, size=(1, nwalkers)))).T

    sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=(weight_index,), threads=cfg['threads'])

    print("Running Sampler")
    pos, prob, state = sampler.run_mcmc(p0, cfg['burn_in'])

    print("Burn-in complete")
    sampler.reset()
    sampler.run_mcmc(pos, cfg['samples'])

    samples = sampler.flatchain
    np.save(cfg['outdir'] + "samples_w{}.npy".format(weight_index), samples)

    import triangle
    fig = triangle.corner(samples)
    fig.savefig(cfg['outdir'] + "triangle_w{}.png".format(weight_index))
コード例 #18
0
ファイル: werner.py プロジェクト: robintw/WernerModel
 def run(self, y, x, initial_value, num_its, altered):
     """Run the model with the arguments: y_length, x_length, initial_value, number_of_iterations, altered"""
     self.initialise_grid(y, x, initial_value)
     
     self.write_file()
     
     self.initialise_shadow_map()
     
     self.num_iterations = num_its
     
     # Standard parameter values
     self.jump_length = 1
     self.pd_s = 0.6
     self.pd_ns = 0.4
     
     self.altered = altered
     
     if self.altered == True:
         # Create the depth grid
         self.depth = np.load("Gradual_Stepped_Full.npy")
     
     self.avcount = np.zeros(num_its + 1)
     
     # Run the model
     self.main_loop()
     
     print self.avcount
     io.savemat("Counts.mat", { "count":self.avcount})
     np.save("Counts.npy", self.avcount)
コード例 #19
0
ファイル: werner.py プロジェクト: robintw/WernerModel
 def run_altered(self, depth):
     x = 500
     y = 200
     initial_value = 3
     num_its = 2000
     
     self.initialise_grid(y, x, initial_value)
     
     self.write_file()
     
     self.initialise_shadow_map()
     
     self.num_iterations = num_its
     
     # Standard parameter values
     self.jump_length = 1
     self.pd_s = 0.6
     self.pd_ns = 0.4
     
     self.create_gradual_grid(depth)
     
     self.altered = True
     
     self.avcount = np.zeros(num_its + 1)
     
     # Run the model
     self.main_loop()
     
     print self.avcount
     io.savemat("Counts.mat", { "count":self.avcount})
     np.save("Counts.npy", self.avcount)
コード例 #20
0
def calculate_and_save():
    energies = np.empty((B_list.size, 2**number_of_spins))
    for i,B in enumerate(B_list):
        print i
        calc = ising_calculator_AFM(number_of_spins, alpha, B)
        energies[i,:] = calc.find_energies()
    np.save('energy_array', energies)
コード例 #21
0
ファイル: figure1.py プロジェクト: takluyver/fidimag
def relax_system(mesh):

    sim = Sim(mesh, name='relax')
    # sim.set_options(rtol=1e-10,atol=1e-14)
    sim.alpha = 1.0
    sim.gamma = 1.0
    sim.mu_s = 1.0

    sim.set_m(init_m)
    # sim.set_m(random_m)
    # sim.set_m(np.load('m_10000.npy'))

    J = 1.0
    exch = UniformExchange(J)
    sim.add(exch)

    D = 0.09
    dmi = DMI(D)
    sim.add(dmi)

    zeeman = Zeeman([0, 0, 3.75e-3])
    sim.add(zeeman)

    sim.relax(dt=2.0, stopping_dmdt=1e-6, max_steps=1000,
              save_m_steps=100, save_vtk_steps=50)

    np.save('m0.npy', sim.spin)
コード例 #22
0
ファイル: __init__.py プロジェクト: djkirkham/iris
    def assertCubeDataAlmostEqual(self, cube, reference_filename, *args, **kwargs):
        reference_path = self.get_result_path(reference_filename)
        if self._check_reference_file(reference_path):
            kwargs.setdefault("err_msg", "Reference file %s" % reference_path)

            result = np.load(reference_path)
            if isinstance(result, np.lib.npyio.NpzFile):
                self.assertIsInstance(cube.data, ma.MaskedArray, "Cube data was not a masked array.")
                # Avoid comparing any non-initialised array data.
                data = cube.data.filled()
                np.testing.assert_array_almost_equal(data, result["data"], *args, **kwargs)
                np.testing.assert_array_equal(cube.data.mask, result["mask"])
            else:
                np.testing.assert_array_almost_equal(cube.data, result, *args, **kwargs)
        else:
            self._ensure_folder(reference_path)
            logger.warning("Creating result file: %s", reference_path)
            if isinstance(cube.data, ma.MaskedArray):
                # Avoid recording any non-initialised array data.
                data = cube.data.filled()
                with open(reference_path, "wb") as reference_file:
                    np.savez(reference_file, data=data, mask=cube.data.mask)
            else:
                with open(reference_path, "wb") as reference_file:
                    np.save(reference_file, cube.data)
コード例 #23
0
def train_word2id():
    """把训练集的所有词转成对应的id。"""
    time0 = time.time()
    print('Processing train data.')
    df_train = pd.read_csv('../raw_data/question_train_set.txt', sep='\t', usecols=[0, 2, 4],
                           names=['question_id', 'word_title', 'word_content'], dtype={'question_id': object})
    print('training question number %d ' % len(df_train))
    # 没有 content 的问题用 title 来替换
    na_content_indexs = list()
    for i in tqdm(xrange(len(df_train))):
        word_content = df_train.word_content.values[i]
        if type(word_content) is float:
            na_content_indexs.append(i)
    print('There are %d train questions without content.' % len(na_content_indexs))
    for na_index in tqdm(na_content_indexs):
        df_train.at[na_index, 'word_content'] = df_train.at[na_index, 'word_title']
    # 没有 title 的问题, 丢弃
    na_title_indexs = list()
    for i in xrange(len(df_train)):
        word_title = df_train.word_title.values[i]
        if type(word_title) is float:
            na_title_indexs.append(i)
    print('There are %d train questions without title.' % len(na_title_indexs))
    df_train = df_train.drop(na_title_indexs)
    print('After dropping, training question number(should be 2999952) = %d' % len(df_train))
    # 转为 id 形式
    p = Pool()
    train_title = np.asarray(p.map(get_id4words, df_train.word_title.values))
    np.save('../data/wd_train_title.npy', train_title)
    train_content = np.asarray(p.map(get_id4words, df_train.word_content.values))
    np.save('../data/wd_train_content.npy', train_content)
    p.close()
    p.join()
    print('Finished changing the training words to ids. Costed time %g s' % (time.time() - time0))
コード例 #24
0
def concat_ts(all_ts_files):
    
    import numpy as np
    import os
    
    print all_ts_files

    for i,ts_file in enumerate(all_ts_files):
    
        ## loading ROI coordinates
        ts = np.load(ts_file)
        
        #print "all_ts: " 
        print ts.shape
        
        if i == 0:
            concat_ts = ts.copy()
            #print concat_ts.shape
        else:
            concat_ts = np.concatenate((concat_ts,ts),axis = 0)
            #print concat_ts.shape

    print concat_ts.shape

    ### saving time series
    concat_ts_file = os.path.abspath("concat_ts.npy")
    np.save(concat_ts_file,concat_ts)
    
        
    return concat_ts_file
コード例 #25
0
def test_word2id():
    """把测试集的所有词转成对应的id。"""
    time0 = time.time()
    print('Processing eval data.')
    df_eval = pd.read_csv('../raw_data/question_eval_set.txt', sep='\t', usecols=[0, 2, 4],
                          names=['question_id', 'word_title', 'word_content'], dtype={'question_id': object})
    print('test question number %d' % len(df_eval))
    # 没有 title 的问题用 content 来替换
    na_title_indexs = list()
    for i in xrange(len(df_eval)):
        word_title = df_eval.word_title.values[i]
        if type(word_title) is float:
            na_title_indexs.append(i)
    print('There are %d test questions without title.' % len(na_title_indexs))
    for na_index in na_title_indexs:
        df_eval.at[na_index, 'word_title'] = df_eval.at[na_index, 'word_content']
    # 没有 content 的问题用 title 来替换
    na_content_indexs = list()
    for i in tqdm(xrange(len(df_eval))):
        word_content = df_eval.word_content.values[i]
        if type(word_content) is float:
            na_content_indexs.append(i)
    print('There are %d test questions without content.' % len(na_content_indexs))
    for na_index in tqdm(na_content_indexs):
        df_eval.at[na_index, 'word_content'] = df_eval.at[na_index, 'word_title']
    # 转为 id 形式
    p = Pool()
    eval_title = np.asarray(p.map(get_id4words, df_eval.word_title.values))
    np.save('../data/wd_eval_title.npy', eval_title)
    eval_content = np.asarray(p.map(get_id4words, df_eval.word_content.values))
    np.save('../data/wd_eval_content.npy', eval_content)
    p.close()
    p.join()
    print('Finished changing the eval words to ids. Costed time %g s' % (time.time() - time0))
コード例 #26
0
def main(root='/tmp/measurements', output=None):
    data = []
    for s in os.listdir(root):
        subject = []
        for b in os.listdir(os.path.join(root, s)):
            block = []
            bweight, bspeed, bhand, bpaths = b.split('-')[1:]
            for t in os.listdir(os.path.join(root, s, b)):
                thand, tspeed = re.search(r'(left|right)-speed_(\d\.\d+)', t).groups()
                config = np.tile([
                    C[bweight], C[bspeed], C[bhand], C[bpaths],
                    C[thand], float(tspeed)], (120, 1))
                block.append(
                    np.hstack([
                        config,
                        np.loadtxt(os.path.join(root, s, b, t),
                                   skiprows=1, delimiter=',')]))
            subject.append(block)
        if len(subject) == 3:
            data.append(subject)
        else:
            print('incorrect block count! discarding {}'.format(s))
    data = np.array(data)
    logging.info('loaded data %s', data.shape)
    if output:
        np.save(output, data.astype('f'))
コード例 #27
0
ファイル: Picoscope2000.py プロジェクト: kipfer/pqpico
 def get_buffer_callback(overviewBuffers,overflow,triggeredAt,triggered,auto_stop,nValues):
     
     #print('Callback for saving to disk')
     #create filename based on actual timestamp
     #filename = time.strftime("%Y%m%d_%H_%M_%S_%f.csv")
     filename=datetime.datetime.now()
     filename= filename.strftime("%Y%m%d_%H_%M_%S_%f")
     CH1='CH1_' + filename 
     #CH2='CH2_' + filename
     
     #cast 2d-pointer from c- callback into python pointer 
     ob = ctypes.cast(overviewBuffers,ctypes.POINTER(ctypes.POINTER(ctypes.c_short)))
     
     #create array from pointer data ob[0]-> CH1 ob[1]-> CH2
     streamed_data_CH1=np.fromiter(ob[0], dtype=np.short, count=nValues)
     #streamed_data_CH2=np.fromiter(ob[1], dtype=np.short, count=nValues)
                 
     #save array data into numpy fileformat
     path1 = os.path.normpath('C:\\Users\ckattmann\Documents\GitHub\pqpico\Data')+'/'+CH1
     #path2 = os.path.normpath('C:\\Users\ckattmann\Documents\GitHub\pqpico\Data')+'/'+CH2
                 
     np.save(path1,streamed_data_CH1)
     #np.save(path2,streamed_data_CH2)
     #print('File saved:',CH1,CH2)
     
     return 0
コード例 #28
0
ファイル: base_processor.py プロジェクト: Riashat/betago
    def consolidate_games(self, name, samples):
        print('>>> Creating consolidated numpy arrays')

        if self.use_generator:
            print('>>> Return generator')
            generator = DataGenerator(self.data_dir, samples)
            return generator

        files_needed = set(file_name for file_name, index in samples)
        print('>>> Total number of files: ' + str(len(files_needed)))

        file_names = []
        for zip_file_name in files_needed:
            file_name = zip_file_name.replace('.zip', '') + name
            file_names.append(file_name)

        feature_list = []
        label_list = []
        for file_name in file_names:
            X = np.load(self.data_dir + '/' + file_name + '_features.npy')
            y = np.load(self.data_dir + '/' + file_name + '_labels.npy')
            feature_list.append(X)
            label_list.append(y)
            print('>>> Done')

        features = np.concatenate(feature_list, axis=0)
        labels = np.concatenate(label_list, axis=0)

        feature_file = self.data_dir + '/' + str(self.num_planes) + '_plane_features_' + name
        label_file = self.data_dir + '/' + str(self.num_planes) + '_plane_labels_' + name

        np.save(feature_file, features)
        np.save(label_file, labels)

        return features, labels
コード例 #29
0
ファイル: ListaPrvd_regr.py プロジェクト: lelegan/DLSR
def Cluster(param, DATA_FOLDER):
    ts = ListaSet(param)
    Data = scipy.io.loadmat('./data/filter_template3.mat')
    Filter2 = np.rot90( Data['Filter2'], 2)

#    corList = []
#    TVList= []
    corArr = np.empty(ts.get_num_images())
    TVArr = np.empty(ts.get_num_images())

    for i in range( ts.get_num_images()):
#    for i in range( 100):
        tmp = ts.get_input(i)
        tmp2 = tmp - np.mean( tmp)
        tmp3 = tmp2 / np.linalg.norm(tmp2, 'fro')
#        Cor = scipy.signal.convolve2d(tmp3, Filter2, 'valid')
#        corList.append( Cor)    
        corArr[i] = scipy.signal.convolve2d(tmp3, Filter2, 'valid')
    
        dx = scipy.ndimage.sobel(tmp, 0)
        dy = scipy.ndimage.sobel(tmp, 1)
        mag = np.hypot(dx, dy)
#        TVList.append( np.sum(mag))
        TVArr[i] = np.sum(mag)

        if i % 10000 == 0:
            print i

    np.save( DATA_FOLDER+'/trainCorrelation.npy', corArr)
    np.save( DATA_FOLDER+'/trainTotalVariation.npy', TVArr)
    return
コード例 #30
0
ファイル: read_overlap.py プロジェクト: atlytle/tifr
def convert_single_propagators(files):
    "Construct pion correlators from individual overlap propagators."
    # Some basic checks on the input.
    head0, config0, middle0, mass0 = files[0].split('.')
    for f in files:
        check_length(f)
        head, config, middle, mass = f.split('.')
        if (head != head0) or (middle != middle0) or (mass != mass0):
            print "You might not want to combine these!"
            return 1

    # Construct the block of correlators.
    correlators = []
    for f in files:
        print f
        correlators.append(pion_correlator(f))
    correlators = np.array(correlators)

    # Basic checks on the output.
    print correlators.shape
    assert (len(files), nt) == correlators.shape

    # Write output.
    m = float('0.'+mass0)
    np.save(correlator_name(m), correlators) 
    print correlators[0]
コード例 #31
0
ファイル: data_preprocess_3D.py プロジェクト: jayantp07/Kriti
def create_test_data(patient):
    testing_data = []
    img, origin, spacing = load_itk("./MHA/" + patient + ".mha")
    RESIZE_SPACING = [1, 1, 1]
    resize_factor = spacing / RESIZE_SPACING
    new_real_shape = img.shape * resize_factor
    new_shape = np.round(new_real_shape)
    real_resize = new_shape / img.shape
    new_spacing = spacing / real_resize
    lung_img = np.load("./images1/" + patient + ".npy")
    world_pos, world_neg = generator(patient)
    for i in range(world_pos.shape[0]):
        voxel = world_2_voxel(world_pos[i, :][::-1], origin, spacing)
        voxel = np.round(voxel).astype('int')
        voxel = np.round(voxel * resize_factor).astype('int')
        z = voxel[0]
        y = voxel[1]
        x = voxel[2]
        t1_slice = lung_img[z - 16:z + 16, y - 16:y + 16, x - 16:x + 16].copy()
        t1_slice = resize_image_with_crop_or_pad(t1_slice, [32, 32, 32],
                                                 mode='symmetric')
        print("Processing Positive ", patient)
        print(t1_slice.shape, z, y, x)

        # Add a feature dimension and normalise
        t1_norm = np.expand_dims(normalise_one_one(t1_slice), axis=-1)

        # Randomly flip the image along axis 1
        t1_flipped1 = flip1(t1_norm.copy(), axis=2)

        # Add a Gaussian offset (independently for each channel)
        t1_offset1 = add_gaussian_offset(t1_norm.copy(), sigma=0.1)
        t1_offset2 = add_gaussian_offset(t1_norm.copy(), sigma=0.5)

        # Add Gaussian noise
        t1_noise1 = add_gaussian_noise(t1_norm.copy(), sigma=0.02)
        t1_noise2 = add_gaussian_noise(t1_norm.copy(), sigma=0.05)

        # Elastic transforms according to:
        # [1] Simard, Steinkraus and Platt, "Best Practices for Convolutional
        #     Neural Networks applied to Visual Document Analysis", in Proc. of the
        #     International Conference on Document Analysis and Recognition, 2003.
        t1_trans_low_s = elastic_transform(t1_norm.copy(),
                                           alpha=[1, 1e4, 1e4],
                                           sigma=[1, 8, 8])
        t1_trans_high_s = elastic_transform(t1_norm.copy(),
                                            alpha=[1, 6e4, 6e4],
                                            sigma=[1, 16, 16])

        testing_data.append([np.array(t1_norm), np.array([0, 1])])
        testing_data.append([np.array(t1_flipped1), np.array([0, 1])])
        testing_data.append([np.array(t1_offset1), np.array([0, 1])])
        testing_data.append([np.array(t1_offset2), np.array([0, 1])])
        testing_data.append([np.array(t1_noise1), np.array([0, 1])])
        testing_data.append([np.array(t1_noise2), np.array([0, 1])])
        testing_data.append([np.array(t1_trans_low_s), np.array([0, 1])])
        testing_data.append([np.array(t1_trans_high_s), np.array([0, 1])])

    util = world_neg.shape[0]
    if (world_neg.shape[0] > (2 * world_pos.shape[0])):
        util = 2 * world_pos.shape[0]

    for i in range(util):
        voxel = world_2_voxel(world_neg[i, :][::-1], origin, spacing)
        voxel = np.round(voxel).astype('int')
        voxel = np.round(voxel * resize_factor).astype('int')
        z = voxel[0]
        y = voxel[1]
        x = voxel[2]
        t1_slice = lung_img[z - 16:z + 16, y - 16:y + 16, x - 16:x + 16].copy()
        t1_slice = resize_image_with_crop_or_pad(t1_slice, [32, 32, 32],
                                                 mode='symmetric')
        print("Processing Negative", patient)
        print(t1_slice.shape, z, y, x)

        # Add a feature dimension and normalise
        t1_norm = np.expand_dims(normalise_one_one(t1_slice), axis=-1)

        # Randomly flip the image along axis 1
        t1_flipped1 = flip1(t1_norm.copy(), axis=2)

        # Add a Gaussian offset (independently for each channel)
        t1_offset1 = add_gaussian_offset(t1_norm.copy(), sigma=0.1)

        # Add Gaussian noise
        t1_noise1 = add_gaussian_noise(t1_norm.copy(), sigma=0.02)

        # Elastic transforms according to:
        # [1] Simard, Steinkraus and Platt, "Best Practices for Convolutional
        #     Neural Networks applied to Visual Document Analysis", in Proc. of the
        #     International Conference on Document Analysis and Recognition, 2003.
        t1_trans_low_s = elastic_transform(t1_norm.copy(),
                                           alpha=[1, 1e4, 1e4],
                                           sigma=[1, 8, 8])

        testing_data.append([np.array(t1_norm), np.array([1, 0])])
        testing_data.append([np.array(t1_flipped1), np.array([1, 0])])
        testing_data.append([np.array(t1_offset1), np.array([1, 0])])
        testing_data.append([np.array(t1_noise1), np.array([1, 0])])
        testing_data.append([np.array(t1_trans_low_s), np.array([1, 0])])

        print("#############################")
        print()

    shuffle(testing_data)
    np.save(testing_data_output_path + patient + ".npy", testing_data)
    X = np.array([i[0] for i in testing_data]).reshape(-1, 32, 32, 32, 1)
    Y = np.array([i[1] for i in testing_data])
    print(X.shape, Y.shape)
コード例 #32
0
def cal_cosod_matrics(
    data_type: str = "rgb_sod",
    txt_path: str = "",
    to_append: bool = True,
    xlsx_path: str = "",
    drawing_info: dict = None,
    dataset_info: dict = None,
    save_npy: bool = True,
    curves_npy_path: str = "./curves.npy",
    metrics_npy_path: str = "./metrics.npy",
    num_bits: int = 3,
):
    """
    Save the results of all models on different datasets in a `npy` file in the form of a
    dictionary.

    ::

        {
          dataset1:{
            method1:[fm, em, p, r],
            method2:[fm, em, p, r],
            .....
          },
          dataset2:{
            method1:[fm, em, p, r],
            method2:[fm, em, p, r],
            .....
          },
          ....
        }

    :param data_type: the type of data
    :param txt_path: the path of the txt for saving results
    :param to_append: whether to append results to the original record
    :param xlsx_path: the path of the xlsx file for saving results
    :param drawing_info: the method information for plotting figures
    :param dataset_info: the dataset information
    :param save_npy: whether to save results into npy files
    :param curves_npy_path: the npy file path for saving curve data
    :param metrics_npy_path: the npy file path for saving metric values
    :param num_bits: the number of bits used to format results
    """
    curves = defaultdict(dict)  # Two curve metrics
    metrics = defaultdict(dict)  # Six numerical metrics

    txt_recoder = TxtRecorder(
        txt_path=txt_path,
        to_append=to_append,
        max_method_name_width=max([len(x) for x in drawing_info.keys()]),  # 显示完整名字
    )
    excel_recorder = MetricExcelRecorder(
        xlsx_path=xlsx_path,
        sheet_name=data_type,
        row_header=["methods"],
        dataset_names=sorted(list(dataset_info.keys())),
        metric_names=["sm", "wfm", "mae", "adpf", "avgf", "maxf", "adpe", "avge", "maxe"],
    )

    for dataset_name, dataset_path in dataset_info.items():
        txt_recoder.add_row(row_name="Dataset", row_data=dataset_name, row_start_str="\n")

        # 获取真值图片信息
        gt_info = dataset_path["mask"]
        gt_root = gt_info["path"]
        gt_ext = gt_info["suffix"]
        # 真值名字列表
        gt_index_file = dataset_path.get("index_file")
        if gt_index_file:
            gt_name_list = get_name_with_group_list(data_path=gt_index_file, file_ext=gt_ext)
        else:
            gt_name_list = get_name_with_group_list(data_path=gt_root, file_ext=gt_ext)
        assert len(gt_name_list) > 0, "there is not ground truth."

        # ==>> test the intersection between pre and gt for each method <<==
        for method_name, method_info in drawing_info.items():
            method_root = method_info["path_dict"]
            method_dataset_info = method_root.get(dataset_name, None)
            if method_dataset_info is None:
                colored_print(
                    msg=f"{method_name} does not have results on {dataset_name}", mode="warning"
                )
                continue

            # 预测结果存放路径下的图片文件名字列表和扩展名称
            pre_ext = method_dataset_info["suffix"]
            pre_root = method_dataset_info["path"]
            pre_name_list = get_name_with_group_list(data_path=pre_root, file_ext=pre_ext)

            # get the intersection
            eval_name_list = sorted(list(set(gt_name_list).intersection(set(pre_name_list))))
            num_names = len(eval_name_list)

            if num_names == 0:
                colored_print(
                    msg=f"{method_name} does not have results on {dataset_name}", mode="warning"
                )
                continue

            grouped_data = group_names(names=eval_name_list)
            num_groups = len(grouped_data)

            colored_print(
                f"Evaluating {method_name} with {num_names} images and {num_groups} groups"
                f" (G:{len(gt_name_list)},P:{len(pre_name_list)}) images on dataset {dataset_name}"
            )

            group_metric_recorder = GroupedMetricRecorder()
            inter_group_bar = tqdm(
                grouped_data.items(),
                total=num_groups,
                leave=False,
                ncols=79,
                desc=f"[{dataset_name}]",
            )
            for group_name, names_in_group in inter_group_bar:
                intra_group_bar = tqdm(
                    names_in_group,
                    total=len(names_in_group),
                    leave=False,
                    ncols=79,
                    desc=f"({group_name})",
                )
                for img_name in intra_group_bar:
                    img_name_with_group = os.path.join(group_name, img_name)
                    gt, pre = get_gt_pre_with_name(
                        gt_root=gt_root,
                        pre_root=pre_root,
                        img_name=img_name_with_group,
                        pre_ext=pre_ext,
                        gt_ext=gt_ext,
                        to_normalize=False,
                    )
                    group_metric_recorder.update(group_name=group_name, pre=pre, gt=gt)
            method_results = group_metric_recorder.show(num_bits=num_bits, return_ndarray=False)
            method_curves = method_results["sequential"]
            method_metrics = method_results["numerical"]
            curves[dataset_name][method_name] = method_curves
            metrics[dataset_name][method_name] = method_metrics

            excel_recorder(
                row_data=method_metrics, dataset_name=dataset_name, method_name=method_name
            )
            txt_recoder(method_results=method_metrics, method_name=method_name)

    if save_npy:
        make_dir(os.path.basename(curves_npy_path))
        np.save(curves_npy_path, curves)
        np.save(metrics_npy_path, metrics)
        colored_print(f"all methods have been saved in {curves_npy_path} and {metrics_npy_path}")
    formatted_string = formatter_for_tabulate(metrics)
    colored_print(f"all methods have been tested:\n{formatted_string}")
コード例 #33
0
ファイル: dataset_tool.py プロジェクト: JcYBalaBalA/GAN-Leaks
 def add_labels(self, labels):
     if self.print_progress:
         print('%-40s\r' % 'Saving labels...', end='', flush=True)
     assert labels.shape[0] == self.cur_images
     with open(self.tfr_prefix + '-rxx.labels', 'wb') as f:
         np.save(f, labels.astype(np.float32))
コード例 #34
0
    # vectorize
    cross_corr = np.corrcoef(sub_FS.T)
    tril_inds = np.tril_indices_from(cross_corr, k=-1)
    cc_ravel = cross_corr[tril_inds]

    # cross_corrs.append(cc_ravel)
    cur_FS[i_rs] = cc_ravel

    # plt.matshow(cross_corr, cmap=plt.cm.RdBu)
    # plt.title(rs_base)
    # plt.yticks(np.arange(len(roi_names)), roi_names)
    # plt.show()
    # plt.colorbar()

    if i_rs % 10 == 0:
        np.save('dump_FS_corrs', arr=cur_FS)

# dump the engineered feature space and corresponding meta-info
cur_FS = np.nan_to_num(cur_FS)
joblib.dump((cur_FS, niipaths_all, labels_all, sub_ids_all),
            'all_FS_paths_labels_subids',
            compress=9)

stuff = joblib.load('all_FS_paths_labels_subids')
FS, paths, labels, sub_ids = stuff

# trim cohorts by age and by gender (only males)
import pandas
meta_autism = pandas.read_excel('Finalinput_ABIDE_ASD_TD_MALE_BELOW21.xlsx')
meta_adhd = pandas.read_excel('Finalinput_ADHD200_ADHD_TD_MALE_BELOW21.xlsx')
コード例 #35
0
def main(args):

    if len(args) != 1:
        sys.exit("Usage: python origianl.py <graphml file>")
    
    C=3
    L=6 #损失值
    select_strength=1 #选择强度
    fname = args[0]
    G = networkx.read_graphml(fname)
    G = networkx.convert_node_labels_to_integers(G)
    
    '''
        给每个参与者初始化投资额,利用之前initial_investment.npy文件
    '''
    a=[]    
    defender = [Defender(i) for i in range(0, len(G))]
    for i in range(len(G)):
        defender[i].set_degree(G.degree()[i])
    for i in range(len(G)):
        if defender[i].get_degree()<2:
            defender[i].set_investment(random.uniform(0,0.2))
        elif defender[i].get_degree()<4 and defender[i].get_degree()>=2:
            defender[i].set_investment(random.uniform(0.2,0.4))
        elif defender[i].get_degree()<6 and defender[i].get_degree()>=4:
            defender[i].set_investment(random.uniform(0.4,0.6))
        elif defender[i].get_degree()<8 and defender[i].get_degree()>=6:
            defender[i].set_investment(random.uniform(0.6,0.8))
        elif defender[i].get_degree()>=8:
            defender[i].set_investment(random.uniform(0.8,1))
        a.append(defender[i].get_investment())
    numpy.save('initial_investment.npy',a)
    
    
    for i in range(len(G)):
        sum_weight=0.0+defender[i].get_degree()
        for j in neighbors(G,i):
            sum_weight+=defender[j].get_degree()
        defender[i].set_weight((defender[i].get_degree())/sum_weight)
    
    
    #pdb.set_trace() 
    '''开始演化博弈'''  
    for i in range(300):    
        #计算每个参与者的风险和收益  
        for j in range(0,len(G)):
            sum=0
            for k in neighbors(G,j):
                sum = sum+defender[k].get_weight()*defender[k].get_investment()
            defender[j].set_risk(math.exp(-sum-defender[j].get_weight()*defender[j].get_investment()))
            defender[j].set_payoff(-C*defender[j].get_investment()-L*defender[j].get_risk())
        
        if i==0:
            pp=0
            for jo in range(0,len(G)):
                pp=pp+defender[jo].get_payoff()
            print pp
        
        '''更新策略'''       
        for jj in range(0,len(G)):
            jjj=random_neighbor(G, jj)          
            if defender[jj].get_payoff() < defender[jjj].get_payoff():
                imitation_probabily=1.0/(1+math.exp((-select_strength)*(defender[jjj].get_payoff()-defender[jj].get_payoff())))
                if random.random() <= imitation_probabily:
                    defender[jj].set_investment_update(defender[jjj].get_investment())
                    #defender[jj].set_investment_update(((defender[jj].get_degree()+0.1)/defender[jjj].get_degree()+0.1)*defender[jjj].get_investment())    
                else:
                    defender[jj].set_investment_update((defender[jj]).get_investment())
            else:
                (defender[jj]).set_investment_update((defender[jj]).get_investment())
        
        
        for jjjj in range(0,len(G)):
            defender[jjjj].set_investment(defender[jjjj].get_investment_update())
            
        '''
    for j in range(0,len(G)): 
        print defender[j].get_investment()
        
        '''
        
        p=0
        for j in range(0,len(G)):
            p=p+defender[j].get_payoff()
        print p  
コード例 #36
0
        ph_vol = ph_vol[:, offset:-offset, offset:-offset]

        # psnr and pearsonr correlation
        mse_score = np.mean(np.square(pr_vol - gt_vol))
        psnr_score = calculate_psnr(pr_vol, gt_vol)
        cor_score = calculate_pearsonr(pr_vol, gt_vol)
        mse_scores.append(mse_score)
        psnr_scores.append(psnr_score)
        cor_scores.append(cor_score)

        # save prediction
        pred_save = True
        if pred_save:
            pr_vol_dir = model_folder + '/pred_vols'
            generate_folder(pr_vol_dir)
            np.save(os.path.join(pr_vol_dir, 'Pr_{}.npy'.format(vol_fn)),
                    pr_vol)
            np.save(os.path.join(pr_vol_dir, 'GT_{}.npy'.format(vol_fn)),
                    gt_vol)
            print(pr_vol.shape)

        print('{}: psnr {:.4f}, cor {:.4f}, mse {:.4f}\n'.format(
            vol_fn, psnr_score, cor_score, mse_score))

        # save prediction examples
        prediction_dir = model_folder + '/pred_examples'
        generate_folder(prediction_dir)
        plot_fig_file = prediction_dir + '/{}.png'.format(vol_fn)
        z_index = 158
        x_index = 250
        plot_prediction_zx(plot_fig_file, ph_vol, gt_vol, pr_vol, z_index,
                           x_index)
コード例 #37
0
ファイル: layers.py プロジェクト: yutarochan/Theano-MPI
 def save_weight(self, dir, name):
     #print 'weight saved: ' + name
     np.save(dir + name + '.npy', self.val.get_value())
コード例 #38
0
        count=int(width) * int(height),
        offset=len(header)).reshape((int(height), int(width)))


# fullFile = '/home/nick/data/1pdata/imag003/20181217_noiseburst/fc2_save_2018-12-17-121550-0000.pgm'
dataDir = '/home/nick/data/1pdata/imag003/20181217_noiseburst/'
dataFiles = sorted(os.listdir(dataDir))

#Only load every 4th frame
skipBy = 4

im = read_pgm(os.path.join(dataDir, dataFiles[0]))
indsToRead = range(0, len(dataFiles), skipBy)
imgArr = np.empty((len(indsToRead), im.shape[0], im.shape[1]))

for indFrame, indFile in enumerate(indsToRead):
    im = read_pgm(os.path.join(dataDir, dataFiles[indFile]))
    imgArr[indFrame, :, :] = im

#Save out intermediate data
saveDir = '/home/nick/data/1pdata/tmpData'
outputFn = 'imag003_noiseburst_skipby{}.npy'.format(skipBy)

np.save(os.path.join(saveDir, outputFn), imgArr.astype('uint8'))

# im = read_pgm(fullFile)

# plt.clf()
# plt.imshow(im)
# plt.show()
コード例 #39
0
    sess = tf.InteractiveSession()

    caption_generator = Caption_Generator(
        n_words=n_words,
        dim_embed=dim_embed,
        dim_ctx=dim_ctx,
        dim_hidden=dim_hidden,
        n_lstm_steps=maxlen,
        batch_size=batch_size,
        ctx_shape=ctx_shape)

    context, generated_words, logit_list, alpha_list = caption_generator.build_generator(maxlen=maxlen)
    saver = tf.train.Saver()
    saver.restore(sess, model_path)

    generated_word_index = sess.run(generated_words, feed_dict={context: feat})
    alpha_list_val = sess.run(alpha_list, feed_dict={context: feat})
    generated_words = [ixtoword[x[0]] for x in generated_word_index]
    punctuation = np.argmax(np.array(generated_words) == '.') + 1

    generated_words = generated_words[:punctuation]
    alpha_list_val = alpha_list_val[:punctuation]
    return generated_words, alpha_list_val

#train()
a,b = test()
f = open('test.txt','w')
sep = '\t'
f.write(sep.join(a))
np.save('attention.npy',b)
コード例 #40
0
# Need to modify the word list as well
wordList.append('<pad>')
wordList.append('<EOS>')
vocabSize = vocabSize + 2

if (os.path.isfile('Seq2SeqXTrain.npy')
        and os.path.isfile('Seq2SeqYTrain.npy')):
    xTrain = np.load('Seq2SeqXTrain.npy')
    yTrain = np.load('Seq2SeqYTrain.npy')
    print 'Finished loading training matrices'
    numTrainingExamples = xTrain.shape[0]
else:
    numTrainingExamples, xTrain, yTrain = createTrainingMatrices(
        'conversationDictionary.npy', wordList, maxEncoderLength)
    np.save('Seq2SeqXTrain.npy', xTrain)
    np.save('Seq2SeqYTrain.npy', yTrain)
    print 'Finished creating training matrices'

tf.reset_default_graph()

# Create the placeholders
encoderInputs = [
    tf.placeholder(tf.int32, shape=(None, )) for i in range(maxEncoderLength)
]
decoderLabels = [
    tf.placeholder(tf.int32, shape=(None, )) for i in range(maxDecoderLength)
]
decoderInputs = [
    tf.placeholder(tf.int32, shape=(None, )) for i in range(maxDecoderLength)
]
コード例 #41
0
all_data = open("data/all_data.txt", "r", encoding="utf-8").read()
bucket_structure, data_count = prepare_parameters(hParams, all_data)

# Load Embedding model.
if hParams.embedding_use_pretrained:
	model_embedding, embedding_matrix, keyed_vector = embedding_load("model/EmbeddingModel", hParams, all_data)
else:
	model_embedding, embedding_matrix, keyed_vector = embedding_train("model/EmbeddingModel", hParams, all_data)

VOCAB = list(keyed_vector.vocab.keys())
vocab_length = len(VOCAB)

# Create data matrices.
encX, decX, decy = create_matrix(all_data, VOCAB, bucket_structure, hParams, model_embedding)

# Save to disk.
vocabF = open("data/vocab.txt", "w", encoding="utf-8")
for word in VOCAB:
	vocabF.write(word + "\n")
vocabF.close()

np.save("data/encX", encX)
np.save("data/decX", decX)
np.save("data/decy", decy)

print("Example count:", len(apply_filter(all_data.split("\n"))))
print("Vocabulary length:", vocab_length)
print("Matrix sizes:")
print("encX", encX.shape)
print("decX", decX.shape)
print("decy", decy.shape)
コード例 #42
0
    c_p = 10000

    for i in range(iteration):
        _,cost_val = sess.run([optimizer,cost],feed_dict = {x:s,f:f_ref})
        # print('fs={}'.format(fs))
        if abs(cost_val-c_p) <=0.0000000000001:
            break
        c_p = cost_val
        print('iteration {}: {}'.format(i,cost_val))
    w_v,c_v,h_v = sess.run([w,c,h])


# In[ ]:


np.save('Weights/Joint{}/w.npy'.format(joint_select),w_v)
np.save('Weights/Joint{}/c.npy'.format(joint_select),c_v)
np.save('Weights/Joint{}/h.npy'.format(joint_select),h_v)


# In[ ]:





# In[ ]:



コード例 #43
0
            _loss = keras.losses.BinaryCrossentropy()
            return _loss(y_true, tf.nn.sigmoid(y_pred))
        elif loss_func == 'xent':
            _loss = keras.losses.SparseCategoricalCrossentropy()
            return _loss(y_true, tf.nn.softmax(y_pred))
        elif loss_func == 'balance':
            y_true[y_true == 0] = -1
            return -1 * np.sum(y_true * (y_pred - .5))

    return loss


print(model_dir)
model = keras.models.load_model(model_dir + '.h5',
                                custom_objects={
                                    'custom_loss': custom_loss(),
                                    'loss': custom_loss()
                                },
                                compile=False)

output = model.predict(x_test, batch_size=eval_batch_size)
nat_labels = np.zeros(output.shape).astype(np.float32)
nat_labels[output >= 0.5] = 1.
nat_dists = np.sum(np.absolute(nat_labels - y_test), axis=-1)
nat_acc = np.mean(nat_dists == 0)

print('natural: {:.2f}%'.format(100 * nat_acc))
np.save(
    'preds/pred_{}_{}'.format(model_dir.split('/')[1],
                              dataset.split('/')[-1]), output)
コード例 #44
0
def create_dataset_for_one_song(
    song_name,
    wav_path,
    y,
    sr,
    k,
    idx,
    crop_size=6000,
    beat_dir_path="/n/work1/ooyama/dataset/rwc4-4/annotation/fixed_popular_beat/",
):
    for shift in tqdm(range(-12, 13), desc=song_name):
        shifted_y = y if shift == 0 else pitch_shift(y, sr, n_steps=shift)
        save_name1 = song_name if shift == 0 else f"{song_name}shift{shift}"
        for stretch_i in range(6):  # stretch 5 times randomly
            if stretch_i == 0:
                save_name2 = f"{save_name1}original" if shift == 0 else save_name1
                beat_path = f"{beat_dir_path}{song_name}.BEAT.TXT"
                melspec = convertAudio2MelSpec(wav_path)  # 100Hz
                activation, downbeats = convertBeatText2Activation(
                    beat_path, song_length=len(melspec), units="ms"
                )
                bpm = convertBeatText2Bpm(beat_path, len(melspec))
                max_bpm = np.max(bpm)
            else:
                max_rate = 300 / (max_bpm + 10)
                stretch_rate = random.choice(np.arange(0.5, max_rate, 0.05))
                rounded_rate = np.round(stretch_rate, decimals=2)
                save_name2 = f"{save_name1}stretch{stretch_i}x{rounded_rate}"
                stretched_y = time_stretch(shifted_y, stretch_rate)
                melspec = convertAudio2MelSpec(None, True, stretched_y, sr)  # 100Hz
                stretched_activation = stretch_beat(activation, stretch_rate)
                stretched_bpm = stretch_bpm(bpm, stretch_rate)
                stretched_downbeats = (
                    (np.rint(downbeats / stretch_rate)).astype(np.int64)
                )
            beattheta = activation2beattheta(
                activation if stretch_i == 0 else stretched_activation
            )
            bartheta = activation2bartheta(
                activation if stretch_i == 0 else stretched_activation,
                downbeats if stretch_i == 0 else stretched_downbeats,
            )
            assert np.max(bartheta) > 0.5
            assert np.max(beattheta) > 0.5
            features = [
                ["melspec", melspec],
                ["activation", activation if stretch_i == 0 else stretched_activation],
                ["bpm", bpm if stretch_i == 0 else stretched_bpm],
                ["beattheta", beattheta],
                ["bartheta", bartheta],
            ]
            for feature in features:
                if feature[0] == "bpm":
                    feature[1][feature[1] > 300] = 300
                feature[1] = np.ascontiguousarray(feature[1])
                feature_length = len(feature[1])
                cropped_features = (
                    [feature[1]]
                    if feature_length < crop_size
                    else librosa.util.frame(
                        x=feature[1],
                        frame_length=crop_size,
                        hop_length=crop_size,
                        axis=0,
                    )
                )
                for index, cropped_feature in enumerate(cropped_features):
                    cropped_feature[cropped_feature < 0] = 0
                    fname = feature[0]
                    if fname == "activation":
                        fname = "beat"
                    elif fname == "melspec":
                        fname = f"melspec_sr{sr}_nfft1024"
                    path = gen_path(k, "all", fname)
                    np.save(
                        f"{path}{save_name2}-{str(index)}.{feature[0]}",
                        cropped_feature,
                    )
                    if idx > 25 and (idx - 1) % 25 < 5:
                        path = gen_path(k, "valid", fname)
                        np.save(
                            f"{path}{save_name2}-{str(index)}.{feature[0]}",
                            cropped_feature,
                        )
                    else:
                        path = gen_path(k, "train", fname)
                        np.save(
                            f"{path}{save_name2}-{str(index)}.{feature[0]}",
                            cropped_feature,
                        )
コード例 #45
0
def generate_masks(model, train_iter):
    total_epoch_loss = 0
    total_epoch_acc = 0
    model.eval()
    to_write_all=[]
    max_length=0
    print("length of training set if",len(val_iter))
    with torch.no_grad():
        for idx, batch in enumerate(tqdm(val_iter)):
            text = batch.text[0]
            q=text[0]       
            text_numpy=text.numpy()
            
            n=batch.text[1][0].item()
            mq=[]
            q_len=text[0].size()[0]
            mq.append(q.clone())
            mask = torch.ones(n)
            mask = mask.long()
            for i in range(n):
                mask[i] = 0
                for j in range(i+1,n):
                    mask[j] = 0
                    m = torch.mul(mask,q[0:n])
                    temp = []
                    temp.append(m.clone())
                    temp2 = torch.zeros(q_len-n)
                    temp.append(temp2.long())
                    temp = torch.cat(temp)
                    mq.append(temp.clone())
                    mask[j] = 1
                mask[i] = 1

            mq=torch.stack(mq)
            target = batch.label
            target = torch.autograd.Variable(target).long()
            if torch.cuda.is_available():
                text = text.cuda()
                target = target.cuda()
            
            attMax = torch.ones(1,mq.size()[1])
            max_prediction=-100000
            for txt in mq:      
                txt=txt.view(1,-1)
                prediction ,q_att= model(txt)
                if(prediction[0][target[0].item()].item()>max_prediction):
                    max_prediction=prediction[0][target[0].item()].item()
                    attMax=q_att


            attMax_numpy=attMax.numpy()
            to_write=[]
            to_write.append(text_numpy)
            to_write.append(attMax_numpy)
            to_write = np.asarray(to_write)
            loss = loss_fn(prediction, target)
            num_corrects = (torch.max(prediction, 1)[1].view(target.size()).data == target.data).sum()
            acc = 100.0 * num_corrects/len(batch)
            total_epoch_loss += loss.item()
            total_epoch_acc += acc.item()
            to_write_all.append(to_write)

    to_write_all=np.array(to_write_all)
    np.save("question_attention_masks",to_write_all)
    return total_epoch_loss/len(val_iter), total_epoch_acc/len(val_iter)
コード例 #46
0
def run_analysis(X,
                 Y,
                 T_pi_vals,
                 dim_vals,
                 offset_vals,
                 res_name,
                 num_cv_folds,
                 decoding_window,
                 args,
                 n_init=1,
                 verbose=False,
                 index=1):

    # X: 1363 * 30
    # Y: 1363 * 30
    use_gpu = True
    device = torch.device("cuda:{}".format(args.gpuid))

    results_size = (num_cv_folds, len(dim_vals), len(offset_vals),
                    len(T_pi_vals) + 2)
    results = np.zeros(results_size)  # 5*4*4*12
    min_std = 1e-6
    good_cols = (X.std(axis=0) > min_std)
    X = X[:, good_cols]
    # loop over CV folds
    cv = CrossValidate(X, Y, num_cv_folds, stack=False)
    for X_train, X_test, Y_train, Y_test, fold_idx in cv:
        if fold_idx: break
        if verbose:
            print("fold", fold_idx + 1, "of", num_cv_folds)

        # mean-center X and Y
        X_mean = np.concatenate(X_train).mean(axis=0, keepdims=True)
        X_train_ctd = [Xi - X_mean for Xi in X_train]
        X_test_ctd = X_test - X_mean
        Y_mean = np.concatenate(Y_train).mean(axis=0, keepdims=True)
        Y_train_ctd = [Yi - Y_mean for Yi in Y_train]
        Y_test_ctd = Y_test - Y_mean

        # chunk
        chunk_size = 500
        X_train_seqs, L_train = chunk_long_seq(X_train_ctd[0], 30, chunk_size)
        X_test_seqs, L_test = [X_test_ctd], [X_test_ctd.shape[0]]

        # compute cross-cov mats for DCA
        T_max = 2 * np.max(T_pi_vals)

        # loop over dimensionalities
        for dim_idx in range(len(dim_vals)):
            dim = dim_vals[dim_idx]
            if verbose:
                print("dim", dim_idx + 1, "of", len(dim_vals))

            # loop over T_pi vals
            for T_pi_idx in range(len(T_pi_vals)):

                T_pi = T_pi_vals[T_pi_idx]

                idim = X_test_ctd.shape[-1]
                fdim = dim
                T = T_pi
                params = 'obj={}_encoder={}_fdim={}_context={}_T={}_lr={}_bs={}_dropout={}_rate-lambda={}_ortho-lambda={}_recon-lambda={}_seed={}'.format(
                    args.obj, args.encoder_type, args.fdim, args.input_context,
                    args.T, args.lr, args.batchsize, args.dropout,
                    args.rate_lambda, args.ortho_lambda, args.recon_lambda,
                    args.seed)

                dapc_model = DAPC(args.obj,
                                  idim,
                                  fdim,
                                  T,
                                  encoder_type=args.encoder_type,
                                  ortho_lambda=args.ortho_lambda,
                                  recon_lambda=args.recon_lambda,
                                  dropout=args.dropout,
                                  masked_recon=args.masked_recon,
                                  args=args,
                                  device=device)

                dapc_model = fit_dapc(dapc_model,
                                      X_train_seqs,
                                      L_train,
                                      X_test_seqs,
                                      L_test,
                                      None,
                                      args.lr,
                                      use_gpu,
                                      batch_size=args.batchsize,
                                      max_epochs=args.epochs,
                                      device=device,
                                      snapshot=params + ".cpt",
                                      use_writer=False,
                                      pred_data=[
                                          X_train_ctd[0], Y_train_ctd[0],
                                          X_test_ctd, Y_test_ctd,
                                          decoding_window, offset_vals[-1],
                                          args
                                      ])

                # compute DCA R2 over offsets
                X_train_dapc = dapc_model.encode(
                    torch.from_numpy(X_train_ctd[0]).to(
                        device, dtype=dapc_model.dtype)).cpu().numpy()
                X_test_dapc = dapc_model.encode(
                    torch.from_numpy(X_test_ctd).to(
                        device, dtype=dapc_model.dtype)).cpu().numpy()
                for offset_idx in range(len(offset_vals)):
                    offset = offset_vals[offset_idx]
                    r2_dapc = linear_decode_r2(X_train_dapc,
                                               Y_train_ctd,
                                               X_test_dapc,
                                               Y_test_ctd,
                                               decoding_window=decoding_window,
                                               offset=offset)
                    results[fold_idx, dim_idx, offset_idx, T_pi_idx] = r2_dapc
                    np.save(res_name, results)

    return results
コード例 #47
0
        ad_list = []
        for name, group in tqdm(grouped):
            t = group[feat].to_list()
            if len(t) > 1:
                ad_list.append([str(i) for i in t])
        model = Word2Vec(
            ad_list, sg=1, size=128, window=8, seed=2020, min_count=1,
            workers=int(multiprocessing.cpu_count()) / 2, iter=10)
        embeddings = []
        for i in range(ad_df[feat].max() + 1):
            try:
                embeddings.append(model[str(i)])
            except:
                embeddings.append(np.zeros(128))
        embeddings = np.stack(embeddings)
        np.save("data/w2v_%s.npy" % feat, embeddings)

    # user_id
    grouped = click_log_df.groupby("creative_id")
    ad_list = []
    for name, group in tqdm(grouped):
        t = group["user_id"].to_list()
        if len(t) > 1:
            ad_list.append([str(i) for i in t])
    model = Word2Vec(
        ad_list, sg=1, size=128, window=8, seed=2020, min_count=1,
        workers=int(multiprocessing.cpu_count()) / 2, iter=10)
    embeddings = []
    for i in range(click_log_df["user_id"].max() + 1):
        try:
            embeddings.append(model[str(i)])
コード例 #48
0
    context = zmq.Context()
    req = context.socket(zmq.REQ)   #open a req port to talk to pupil
    req.connect("tcp://%s:%s" %(addr,req_port))
    req.send(b'SUB_PORT')   # ask for the sub port
    sub_port = req.recv()

    # open a sub port to listen to pupil in eye_1_3d
    sub_1_3d = context.socket(zmq.SUB)
    sub_1_3d.connect(b"tcp://%s:%s" %(addr.encode('utf-8'),sub_port))
    sub_1_3d.setsockopt(zmq.SUBSCRIBE, b'pupil.1.3d')

    need_calculate = input("Start Calculating convert matrix?(Y/n) : ")

    if (need_calculate.upper() == "Y") :
        convert_matrix = make_convert_matrix(sub_1_3d)
        np.save('./convert_matrix',convert_matrix)

    else:
        convert_matrix = np.load('convert_matrix.npy')
    print("convert matrix : ", convert_matrix)

    input("Start convert blurred image(press enter)")
    time_0 = time.time()

    # Start convert blurred image
    try:
        while True:
            current_time = time.time()

            # Collect Data from pupil_tracker &  Wait for a coherent pair of frames: depth and color
            sub_1_3d.connect(b"tcp://%s:%s" %(addr.encode('utf-8'),sub_port))
コード例 #49
0
ファイル: HW2ex1.py プロジェクト: lnsongxf/TextMiningBGSE
#check words of first document
# are the words of the doc present at the unique variable
check=[docsobj.unique[i] for i in range(0,len(docsobj.unique)) if docsobj.unique[i] in docsobj.tokens[0]]
# extract the counts from the actual doc tokens
check2=[docsobj.doc_term[0][i] for i in range(0,len(docsobj.unique)) if docsobj.doc_term[0][i]>0]
# extract the counted words from .unique
counts_doc_term=[docsobj.unique[i] for i in range(0,len(docsobj.unique)) if docsobj.doc_term[0][i]>0]
docsobj.docs[0]

=======
docsobj = RawDocs(data.speech[0:100],'stopwords.txt')
dic=['Mister','president']
docsobj.doc_term()
docsobj.count(dic)
>>>>>>> Stashed changes
np.save('sample_doc_term.npy',docsobj.doc_term)

# Exercise 2
data = pd.read_table("../HW1/output_hw1ex4_fdez_verdu_kreienkamp.CSV",encoding="utf-8")

#convert text to lowercase
for i in range(0, len(data.Text)):
    data.Text[i] = data.Text[i].lower()


#with one dictionary 
docsobj = RawDocs(data.Text[0:100],'stopwords.txt')
docsobj.token_clean(2)
docsobj.stopword_remove()
# If running first time
docsobj.doc_term()
コード例 #50
0
def main():
    path = "/home/yren2/cs66/labs/FinalProject-yren2-hhuang2/code/Dota2"
    allFiles = glob.glob(os.path.join(path,"*.csv"))

    np_array_list = []
    for file_ in allFiles:
        df = pd.read_csv(file_,index_col=None, header=0)
        np_array_list.append(df.as_matrix())

    comb_np_array = np.vstack(np_array_list)
    big_frame = pd.DataFrame(comb_np_array)
    big_frame.columns = ["Match ID", "League", "Start Date", "Duration (s)", "Duration (mm:ss)", \
        "Radiant Team", "Team A", "Team A Heroes",	"Team B", "Team B Heroes", "Winner"]

    rawData = big_frame.drop_duplicates(["Match ID"], keep='last')

    # get data from only 2014
    rawData = rawData[rawData["Start Date"].str.contains("2013")]


    with open('heroes.json') as json_data:
        d = json.load(json_data)

    heroes = {}
    for i in range(len(d["heroes"])):
        name = d["heroes"][i]["localized_name"]
        hid = d["heroes"][i]["id"]
        heroes[name] = hid-1

    sorted_heores = OrderedDict(sorted(heroes.items(), key=lambda t: t[1]))

    data = np.zeros((1,115), dtype=float) #initialize so it's easier to stack
    label = np.zeros((1,1), dtype=float) #initialize so it's easier to stack

    for index, row in rawData.iterrows():
        if row["Radiant Team"] == "Team A":
            team1 = 1
            team2 = -1
            if row["Winner"] == "Team A":
                label = np.vstack((label, [1]))
            else:
                label = np.vstack((label, [-1]))
        else:
            team1 = -1
            team2 = 1
            if row["Winner"] == "Team A":
                label = np.vstack((label, [-1]))
            else:
                label = np.vstack((label, [1]))

        hero1 = row["Team A Heroes"]
        hero2 = row["Team B Heroes"]

        data = np.vstack((data, createRow(heroes, hero1, team1, hero2, team2)))

    data = np.delete(data, (0), axis=0) #delete first row of 0
    label = np.delete(label, (0), axis=0) #delete first row of 0
    # sdata = sparse.csr_matrix(data.astype(float))
    # sparse.save_npz("data.npz", sdata)

    X_train, X_test, y_train, y_test = train_test_split(data, label, test_size = 0.2, random_state = 42)
    sdata = sparse.csr_matrix(X_train.astype(float))
    sparse.save_npz("trainData2014.npz", sdata)
    sdata = sparse.csr_matrix(X_test.astype(float))
    sparse.save_npz("testData2014.npz", sdata)

    np.save("trainLabel2014.npy", y_train)
    np.save("testLabel2014.npy", y_test)
コード例 #51
0
def main(args):
    if args.gpu and torch.cuda.is_available():
        torch.cuda.manual_seed_all(args.seed)

    np.random.seed(args.seed)
    torch.manual_seed(args.seed)

    if args.n_workers == -1:
        args.n_workers = args.gpu * 4 * torch.cuda.device_count()

    device = torch.device("cuda" if args.gpu else "cpu")

    # Load trained ANN from disk.
    if args.arch == 'vgg15ab':
        ann = vgg_15_avg_before_relu(dataset=args.dataset)
    # add other architectures here#
    else:
        raise ValueError('Unknown architecture')

    ann.features = torch.nn.DataParallel(ann.features)
    ann.cuda()
    if not os.path.isdir(args.job_dir):
        os.mkdir(args.job_dir)
    f = os.path.join('.', args.model)
    try:
        dictionary = torch.load(f=f)['state_dict']
    except KeyError:
        dictionary = torch.load(f=f)
    ann.load_state_dict(state_dict=dictionary, strict=True)

    if args.dataset == 'imagenet':
        input_shape = (3, 224, 224)

        normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                         std=[0.229, 0.224, 0.225])
        # the actual data to be evaluated
        val_loader = ImageNet(image_encoder=RepeatEncoder(time=args.time,
                                                          dt=1.0),
                              label_encoder=None,
                              root=args.data,
                              download=False,
                              transform=transforms.Compose([
                                  transforms.Resize((256, 256)),
                                  transforms.CenterCrop(224),
                                  transforms.ToTensor(),
                                  normalize,
                              ]),
                              split='val')
        # a wrapper class
        dataloader = DataLoader(
            val_loader,
            batch_size=args.batch_size,
            shuffle=True,
            num_workers=4,
            pin_memory=args.gpu,
        )
        # A loader of samples for normalization of the SNN from the training set
        norm_loader = ImageNet(
            image_encoder=RepeatEncoder(time=args.time, dt=1.0),
            label_encoder=None,
            root=args.data,
            download=False,
            split='train',
            transform=transforms.Compose([
                transforms.Resize(256),
                transforms.CenterCrop(224),
                transforms.ToTensor(),
                normalize,
            ]),
        )

    elif args.dataset == 'cifar100':
        input_shape = (3, 32, 32)
        print('==> Using Pytorch CIFAR-100 Dataset')
        normalize = transforms.Normalize(mean=[0.507, 0.487, 0.441],
                                         std=[0.267, 0.256, 0.276])
        val_loader = CIFAR100(image_encoder=RepeatEncoder(time=args.time,
                                                          dt=1.0),
                              label_encoder=None,
                              root=args.data,
                              download=True,
                              train=False,
                              transform=transforms.Compose([
                                  transforms.RandomCrop(32, padding=4),
                                  transforms.RandomHorizontalFlip(0.5),
                                  transforms.ToTensor(),
                                  normalize,
                              ]))

        dataloader = DataLoader(
            val_loader,
            batch_size=args.batch_size,
            shuffle=True,
            num_workers=0,
            pin_memory=args.gpu,
        )

        norm_loader = CIFAR100(image_encoder=RepeatEncoder(time=args.time,
                                                           dt=1.0),
                               label_encoder=None,
                               root=args.data,
                               download=True,
                               train=True,
                               transform=transforms.Compose([
                                   transforms.RandomCrop(32, padding=4),
                                   transforms.RandomHorizontalFlip(0.5),
                                   transforms.ToTensor(),
                                   normalize,
                               ]))
    else:
        raise ValueError('Unsupported dataset.')

    if args.eval_size == -1:
        args.eval_size = len(val_loader)

    for step, batch in enumerate(
            torch.utils.data.DataLoader(norm_loader, batch_size=args.norm)):
        data = batch['image']
        break

    snn = ann_to_snn(ann,
                     input_shape=input_shape,
                     data=data,
                     percentile=args.percentile)

    torch.cuda.empty_cache()
    snn = snn.to(device)

    correct = 0
    t0 = time()
    accuracies = np.zeros((args.time, (args.eval_size // args.batch_size) + 1),
                          dtype=np.float32)
    for step, batch in enumerate(tqdm(dataloader)):
        if (step + 1) * args.batch_size > args.eval_size:
            break
        # Prep next input batch.
        inputs = batch["encoded_image"]
        labels = batch["label"]
        inpts = {"Input": inputs}
        if args.gpu:
            inpts = {k: v.cuda() for k, v in inpts.items()}

        snn.run(inpts=inpts,
                time=args.time,
                step=step,
                acc=accuracies,
                labels=labels,
                one_step=args.one_step)
        last_layer = list(snn.layers.keys())[-1]
        output_voltages = snn.layers[last_layer].summed
        prediction = torch.softmax(output_voltages, dim=1).argmax(dim=1)
        correct += (prediction.cpu() == labels).sum().item()
        snn.reset_()
    t1 = time() - t0

    final = accuracies.sum(axis=1) / args.eval_size

    plt.plot(final)
    plt.suptitle('{} {} ANN-SNN@{} percentile'.format(args.dataset, args.arch,
                                                      args.percentile),
                 fontsize=20)
    plt.xlabel('Timestep', fontsize=19)
    plt.ylabel('Accuracy', fontsize=19)
    plt.grid()
    plt.show()
    plt.savefig('{}/{}_{}.png'.format(args.job_dir, args.arch,
                                      args.percentile))
    np.save(
        '{}/voltage_accuracy_{}_{}.npy'.format(args.job_dir, args.arch,
                                               args.percentile), final)

    accuracy = 100 * correct / args.eval_size

    print(f"SNN accuracy: {accuracy:.2f}")
    print(f"Clock time used: {t1:.4f} ms.")
    path = os.path.join(args.job_dir, "results", args.results_file)
    os.makedirs(os.path.dirname(path), exist_ok=True)
    if not os.path.isfile(path):
        with open(path, "w") as f:
            f.write(
                "seed,simulation time,batch size,inference time,accuracy\n")
    to_write = [args.seed, args.time, args.batch_size, t1, accuracy]
    to_write = ",".join(map(str, to_write)) + "\n"
    with open(path, "a") as f:
        f.write(to_write)

    return t1
コード例 #52
0
# Load the roi-stats for leaf rois (from the cluster job)
print("Loading leaf roi-stats")
df = pd.DataFrame(np.load('roi-stats.npy', allow_pickle=True))

# Pivot so roi names are in the columns
print("Pivoting stats")
pdf = df[['body', 'roi', 'voxels_s1']].pivot('body', 'roi')
pdf = pdf.fillna(0).astype(np.int64)
pdf.columns = pdf.columns.droplevel()

# Compute non-leaf roi stats by summing the proper leaf rois for each
print("Computing non-leaf stats")
for roi in tqdm(leaf_descendants.keys()):
    if roi not in pdf.columns:
        ld = leaf_descendants[roi]
        pdf[roi] = pdf.loc[:, ld].sum(axis=1)

# Sort columns by name
pdf = pdf.sort_index(axis=1)

# Save as npy and csv
p = 'roi-stats-pivoted-plus-nonleaf-rois.npy'
print(f"Saving {p}")
np.save(p, pdf.to_records(index=True))

p = 'roi-stats-pivoted-plus-nonleaf-rois.csv'
print(f"Saving {p}")
pdf.to_csv('roi-stats-pivoted-plus-nonleaf-rois.csv', index=True, header=True)

print("DONE")
コード例 #53
0
features = model.predict(x)
print(features.shape)
features = [features]
for i in range(1, 98):
    img_path = 'old_%d.jpg' % i
    print(img_path)
    img = image.load_img(img_path, target_size=(224, 224))
    x = image.img_to_array(img)
    x = np.expand_dims(x, axis=0)
    x = preprocess_input(x)
    temp = model.predict(x)
    features.append(temp)

features = np.array(features)
print(features.shape) #(99, 1, 7, 7, 512)
np.save('old_features.npy', features)


img_path = 'ad_0.jpg'
img = image.load_img(img_path, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
features = model.predict(x)
print(features.shape)
features = [features]
for i in range(1, 100):
    img_path = 'ad_%d.jpg' % i
    print(img_path)
    img = image.load_img(img_path, target_size=(224, 224))
    x = image.img_to_array(img)
コード例 #54
0
                        if energy_diff > 0:
                            flip_prob = flip_probs[i]
                            flip = np.random.choice([1, flip_state], size=1, p=[1-flip_prob, flip_prob])[0]
                            flips[a] = flip
                        else:
                            flip_prob = flip_others_probs[i]
                            flip = np.random.choice([1, flip_state], size=1, p=[1-flip_prob, flip_prob])[0]
                            flips[a] = flip
                    flipped_s = s * flips
                    new_mus.append(mu + flipped_s*sigma*zoom_factor)
            sigma *= zoom_factor
            mus = new_mus
            ground_energies[f,i]=total_hamiltonian(mus[0],C_i,C_ij)/(train_size-1)
            ground_energies_test[f,i] = total_hamiltonian(mus[0],C_i_test,C_ij_test)/(test_size-1)

            np.save('./mus/' +'mus' + str(train_size) + "_fold" + str(f) + '_iter' + str(i) + '.npy', np.array(mus))


    final_predictions_train=[]
    final_predictions_test=[]
    strong_classifier_train = strong_classifier(predictions_train , mus[0])
    strong_classifier_test = strong_classifier(predictions_test , mus[0])
    for i in range(len(tag_train)) :
        final_predictions_train.append([strong_classifier_train[i] , tag_train[i]])

    for i in range(len(tag_test)) :
        final_predictions_test.append([strong_classifier_test[i], tag_test[i]])

    np.save("./strong_train_predictions/prediction_lables_train_f"+str(f)+".npy", final_predictions_train)
    np.save("./strong_train_predictions/prediction_lables_test_f"+str(f)+".npy", final_predictions_test)
コード例 #55
0
def main():
    ###make sure to change these when running in a new enviorment!###
    #location of data directory
    filepath_cat1 = cu.get_output_path() + 'processed_data/Berland_groupcat/'
    filepath_cat2 = cu.get_output_path() + 'processed_data/NYU_VAGC/'
    #save data to directory...
    savepath1 = filepath_cat1 + 'nyu_vagc_match/'
    savepath2 = filepath_cat2 + 'berland_groupcat_match/'
    #################################################################

    catalogues_1 = ['mr19_groups', 'smthresh10.2.groups', 'smthresh9.8.groups']
    catalogues_2 = ['nyu_vagc_dr7']

    catalogue1 = catalogues_1[0]
    catalogue2 = catalogues_2[0]
    print catalogue1, 'match into', catalogue2

    f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5',
                   'r')  #open catalogue file
    GC = f1.get(catalogue1)

    f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5',
                   'r')  #open catalogue file
    W = f2.get(catalogue2)

    da = 2.0 * 1.0 / 3600.0  #matching length
    result = cu.spherematch(GC['RA'],
                            GC['DEC'],
                            W['RA'],
                            W['DEC'],
                            tol=da,
                            nnearest=1)

    #check to see if anything was matched to the same object
    repeats = [
        item for item, count in Counter(result[1]).iteritems() if count > 1
    ]
    if len(repeats) > 0:
        print 'number of double matched objects:', len(repeats)

    #check to see if every object has a match
    if len(result[0]) == len(GC):
        print 'a match was found for every object.'
    else:
        print 'some objects do not have matches.'

    filename1 = catalogue2 + '_' + catalogue1 + '_match'
    filename2 = catalogue1 + '_' + catalogue2 + '_match'

    np.save(savepath1 + filename1, result[1])
    np.save(savepath2 + filename2, result[0])

    catalogue1 = catalogues_1[1]
    catalogue2 = catalogues_2[0]
    print catalogue1, 'match into', catalogue2

    f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5',
                   'r')  #open catalogue file
    GC = f1.get(catalogue1)

    f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5',
                   'r')  #open catalogue file
    W = f2.get(catalogue2)

    da = 2.0 * 1.0 / 3600.0  #matching length
    result = cu.spherematch(GC['ra'],
                            GC['dec'],
                            W['RA'],
                            W['DEC'],
                            tol=da,
                            nnearest=1)

    #check to see if anything was matched to the same object
    repeats = [
        item for item, count in Counter(result[1]).iteritems() if count > 1
    ]
    if len(repeats) > 0:
        print 'number of double matched objects:', len(repeats)

    #check to see if every object has a match
    if len(result[0]) == len(GC):
        print 'a match was found for every object.'
    else:
        print 'some objects do not have matches.'

    filename1 = catalogue2 + '_' + catalogue1 + '_match'
    filename2 = catalogue1 + '_' + catalogue2 + '_match'

    np.save(savepath1 + filename1, result[1])
    np.save(savepath2 + filename2, result[0])

    catalogue1 = catalogues_1[2]
    catalogue2 = catalogues_2[0]
    print catalogue1, 'match into', catalogue2

    f1 = h5py.File(filepath_cat1 + catalogue1 + '.hdf5',
                   'r')  #open catalogue file
    GC = f1.get(catalogue1)

    f2 = h5py.File(filepath_cat2 + catalogue2 + '.hdf5',
                   'r')  #open catalogue file
    W = f2.get(catalogue2)

    da = 2.0 * 1.0 / 3600.0  #matching length
    result = cu.spherematch(GC['ra'],
                            GC['dec'],
                            W['RA'],
                            W['DEC'],
                            tol=da,
                            nnearest=1)

    #check to see if anything was matched to the same object
    repeats = [
        item for item, count in Counter(result[1]).iteritems() if count > 1
    ]
    if len(repeats) > 0:
        print 'number of double matched objects:', len(repeats)

    #check to see if every object has a match
    if len(result[0]) == len(GC):
        print 'a match was found for every object.'
    else:
        print 'some objects do not have matches.'

    filename1 = catalogue2 + '_' + catalogue1 + '_match'
    filename2 = catalogue1 + '_' + catalogue2 + '_match'

    np.save(savepath1 + filename1, result[1])
    np.save(savepath2 + filename2, result[0])
コード例 #56
0
def main():

    parser = argparse.ArgumentParser()
    ############## environment parameters ##############
    parser.add_argument('--edge_capability', default = 3*1e2*GHZ, metavar='G', help = "total edge CPU capability", type=int)
    parser.add_argument('--cloud_capability', default = 2.1*1e3*GHZ, metavar='G', help = "total cloud CPU capability", type=int)  # clock per tick
    parser.add_argument('--task_rate', default = 10, metavar='G', help = "application arrival task rate")
    parser.add_argument('--channel', default = WIRED, metavar='G')
    parser.add_argument('--applications', default = (SPEECH_RECOGNITION, NLP, FACE_RECOGNITION), metavar='G')#, SEARCH_REQ, LANGUAGE_TRANSLATION, PROC_3D_GAME, VR, AR
    parser.add_argument('--use_beta', action = 'store_true', help = "use 'offload' to cloud")
    parser.add_argument('--silence', action = 'store_true', help= "shush environment messages")

    parser.add_argument('--comment', default=None)


    ############## Hyperparameters ##############
    parser.add_argument('--log_interval', default = 20 , metavar='N', help="print avg reward in the interval", type=int)
    parser.add_argument('--max_episodes', default = 1000, metavar='N', help="max training episodes", type=int)
    parser.add_argument('--max_timesteps', default = 2000, metavar='N', help="max timesteps in one episode", type=int)

    parser.add_argument('--update_timestep', default = 1000, metavar='N', help="update policy every n timesteps", type=int)
    parser.add_argument('--action_std', default = 0.5 , metavar='N', help="constant std for action distribution (Multivariate Normal)", type=float)
    parser.add_argument('--K_epochs', default = 80  , metavar='N', help="update policy for K epochs")
    parser.add_argument('--eps_clip', default = 0.2 , metavar='N', help="clip parameter for PPO", type=float)
    parser.add_argument('--gamma', default = 0.9   , metavar='N', help="discount factor", type=float)

    parser.add_argument('--lr', default = 0.0003 , metavar='N', help="parameters for Adam optimizer", type=float)
    parser.add_argument('--betas', default = (0.9, 0.999), metavar='N')
    parser.add_argument('--random_seed', default = None, metavar='N')
    #############################################


    ############## save parameters ##############
    file_name = 'ppo_fixed_len'+str(datetime.now())
    eval_dir = "./results/{}/eval_results".format(file_name)
    model_dir = "./results/{}/pytorch_models".format(file_name)

    if not os.path.exists(eval_dir):
        os.makedirs(eval_dir)
    if not os.path.exists(model_dir):
        os.makedirs(model_dir)

    args = parser.parse_args()
    args_dict = vars(args)

    ############## parser arguments to plain variabales ##############
    edge_capability = args.edge_capability
    cloud_capability = args.cloud_capability
    task_rate = args.task_rate
    channel = args.channel
    applications = args.applications
    use_beta = args.use_beta
    silence = args.silence

    number_of_apps = len(applications)
    cloud_policy = [1/number_of_apps]*number_of_apps
    args_dict['number_of_apps'] = number_of_apps
    args_dict['cloud_policy'] = cloud_policy

    log_interval = args.log_interval
    max_episodes = args.max_episodes
    max_timesteps = args.max_timesteps
    update_timestep = args.update_timestep
    action_std = args.action_std
    K_epochs = args.K_epochs
    eps_clip = args.eps_clip
    gamma = args.gamma
    lr = args.lr
    betas = args.betas
    random_seed = args.random_seed
    ##################################################################


    with open("./results/{}/args.json".format(file_name), 'w') as f:
        json.dump(args_dict, f, indent='\t')

    # import pdb; pdb.set_trace()
    # creating environment
    env = environment.Environment_sosam(task_rate, *applications, use_beta=use_beta)
    state = env.init_for_sosam(edge_capability, cloud_capability, channel)
    state_dim = env.state_dim
    action_dim = env.action_dim

    if random_seed:
        print("Random Seed: {}".format(random_seed))
        torch.manual_seed(random_seed)
        # env.seed(random_seed)
        np.random.seed(random_seed)

    memory = Memory()
    ppo = PPO(state_dim, action_dim, action_std, lr, betas, gamma, K_epochs, eps_clip)

    # logging variables
    running_reward = 0
    avg_length = 0
    time_step = 0
    evaluations_empty_reward = []
    evaluations = []
    # evaluations_empty_reward_1000 = []
    # evaluations_1000 = []

    # training loop
    for i_episode in range(1, max_episodes+1):
        state = env.reset()
        for t in range(max_timesteps):
            time_step +=1
            # Running policy_old:
            action = ppo.select_action(state, memory)
            if t%250==0:
                print("---------------------------------------")
                print("estimated arrival: {}".format(state[:8]))
                print("just arrived: {}".format(state[8:16]))
                print("queue length: {}".format(state[16:24]))
                print("queue explosion: {}".format(state[24:32]))
                if use_beta:
                    print("c_estimated arrival: {}".format(state[32:40]))
                    print("c_just arrived: {}".format(state[40:48]))
                    print("c_queue length: {}".format(state[48:56]))
                    print("c_queue explosion: {}".format(state[56:]))
                print("---------------------------------------")
                print("------action\t{}".format(action))
                print("---------------------------------------")
            state, cost, done = env.step_together(t, action, cloud_policy, silence=silence)
            if t%250==0:
                print("new_estimated arrival: {}".format(state[:8]))
                print("new_just arrived: {}".format(state[8:16]))
                print("new_queue length: {}".format(state[16:24]))
                print("new_queue explosion: {}".format(state[24:32]))
                if use_beta:
                    print("new_c_estimated arrival: {}".format(state[32:40]))
                    print("new_c_just arrived: {}".format(state[40:48]))
                    print("new_c_queue length: {}".format(state[48:56]))
                    print("new_c_queue explosion: {}".format(state[56:]))

            reward = -cost
            # Saving reward:
            memory.rewards.append(reward)

            # update if its time
            if time_step % update_timestep == 0:
                ppo.update(memory)
                memory.clear_memory()
                time_step = 0
            running_reward += reward
            if t%250==0:
                print("---------------------------------------")
                print("cost:{}, episode reward{}".format(cost, running_reward))
                print("---------------------------------------")
            if done:
                break
            # if t%200==0:
            #     print("episode {}, average length {}, running_reward{}".format(i_episode, avg_length, running_reward))

        avg_length += t
        evaluations_empty_reward.append(evaluate_policy(env, ppo, cloud_policy, memory, epsd_length=max_timesteps*2))
        evaluations.append(evaluate_policy(env, ppo, cloud_policy, memory, epsd_length=max_timesteps*2, empty_reward=False))
        # evaluations_empty_reward_1000.append(evaluate_policy(env, ppo, cloud_policy, memory, epsd_length=1000))
        # evaluations_1000.append(evaluate_policy(env, ppo, cloud_policy, memory, epsd_length=1000, empty_reward=False))
        np.save("{}/eval_empty_reward".format(eval_dir), evaluations_empty_reward)
        np.save("{}/eval".format(eval_dir), evaluations)
        # np.save("{}/eval_empty_reward_1000".format(eval_dir), evaluations_empty_reward_1000)
        # np.save("{}/eval_1000".format(eval_dir), evaluations_1000)
        # stop training if avg_reward > solved_reward
        # if running_reward > (log_interval*solved_reward):
        #     print("########## Solved! ##########")
        #     torch.save(ppo.policy.state_dict(), './PPO_continuous_solved_{}.pth'.format(env_name))
        #     break

        # save every 500 episodes
        if i_episode % 50 == 0:
            ppo.save('env3_{}_{}'.format(i_episode, t), directory=model_dir)

        # logging
        if i_episode % log_interval == 0:
            avg_length = int(avg_length/log_interval)
            running_reward = int((running_reward/log_interval))

            print('Episode {} \t Avg length: {} \t Avg reward: {}'.format(i_episode, avg_length, running_reward))
            running_reward = 0
            avg_length = 0
コード例 #57
0
ファイル: mesh_generator.py プロジェクト: jpra2/Upscaling
    def set_area(self):
        gids_volumes = self.mb.tag_get_data(self.tags['GIDV'],
                                            self.all_volumes,
                                            flat=True)
        map_volumes = dict(zip(self.all_volumes, gids_volumes))
        areas = np.zeros(len(self.all_faces))
        normais = np.zeros((len(self.all_faces), 3))
        Adjs = [self.mb.get_adjacencies(f, 3) for f in self.all_faces]
        vertss = [
            self.mtu.get_bridge_adjacencies(f, 2, 0) for f in self.all_faces
        ]
        lim = 1e-9
        bound_faces = self.mb.create_meshset()
        bfs = []
        for i, f in enumerate(self.all_faces):
            elems = Adjs[i]
            if len(elems) < 2:
                bfs.append(f)
            verts = vertss[i]
            coords = self.mb.get_coords(verts).reshape([len(verts), 3])
            mins = coords.min(axis=0)
            maxs = coords.max(axis=0)
            dx, dy, dz = np.absolute(maxs - mins)
            if dx < lim:
                dx = 1.0
            if dy < lim:
                dy = 1.0
            if dz < lim:
                dz = 1.0
            area = dx * dy * dz
            areas[i] = area
            if len(elems) > 1:

                id0 = map_volumes[elems[0]]
                id1 = map_volumes[elems[1]]
                cent0 = self.all_centroids[id0]
                cent1 = self.all_centroids[id1]
                direction = cent1 - cent0
                norma = np.linalg.norm(direction)
                uni = np.absolute(direction / norma)
                normais[i] = uni
            else:
                p0 = coords[0]
                p1 = coords[1]
                p2 = coords[2]
                direction = np.cross(p0 - p1, p0 - p2)
                norma = np.linalg.norm(direction)
                uni = np.absolute(direction / norma)
                normais[i] = uni

        self.mb.tag_set_data(self.tags['AREA'], self.all_faces, areas)
        self.mb.tag_set_data(self.tags['NORMAL'], self.all_faces, normais)
        bfs = rng.Range(bfs)
        self.mb.add_entities(bound_faces, bfs)
        self.mb.tag_set_data(self.tags['BOUND_FACES'], 0, bound_faces)
        self.bound_faces = bfs
        self.faces_in = rng.subtract(self.all_faces, bfs)
        self.Adjs_in = np.array(
            [np.array(self.mb.get_adjacencies(f, 3)) for f in self.faces_in])
        Adjs_d = self.Adjs_in[:, 1]
        Adjs_e = self.Adjs_in[:, 0]
        os.chdir(flying_dir)
        np.save('Adjs_d', Adjs_d)
        np.save('Adjs_e', Adjs_e)
コード例 #58
0
ファイル: test_GAN.py プロジェクト: sk2299/HCLMP
G.load_state_dict(torch.load('./WGAN_for_DOS/generator_MP2020.pt'))
G.eval()

def sample_generator(G, num_samples, feature):
    generated_data_all = 0
    num_sam = 100
    for i in range(num_sam):
        latent_samples = Variable(G.sample_latent(num_samples))
        latent_samples = latent_samples
        generated_data = G(torch.cat((feature, latent_samples), dim=1))
        generated_data_all += generated_data
    generated_data = generated_data_all/num_sam
    return generated_data

print('Testing...')
gen_data = sample_generator(G, batch_size, test_feat).detach()
print('Done!')
#gen_data = gen_data*std+mean
test_label = (test_label-mean)/std

MAE = torch.mean(torch.abs(gen_data-test_label))
print('MAE:', MAE.numpy())

# save testing results
np.save('./WGAN_for_DOS/test_label.npy', test_label)
np.save('./WGAN_for_DOS/test_pred.npy', gen_data)




コード例 #59
0
    data_size = [100, 500, 1000, 5000, 10000]
    if args.all:
        data_size.append(70000)

    results = []
    basename = os.path.basename(os.path.splitext(__file__)[0])
    log_filename = os.path.join(LOG_DIR, basename + ".json")
    for n in data_size:
        X_train = X[:n]
        y_train = y[:n]
        n = X_train.shape[0]
        for name, method in methods:
            print("Fitting {} on {} samples...".format(name, n))
            t0 = time()
            np.save(
                os.path.join(LOG_DIR, "mnist_{}_{}.npy".format("original", n)), X_train
            )
            np.save(
                os.path.join(LOG_DIR, "mnist_{}_{}.npy".format("original_labels", n)),
                y_train,
            )
            X_embedded, n_iter = method(X_train)
            duration = time() - t0
            precision_5 = nn_accuracy(X_train, X_embedded)
            print(
                "Fitting {} on {} samples took {:.3f}s in {:d} iterations, "
                "nn accuracy: {:0.3f}".format(name, n, duration, n_iter, precision_5)
            )
            results.append(dict(method=name, duration=duration, n_samples=n))
            with open(log_filename, "w", encoding="utf-8") as f:
                json.dump(results, f)
コード例 #60
0
                BoxCoords[yVAL,xVAL,(idBoxMask*4)+3] = h 
    
    # iterate through every list element
    for i in range(0, len(ListElements)):
        currentField = ListElements[i]
        cx, cy, cx2, cy2 = getBounds(currentField)
        x, y, w, h = getCoords(currentField)
        w = width / 2
        # fill segment boxes 
        for xVAL in range(cx, cx2):
            for yVAL in range(cy2, cy):
                SegmentImage[yVAL,xVAL] = segMapper(currentField.id)

        # Fill box masks and box coords for list items once per row
        if currentField.id == "ItemName":
            for xVal in range(0,width-1):
                for yVal in range(cy2, cy):
                    BoxMasks[yVal,xVal,(num_anchors-1)*2] = 1 
                    BoxMasks[yVal,xVal,((num_anchors-1)*2)+1] = 0 

                    BoxCoords[yVal,xVal,(num_anchors-1)*4] = w # center x is for list items also w 
                    BoxCoords[yVal,xVal,((num_anchors-1)*4)+1] = y 
                    BoxCoords[yVal,xVal,((num_anchors-1)*4)+2] = w 
                    BoxCoords[yVal,xVal,((num_anchors-1)*4)+3] = h 

    # save target data
    np.save(seg_path + str(id) +".npy", SegmentImage)
    np.save(box_path + str(id) +".npy", BoxMasks)
    np.save(coord_path + str(id) +".npy", BoxCoords)

print("--- %s seconds ---" % (time.time() - start_time))