def run(expt, display=False): if isinstance(expt, str): expt = get_experiment(expt) storage.ensure_directory(expt.figures_dir()) storage.ensure_directory(expt.outputs_dir()) mkl.set_num_threads(1) v = gnp.garray(expt.dataset.load().as_matrix()) v = 0.999 * v + 0.001 * 0.5 if expt.permute: idxs = np.random.permutation(v.shape[0]) v = v[idxs] if display: expt.diagnostics += ['objective'] expt.show_after = expt.save_after visuals = Visuals(expt, v) if expt.init_rbm == 'base_rates': init_rbm = None elif isinstance(expt.init_rbm, TrainedRBM): init_rbm = load_trained_rbm(expt.init_rbm.location).convert_to_garrays() else: raise RuntimeError('Unknown init_rbm') assert isinstance(expt.training, rbm_training.TrainingParams) rbm_training.train_rbm(v, expt.nhid, expt.training, after_step=visuals.after_step, init_rbm=init_rbm)
def handle_mkl(max_threads): """Set max threads if mkl is availavle""" try: import mkl mkl.set_num_threads(max_threads) except ImportError: pass
def configure(num_jobs=8, TEST=False, subtract=0, num_proc=None, num_thread_per_proc=None): ''' num_jobs is typically the # of genes we are parallelizing over ''' if num_proc is None: num_proc = multiprocessing.cpu_count() - subtract if num_jobs > num_proc: num_jobs = num_proc if num_thread_per_proc is None: num_thread_per_proc = int(np.floor(num_proc/num_jobs)) if TEST: num_jobs = 1 num_thread_per_proc = 1 try: import mkl mkl.set_num_threads(num_thread_per_proc) except ImportError: print "MKL not available, so I'm not adjusting the number of threads" print "Launching %d jobs with %d MKL threads each" % (num_jobs, num_thread_per_proc) return num_jobs
def silhouette_original_clusterings(dataset='CB1', neuropil='Antennal_lobe', clusterer_or_k=60): """Returns a pandas dataframe with the silhouette index of each cluster member. The dataframe have columns (cluster_id, member_id, silhouette). """ # Read the expression matrix print('Reading expression matrix') Xdf = ExpressionDataset.dataset(dset=dataset, neuropil=neuropil).Xdf(index_type='string') # Generate a flat map cluster_id -> members print('Finding cluster assignments') clusters_df, _ = get_original_clustering(dataset=dataset, neuropil=neuropil, clusterer_or_k=clusterer_or_k) dfs = [] for cluster_id, members in zip(clusters_df.cluster_id, clusters_df.original_voxels_in_cluster): dfs.append(pd.DataFrame({'cluster_id': cluster_id, 'member_id': members})) members_df = pd.concat(dfs).set_index('member_id').loc[Xdf.index] # Compute the distance matrix - this must be parameterised print('Computing distance') import mkl mkl.set_num_threads(6) D = dicedist_metric(Xdf) # Compute silhouette # Here we could go for the faster implementation in third_party, if needed print('Computing silhouette index') members_df['silhouette'] = silhouette_samples(D.values, members_df.cluster_id.values, metric='precomputed') return (members_df. reset_index(). rename(columns=lambda col: {'index': 'member_id'}.get(col, col)) [['cluster_id', 'member_id', 'silhouette']])
def run_all(nsubs, nrois, nthreads=2): import mkl mkl.set_num_threads(nthreads) print "setup" cmats, design, grp = gen_data(nsubs=nsubs, nrois=nrois, nvoxs=nrois) print "degree" dall = time_fun(local_degree, cmats, design) print "glm" gall = time_fun(local_glm, cmats, design) print "mdmr" mall = time_fun(local_mdmr, cmats, design) print "svm" sall = time_fun(local_svm, cmats, grp) print "kmeans" kall = time_fun(local_kmeans, cmats, grp) print "end" times = np.vstack((dall, gall, mall, sall, kall)) return times
def pool_threading(nthreads=None): if nthreads is None: nthreads = omp_num_threads() try: import mkl old_mkl_num_threads = mkl.get_max_threads() mkl.set_num_threads(1) except ImportError: pass old_omp_num_threads = os.getenv('OMP_NUM_THREADS') os.environ['OMP_NUM_THREADS'] = '1' pool = multiprocessing.dummy.Pool(nthreads) yield pool pool.close() pool.join() try: mkl.set_num_threads(old_mkl_num_threads) except NameError: pass if old_omp_num_threads is not None: os.environ['OMP_NUM_THREADS'] = old_omp_num_threads else: del os.environ['OMP_NUM_THREADS']
def fftvec(vec): """ performs a fft on a vector with 3 components in the first index position This is really just a wrapper for fft, fftn and their inverses """ try: from anfft import fft, fftn fft_type = 1 except: # print "Could not import anfft, importing scipy instead." #Update 9/18/2013 -- numpy with mkl is way faster than scipy import mkl mkl.set_num_threads(8) from numpy.fft import fft, fftn fft_type = 0 if force_gpu: fft_type = 2 #set gpu fft's manually -- not sure what a automatic way would look like from numpy import complex64, shape, array, empty if vec.ndim > 2: if vec.shape[0] == 3: # "Vector": first index has size 3 so fft the other columns if fft_type==1: return array([fftn(i,measure=True) for i in vec]).astype(complex64) # result = empty(vec.shape, dtype=complex64) # result[0] = fftn(vec[0], measure=True).astype(complex64) # result[1] = fftn(vec[1], measure=True).astype(complex64) # result[2] = fftn(vec[2], measure=True).astype(complex64) # return result elif fft_type==0: return fftn(vec, axes=range(1,vec.ndim)).astype(complex64) elif fft_type==2: # return array([gpu_fft(i) for i in vec.astype(complex64)]) result = empty(vec.shape, dtype=complex64) result[0] = gpu_fft(vec[0].copy()) result[1] = gpu_fft(vec[1].copy()) result[2] = gpu_fft(vec[2].copy()) return result else: # "Scalar", fft the whole thing if fft_type==1: return fftn(vec,measure=True).astype(complex64) elif fft_type==0: return fftn(vec).astype(complex64) elif fft_type==2: return gpu_fft(vec.copy()) elif vec.ndim == 1: #Not a vector, so use fft if fft_type==1: return fft(vec,measure = True).astype(complex64) elif fft_type==0: return fft(vec).astype(complex64) elif fft_type==2: return gpu_fft(vec.astype(complex64)) else: #0th index is 3, so its a vector #return fft(vec, axis=1).astype(complex64) return array([fft(i) for i in vec])
def set_thread_count(thread_count): ''' Set the number of threads to be used by OpenMP :Parameters: thread_count : int Number of threads to be used by OpenMP ''' if mkl is not None: mkl.set_num_threads(thread_count) _omp.set_num_threads(thread_count)
def parallel_loop(args): import numpy as np import time import pysparsefht from utils import random_k_sparse try: import mkl as mkl_service # for such parallel processing, it is better # to deactivate multithreading in mkl mkl_service.set_num_threads(1) except ImportError: pass n = args[0] b = np.arange(1, n-1) K = 2**b B = 2**b # compute value of C C = np.empty_like(b) C[:np.floor(n/3)] = n/b[:np.floor(n/3)] C[np.floor(n/3):np.floor(2*n/3)] = 3 C[np.floor(2*n/3):] = n / (n - b[np.floor(2*n/3):]) algo_name = params['algo_name'] seed = args[1] if algo_name == 'RANDOM': algo = pysparsefht.ALGO_RANDOM elif algo_name == 'DETERMINISTIC': algo = pysparsefht.ALGO_OPTIMIZED else: ValueError('No such algorithm.') # initialize rng np.random.seed(seed) # a list for return values ret = [] # generate a seed for the C RNG C_seed = np.random.randint(4294967295, dtype=np.uint32) # create sparse vector Tsfht, Tfht = pysparsefht.benchmark(K, B, C, 2**n, loops=params['inner_loops'], warm=params['warm'], body=params['body'], max_mag=params['max_mag'], sfht_max_iter=params['max_iter'], seed=C_seed) return [Tsfht, Tfht, b, C]
def set_num_threads(nt): try: import mkl mkl.set_num_threads(nt) except Exception as e: print('Unable to set numthreads in mkl: ' + str(e)) cv2.setNumThreads(nt) nt = str(nt) os.environ['OPENBLAS_NUM_THREADS'] = nt os.environ['NUMEXPR_NUM_THREADS'] = nt os.environ['OMP_NUM_THREADS'] = nt os.environ['MKL_NUM_THREADS'] = nt
def _initialize_fft(self): """ Define the two-dimensional FFT methods. """ if self.use_mkl: import mkl mkl.set_num_threads(self.nthreads) import mkl_fft self.fft = (lambda x : mkl_fft.fft2(x)) self.ifft = (lambda x : mkl_fft.ifft2(x)) else: self.fft = (lambda x : np.fft.fft2(x)) self.ifft = (lambda x : np.fft.ifft2(x))
def set_num_threads(nt): "Get numpy (and others) to use `nt` threads" try: import mkl mkl.set_num_threads(nt) except: pass torch.set_num_threads(1) os.environ['IPC_ENABLE'] = '1' for o in [ 'OPENBLAS_NUM_THREADS', 'NUMEXPR_NUM_THREADS', 'OMP_NUM_THREADS', 'MKL_NUM_THREADS' ]: os.environ[o] = str(nt)
def parallel_montecarlo(filename, mapper, reducer, jobs_params, n_repetitions, seed=None, n_cpu = None): """ This function implements a basic map-reduce framework based on multiprocessing.Pool. Inputs: filename - name of output, where the results will be saved as a pickle. mapper - this is the function that runs the computaitonal job, given an element of job_params. reducer - function for aggregating n_repetitions runs of a job. job_params - list of job parameters. n_repetitions - number of times to run each job. seed - Random seed to be used. To have reproducible results, always specify a seed. n_cpu - number of processes to use. The default is to use all available cores. Outputs: reduced_results - output of reducer on the various simulations Also, the results will be saved to a pickle file Example: (this computes the means of 3 normal random variables with different means) >> parallel_montecarlo('testing', numpy.random.normal, numpy.mean, [-1,0,+1], 1000) n_cpu: 4 Saving to ./pickles/testing.pickle.gz Saved fields: n_repetitions, name, results, seed, xs Out[14]: [-0.9465148770830919, 0.03763575004851667, 1.056358627427924] """ mkl.set_num_threads(1) if n_cpu is None: n_cpu = get_n_cpu() #print(f'n_cpu: {n_cpu}') SEED = seed if seed is not None else 0 N_SEED_INTS = 4 mkl_random.seed(SEED) iteration_parameters = zip(mkl_random.randint(0, 2**32, size=(len(jobs_params)*n_repetitions, N_SEED_INTS)), itertools.cycle(jobs_params)) wrapped_job_computation_func = functools.partial(set_random_seed_and_apply_func, mapper) if n_cpu == 1: results = list(itertools.starmap(wrapped_job_computation_func, iteration_parameters)) else: with multiprocessing.Pool(processes=n_cpu) as p: results = list(p.starmap(wrapped_job_computation_func, iteration_parameters)) results_grouped_by_params = [results[i::len(jobs_params)] for i in range(len(jobs_params))] reduced_results = list(map(reducer, results_grouped_by_params)) if filename is not None: pickler.dump(filename, name=filename, xs=jobs_params, results=np.array(reduced_results), n_repetitions=n_repetitions, seed=SEED) return reduced_results
def set_fft_threads(n): """Sets number of threads used in fft functions.""" out = CDDMConfig.fft_threads CDDMConfig.fft_threads = int(n) try: import mkl mkl.set_num_threads(n) except ImportError: pass try: import pyfftw pyfftw.config.NUM_THREADS = n except ImportError: pass return out
def dtest(n=50, d=0.0, r=0.0, model_covariate=True, niters=100, nperms=4999): import mkl mkl.set_num_threads(2) d = float(d) r = float(r) # Data/Distances pvals = np.zeros(niters) Fvals = np.zeros(niters) for i in xrange(niters): # Design ## Categorical gp = np.repeat([0, 1], n/2) np.random.shuffle(gp) x = gp*d + np.random.standard_normal(n) ## Continuous # see http://stackoverflow.com/questions/16024677/generate-correlated-data-in-python-3-3 # and http://stats.stackexchange.com/questions/19367/creating-two-random-sequences-with-50-correlation?lq=1 uncorrelated = np.random.standard_normal((2,n)) motion = uncorrelated[0] y = r*motion + np.sqrt(1-r**2)*uncorrelated[1] ## Design Matrix if model_covariate: design = np.vstack((np.ones(n), gp, motion)).T else: design = np.vstack((np.ones(n), gp)).T # Date points = np.vstack((x,y)).T # Distances dmat = euclidean_distances(points) dmats = dmat[np.newaxis,:,:] # Only the group effect is the variable of interest cols = [1] # Call MDMR pval, Fval, _, _ = mdmr(dmats, design, cols, nperms) pvals[i] = pval Fvals[i] = Fval return pvals, Fvals
def dtest(n=50, d=0.0, r=0.0, model_covariate=True, niters=100, nperms=4999): import mkl mkl.set_num_threads(2) d = float(d) r = float(r) # Data/Distances pvals = np.zeros(niters) Fvals = np.zeros(niters) for i in xrange(niters): # Design ## Categorical gp = np.repeat([0, 1], n / 2) np.random.shuffle(gp) x = gp * d + np.random.standard_normal(n) ## Continuous # see http://stackoverflow.com/questions/16024677/generate-correlated-data-in-python-3-3 # and http://stats.stackexchange.com/questions/19367/creating-two-random-sequences-with-50-correlation?lq=1 uncorrelated = np.random.standard_normal((2, n)) motion = uncorrelated[0] y = r * motion + np.sqrt(1 - r**2) * uncorrelated[1] ## Design Matrix if model_covariate: design = np.vstack((np.ones(n), gp, motion)).T else: design = np.vstack((np.ones(n), gp)).T # Date points = np.vstack((x, y)).T # Distances dmat = euclidean_distances(points) dmats = dmat[np.newaxis, :, :] # Only the group effect is the variable of interest cols = [1] # Call MDMR pval, Fval, _, _ = mdmr(dmats, design, cols, nperms) pvals[i] = pval Fvals[i] = Fval return pvals, Fvals
def set_numpy_threads(num_threads): # Currently only MKL is supported on Windows as it's installed alongside # the other packages via conda. A "traditional" virtual environment requires # access to a compiler and other libraries for successful compilation. if platform.system().lower() == 'windows': import mkl mkl.set_num_threads(num_threads) return mkl.get_max_threads(), MKL_LIBRARY candidates, other_candidates = _identify_library_paths() if len(candidates) > 0: set_threads, library = _set_numpy_threads(candidates, num_threads) else: set_threads, library = _set_numpy_threads(other_candidates, num_threads) return set_threads, library
def set_num_threads(nt: int = 1): """ Set some machine parameters which control the number of threads being spawned. Default is set to 1. """ import os import mkl mkl.set_num_threads(nt) nt = str(nt) os.environ['OPENBLAS_NUM_THREADS'] = nt os.environ['NUMEXPR_NUM_THREADS'] = nt os.environ['OMP_NUM_THREADS'] = nt os.environ['MKL_NUM_THREADS'] = nt
def prep_cwas_workflow(c, subject_infos): from CPAC.cwas import create_cwas import numpy as np try: import mkl mkl.set_num_threads(c.cwasThreads) except ImportError: pass print 'Preparing CWAS workflow' p_id, s_ids, scan_ids, s_paths = (list(tup) for tup in zip(*subject_infos)) print 'Subjects', s_ids # Read in list of subject functionals lines = open(c.cwasFuncFiles).readlines() spaths = [ l.strip().strip('"') for l in lines ] # Read in design/regressor file regressor = np.loadtxt(c.cwasRegressorFile) wf = pe.Workflow(name='cwas_workflow') wf.base_dir = c.workingDirectory cw = create_cwas() cw.inputs.inputspec.roi = c.cwasROIFile cw.inputs.inputspec.subjects = spaths cw.inputs.inputspec.regressor = regressor cw.inputs.inputspec.cols = c.cwasRegressorCols cw.inputs.inputspec.f_samples = c.cwasFSamples cw.inputs.inputspec.strata = c.cwasRegressorStrata # will stay None? cw.inputs.inputspec.parallel_nodes = c.cwasParallelNodes cw.inputs.inputspec.memory_limit = c.cwasMemory cw.inputs.inputspec.dtype = c.cwasDtype ds = pe.Node(nio.DataSink(), name='cwas_sink') out_dir = os.path.dirname(s_paths[0]).replace(s_ids[0], 'cwas_results') ds.inputs.base_directory = out_dir ds.inputs.container = '' wf.connect(cw, 'outputspec.F_map', ds, 'F_map') wf.connect(cw, 'outputspec.p_map', ds, 'p_map') wf.run(plugin='MultiProc', plugin_args={'n_procs': c.numCoresPerSubject})
def set_num_threads(nt=1, disp=1): """see https://github.com/numbbo/coco/issues/1919 and https://twitter.com/jeremyphoward/status/1185044752753815552 """ try: import mkl except ImportError: disp and print("mkl is not installed") else: mkl.set_num_threads(nt) nt = str(nt) for name in [ 'OPENBLAS_NUM_THREADS', 'NUMEXPR_NUM_THREADS', 'OMP_NUM_THREADS', 'MKL_NUM_THREADS' ]: os.environ[name] = nt disp and print("setting mkl threads num to", nt)
def run_generation_games(gen_dir, ga_popsize, my_network, my_timeout, train_env, cst_decks, ini_stack, sb_amount, nb_hands, q): mkl.set_num_threads(64) # Neural network layer size reference ref_full_dict = DeepBot(network=my_network).full_dict #Empty jobs list jobs = [] for bot_id in range(1,ga_popsize+1): #Load the bot with open(gen_dir+'/bots/'+str(bot_id)+'/bot_'+str(bot_id)+'_flat.pkl', 'rb') as f: deepbot_flat = pickle.load(f) deepbot_dict = get_full_dict(all_params = deepbot_flat, ref_full_dict = ref_full_dict) deepbot = DeepBot(id_=bot_id, network=my_network, full_dict = deepbot_dict) #Enqueue job to play bot's games try: jobs.append(q.enqueue(run_games, timeout=my_timeout, kwargs = dict(train_env=train_env, deepbot=deepbot, cst_decks = cst_decks, ini_stack = ini_stack, sb_amount=sb_amount, nb_hands = nb_hands))) except ConnectionError: print('Currently not connected to redis server') continue last_enqueue_time = time.time() # Fetch jobs' statusses every second while True: for i in range(len(jobs)): if jobs[i].result is not None and not isinstance(jobs[i], FakeJob): jobs[i] = FakeJob(jobs[i]) all_earnings = [j.result for j in jobs] time.sleep(1) #1 second # If all jobs are done, break if None not in all_earnings: break # If jobs are not finished after timeout threshold, reenqueue. # Helps when connection occasionaly breaks. May also be the sign of an error in u_training_games.py. if time.time() - last_enqueue_time > my_timeout: print('Reenqueuing unfinished jobs '+ str({sum(x is None for x in all_earnings)})) for i in range(len(jobs)): if jobs[i].result is None: try: jobs[i].cancel() jobs.append(q.enqueue(run_games, timeout=my_timeout, kwargs = dict(train_env=train_env, deepbot=deepbot, cst_decks = cst_decks, ini_stack = ini_stack, sb_amount=sb_amount, nb_hands = nb_hands))) except ConnectionError: print('Currently not connected to redis server') continue last_enqueue_time = time.time() if verbose: print("Number of jobs remaining: " + str(sum([all_earnings[i]==None for i in range(len(all_earnings))]))) return all_earnings
def __enter__(self): """ Construct the run time environment """ mkl.set_num_threads(self._threads_num) if self._rd: file_name = "Log {0}.txt".format(strftime("%Y-%m-%d %H-%M-%S")) fp = open(self._path / file_name, 'w', buffering=1) self._stdout = sys.stdout sys.stdout = self._fp = fp print("Entering run time environment at {0}".format( strftime(TIME_FORMAT))) print("=" * 80, flush=True) return self
def _set_external_fft_threads(n): if DTMMConfig.thread_pool: #disable native threading if we are to use threadpool num = 1 else: num = n try: import mkl mkl.set_num_threads(num) except ImportError: pass try: import pyfftw #threadpool does not seem to work properly with pyfftw, so we use n and disable it in fft.py pyfftw.config.NUM_THREADS = n except ImportError: pass
def dtest(pos_nodes=0, effect=0.0, dist="euclidean", n=100, nodes=400, nperms=4999, iters=100): import mkl mkl.set_num_threads(2) print "Start" #print "Categorical Effect" grp = np.repeat([0, 1], n/2) np.random.shuffle(grp) #print "Design Matrix" design = np.vstack((np.ones(n), grp)).T cols = [1] #print "Distance Matrices" dmats = np.zeros((iters,n,n)) for i in xrange(iters): #if (i % 10) == 0: # print i, # Data ## Fist, I created the matrix with the random data points = np.random.standard_normal((n,nodes)) ## Second, I select a random selection of nodes to add the effect neg_nodes = nodes - pos_nodes change_nodes = np.repeat([0,1], [neg_nodes, pos_nodes]) np.random.shuffle(change_nodes) ## Finally, I add the effect to a select subjects and nodes for i in (change_nodes==1).nonzero()[0]: points[grp==1,i] += effect # Compute Distances if dist == "euclidean": dmat = euclidean_distances(points) elif dist == "pearson": dmat = compute_distances(points) else: raise Exception("Unknown distance measure %s" % dist) dmats[i] = dmat #print "" #print "MDMR" pvals = []; Fvals = []; pvals, Fvals, _, _ = mdmr(dmats, design, cols, nperms) #print "Done" return pvals, Fvals
def _initialize_mp_worker(mkey, func, threads, log_queue): _initialize_worker(log_queue) global __work_model, __work_func, __is_mp_worker __work_model = mkey __work_func = func __is_mp_worker = True import numba numba.config.NUMBA_NUM_THREADS = threads try: import mkl _log.debug('configuring Numba thread count') mkl.set_num_threads(threads) except ImportError: pass _log.debug('worker %d ready', os.getpid())
def initLUSolver(self): """LU solver initlization Due to the A matrix never change, expansive LU factorization can only run once Author:Bin Wang([email protected]) Date: April. 2020 """ #Assemble linear system for BEM for domain in self.BEMobjs: A, b, B = domain.AssembleMatrix() #Compute LU factorization at the beginning self.LU_pivs.append(lu_factor(A)) self.bs.append(b) self.Bs.append(B) #Set disable mkl backend multithreading mkl.set_num_threads(1)
def main(): logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) mkl.set_num_threads(3) parser = argparse.ArgumentParser("Create topics from word2vec model") parser.add_argument("--embedding-model", type=str) parser.add_argument("--topic-model", type=str) args = parser.parse_args() assert "topic" in args.topic_model, "'%s' not a topic model" % args.topic_model assert "embedding" in args.embedding_model, "'%s' not an embedding model" % args.embedding_model word2vec = Word2Vec.load_word2vec_format(args.embedding_model, binary=True) calculate_similarities(word2vec, os.path.basename(args.embedding_model), args.topic_model, all_pairwise=False)
def main(ctx, config, debug): '''leitmotiv - Extract Trends from Image Collections''' if debug: __LOGGER_OPTIONS['handlers']['console']['level'] = 'DEBUG' __LOGGER_OPTIONS['loggers']['leitmotiv']['level'] = 'DEBUG' __LOGGER_OPTIONS['loggers']['peewee'] = { 'handlers': ['console'], 'level': 'DEBUG', 'propagate': False } logging.config.dictConfig(__LOGGER_OPTIONS) yaml = YAML() with open(config) as f: config = yaml.load(f)['leitmotiv'] mkl.set_num_threads(config['num_threads']) library.LIBRARY_PATH = config['library']['path'] ctx.obj = config
def set_threads(num_threads=None, verbose=False, no_guessing=False): ''' Get and set the number of threads used by FFT libraries. Parameters ---------- num_threads : int, default None Number of threads requested. If None, do not set threads. verbose : bool, default False If True, output debug messages to STDOUT. no_guessing : bool, default false If False and MKL is not found at all, return a guess of 1 thread since numpy.fft and scipy.fftpack are single-threaded without MKL. If True, return len(os.sched_getaffinity(0)) or os.cpu_count(). Returns ------- int or None The number of threads successfully set, or None on failure. ''' try: import mkl except ImportError: if hasattr(np, '__mkl_version__') or no_guessing: # MKL present but no mkl-service, so guess number of CPUs if verbose: print(f'TAG: WARNING: mkl-service module was not ' f'found. Number of threads is likely inaccurate!') if hasattr(os, 'sched_getaffinity'): return len(os.sched_getaffinity(0)), 'os.sched_getaffinity' else: return os.cpu_count(), 'os.cpu_count' else: # no MKL, so assume not threaded return 1, 'guessing' else: if num_threads: mkl.set_num_threads(num_threads) return mkl.get_max_threads(), 'mkl.get_max_threads' return None, None
def run(self): """ Run the forward simulation """ # ----------------- Validate Parameters ----------------- # print('Validating parameters...') self.validate() sim_mesh = self.meshGenerator.mesh # grab the discretize mesh off of the mesh object print(' max x: {}, min z: {}, max z: {}'.format( sim_mesh.vectorNx.max(), sim_mesh.vectorNz.min(), sim_mesh.vectorNz.max() )) # save simulation parameters self.save() # --------------- Set the number of threads --------------- # mkl.set_num_threads(self.num_threads) # ----------------- Set up the simulation ----------------- # physprops = self.physprops prb = self.prob # survey = self.survey # prb.pair(survey) # ----------------- Run the the simulation ----------------- # print('Starting Simulation') t = time.time() fields = prb.fields(physprops.model) np.save( '/'.join([self.directory, self.fields_filename]), fields[:, '{}Solution'.format(self.formulation)] ) print(' ... Done. Elapsed time : {}'.format(time.time()-t)) self._fields = fields return fields
def __init__(self, fingerprintsA, fingerprintsB=None, chemicalKernelmat=None, nthreads=4): self.dtype = 'float64' self.nthreads = nthreads try: import mkl mkl.set_num_threads(self.nthreads) except: raise Warning( 'NUMPY DOES NOT SEEM TO BE LINKED TO MKL LIBRARY SO NTHREADS IS IGNORED' ) self.fingerprintsA = fingerprintsA self.fingerprints_infoA = self.get_info(fingerprintsA) pairsA = self.fingerprints_infoA['pairs'] Nframe = len(fingerprintsA) if fingerprintsB is not None: self.fingerprintsB = fingerprintsB self.fingerprints_infoB = self.get_info(fingerprintsB) pairsB = self.fingerprints_infoB['pairs'] Mframe = len(fingerprintsB) else: self.fingerprintsB = None pairsB = pairsA Mframe = Nframe # initialize data container self._storage = { pA + pB: np.zeros((Nframe, Mframe), dtype=self.dtype) for pA in pairsA for pB in pairsB } self.set_partial_kernels() self.chemicalKernelmat = chemicalKernelmat self.set_kernel(chemicalKernelmat)
def disable_multi_threading(): import ctypes from ctypes.util import find_library import os # OpenBLAS-based multi-threading libraries try_paths = [ '/opt/OpenBLAS/lib/libopenblas.so', '/lib/libopenblas.so', '/usr/lib/libopenblas.so.0', find_library('openblas') ] openblas_lib = None for libpath in try_paths: try: openblas_lib = ctypes.cdll.LoadLibrary(libpath) break except (OSError, TypeError): continue if openblas_lib is not None: try: openblas_lib.openblas_set_num_threads(1) except: pass # MKL-based multi-threading libraries try: import mkl mkl.set_num_threads(1) except: pass # Set OS variables os.environ["OMP_NUM_THREADS"] = "1" # export OMP_NUM_THREADS=1 os.environ["OPENBLAS_NUM_THREADS"] = "1" # export OPENBLAS_NUM_THREADS=1 os.environ["MKL_NUM_THREADS"] = "1" # export MKL_NUM_THREADS=1 os.environ[ "VECLIB_MAXIMUM_THREADS"] = "1" # export VECLIB_MAXIMUM_THREADS=4 os.environ["NUMEXPR_NUM_THREADS"] = "1" # export NUMEXPR_NUM_THREADS=6
def _initialize_mp_worker(mkey, func, threads, log_queue): _initialize_worker(log_queue) global __work_model, __work_func nnt_env = os.environ.get('NUMBA_NUM_THREADS', None) if nnt_env is None or int(nnt_env) > threads: _log.debug('configuring Numba thread count') import numba numba.config.NUMBA_NUM_THREADS = threads try: import mkl _log.debug('configuring MKL thread count') mkl.set_num_threads(threads) except ImportError: pass __work_model = mkey # deferred function unpickling to minimize imports before initialization __work_func = pickle.loads(func) _log.debug('worker %d ready (process %s)', os.getpid(), mp.current_process())
def mpirun(f, arguments, comm=MPI.COMM_WORLD, bcast=True): ''' Wrapper for the parallel running of f using the mpi4py. Parameters ---------- f : callable The function to be parallelly run using the mpi4py. arguments : list of tuple The list of arguments passed to the function f. comm : MPI.Comm, optional The MPI communicator. bcast : True or False When True, broadcast the result for all processes; Otherwise only the rank 0 process hold the result. Returns ------- list The returned values of f with respect to the arguments. ''' size = comm.Get_size() rank = comm.Get_rank() if size > 1: import mkl mkl.set_num_threads(1) temp = [] for i, argument in enumerate(arguments): if i % size == rank: temp.append(f(*argument)) temp = comm.gather(temp, root=0) result = [] if rank == 0: for i in range(len(arguments)): result.append(temp[i % size][i // size]) if bcast: result = comm.bcast(result, root=0) return result
def __init__(self, *args, **kwargs): super(Worker, self).__init__() self.nthreads = pyrat._nthreads # number of threads for processing if pyrat._debug is True: self.nthreads = 1 try: import mkl if self.nthreads > 1: # switch of mkl multithreading mkl.set_num_threads(1) # because we do it ourself else: mkl.set_num_threads(999) except ImportError: pass # self.blockprocess = True # blockprocessing on/off # self.blocksize = 128 # size of single block for para in self.para: # copy defaults to self setattr(self, para['var'], para['value']) for (k, v) in kwargs.items(): # copy keywords to self setattr(self, k, v) # eventually overwriting defaults if not hasattr(self, 'layer'): # if no keyword was used self.layer = pyrat.data.active # use active layer # -------------------------------------------------- self.name = self.__class__.__name__ # name of worker class (string) self.input = '' # input layer(s) self.output = '' # output layer(s) self.blockoverlap = 0 # block overlap self.vblock = False # vertical blocks on/off self.blocks = [] # list of block boundaries self.valid = [] # valid part of each block # self.block = False # actual block range / validity self.allowed_ndim = False self.require_para = False self.allowed_dtype = False
def mpirun(f,arguments,comm=MPI.COMM_WORLD,bcast=True): ''' Wrapper for the parallel running of f using the mpi4py. Parameters ---------- f : callable The function to be parallelly run using the mpi4py. arguments : list of tuple The list of arguments passed to the function f. comm : MPI.Comm, optional The MPI communicator. bcast : True or False When True, broadcast the result for all processes; Otherwise only the rank 0 process hold the result. Returns ------- list The returned values of f with respect to the arguments. ''' size=comm.Get_size() rank=comm.Get_rank() if size>1: import mkl mkl.set_num_threads(1) temp=[] for i,argument in enumerate(arguments): if i%size==rank: temp.append(f(*argument)) temp=comm.gather(temp,root=0) result=[] if rank==0: for i in range(len(arguments)): result.append(temp[i%size][i//size]) if bcast: result=comm.bcast(result,root=0) return result
def main(): file = open("w8a.txt", "r") labels = [] features = [] for line in file: mylist = line.split(" ") labels.append(int(mylist[0])) example = [0 for i in range(300)] for i in range(1,len(mylist)-1): indexAndValue = mylist[i].split(":") index = indexAndValue[0] index = int(index) example[index-1] = int(indexAndValue[1]) features.append(example) labels = np.array(labels) features = np.array(features) initial_w = np.random.random_sample(300) initial_w = initial_w * (math.sqrt(1/150)) num_iterations = 1000 learning_rate = 0.001 decay = learning_rate/num_iterations mkl.set_num_threads(4) print("Starting Gradient Descent at loss = {}".format(compute_loss(initial_w, features,labels))) print("Running...") w = gradient_descent_runner(initial_w, features, labels, learning_rate, num_iterations, decay) print("After {0} iterations, loss = {1}, Elapsed Time = {2} ".format(num_iterations, compute_loss(w, features, labels), total_time))
def __exit__(self, exc_type, exc_value, traceback): """ Close the opened file and restore sys.stdout before exit """ mkl.set_num_threads(mkl.get_max_threads()) if self._rd: print("=" * 80) if exc_type is None: print("Non exception has occurred!") else: print("Exc_type: {0}".format(exc_type)) print("Exc_value: {0}".format(exc_value)) print("Traceback:") print_tb(traceback, file=self._fp) print("=" * 80) print("Exit run time environment at: {0}".format( strftime(TIME_FORMAT)), flush=True) sys.stdout = self._stdout self._fp.close() return False
def start_benchmark(): print "Benchmark starting timing with numpy %s\nVersion: %s" % (numpy.__version__, sys.version) print ("-" * 80) for cur_threads in threads_range: header_set = False # This doesn't work: os.environ is not adjusted #os.environ[THREADS_LIMIT_ENV] = '%d' % cur_threads mkl.set_num_threads(cur_threads) print "Maximum number of threads used for computation is : %d" % cur_threads header_str = "%20s" % "Function" header_str += ' - %9s - Speedup' % 'Time [ms]' if cur_threads == 1: timings_single = [] for ii,fun in enumerate(tests): result_str = "%20s" % fun.__name__ t = timeit.Timer(stmt="%s()" % fun.__name__, setup="from __main__ import %s" % fun.__name__) res = t.repeat(repeat=3, number=1) timing = 1000.0 * sum(res)/len(res) if cur_threads == 1: timings_single.append(timing) result_str += ' - %9.1f - %5.1f' % (timing, timings_single[ii]/timing) if not header_set is True: print header_str header_set = True print result_str
### TODO: Need to be test yet! import mkl mkl.set_num_threads(56) import json import torch from sqlnet.utils import * from sqlnet.model.seq2sql import Seq2SQL from sqlnet.model.sqlnet import SQLNet import numpy as np import datetime from logger import Logger import argparse if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--constraint', action='store_true', help='If set, use constraint data; used to test constraint query translation.') parser.add_argument('--toy', action='store_true', help='If set, use small data; used for fast debugging.') parser.add_argument('--suffix', type=str, default='', help='The suffix at the end of saved model name.') parser.add_argument('--ca', action='store_true', help='Use conditional attention.') parser.add_argument('--dataset', type=int, default=0, help='0: original dataset, 1: re-split dataset') parser.add_argument('--rl', action='store_true', help='Use RL for Seq2SQL(requires pretrained model).') parser.add_argument('--baseline', action='store_true', help='If set, then train Seq2SQL model; default is SQLNet model.') parser.add_argument('--train_emb', action='store_true', help='Train word embedding for SQLNet(requires pretrained model).')
from rglib.mps import MPS sys.path.insert(0,'../') from model_rashba.views import majorana4mps,majorana4vec,dos4mps,dos4vec,sweep_fidelity,sweep_fidelity2 from model_bizard.views import smajorana4mps from localsetting import ONSV from models import * #MPI setting try: from mpi4py import MPI COMM=MPI.COMM_WORLD SIZE=COMM.Get_size() RANK=COMM.Get_rank() #if SIZE!=1: import mkl mkl.set_num_threads(1) except: print 'WARNING, NOT USING MULTITHREADING.' COMM=None SIZE=1 RANK=0 def heisenberg_vmps(J,Jz,h,nsite,append=False): ''' Run vMPS for Heisenberg model. ''' filename='mps_heisenberg_%s.dat'%(nsite) model=HeisenbergModel(J=J,Jz=Jz,h=h,nsite=nsite) if append: mps=MPS.load(filename)
def __enter__(self): mkl.set_num_threads(self.num_threads)
def execute(self): #import the algorithm module try: importStr = 'from algorithm.rating.' + self.config[ 'recommender'] + ' import ' + self.config['recommender'] exec(importStr) except ImportError: importStr = 'from algorithm.ranking.' + self.config[ 'recommender'] + ' import ' + self.config['recommender'] exec(importStr) if self.evaluation.contains('-cv'): k = int(self.evaluation['-cv']) if k <= 1 or k > 10: k = 3 mkl.set_num_threads(max(1, mkl.get_max_threads() / k)) #create the manager manager = Manager() m = manager.dict() i = 1 tasks = [] binarized = False if self.evaluation.contains('-b'): binarized = True for train, test in DataSplit.crossValidation(self.trainingData, k, binarized=binarized): fold = '[' + str(i) + ']' if self.config.contains('social'): recommender = self.config[ 'recommender'] + "(self.config,train,test,self.relation,fold)" else: recommender = self.config[ 'recommender'] + "(self.config,train,test,fold)" #create the process p = Process(target=run, args=(m, eval(recommender), i)) tasks.append(p) i += 1 #start the processes for p in tasks: p.start() if not self.evaluation.contains('-p'): p.join() #wait until all processes are completed if self.evaluation.contains('-p'): for p in tasks: p.join() #compute the mean error of k-fold cross validation self.measure = [dict(m)[i] for i in range(1, k + 1)] res = [] for i in range(len(self.measure[0])): if self.measure[0][i][:3] == 'Top': res.append(self.measure[0][i]) continue measure = self.measure[0][i].split(':')[0] total = 0 for j in range(k): total += float(self.measure[j][i].split(':')[1]) res.append(measure + ':' + str(total / k) + '\n') #output result currentTime = strftime("%Y-%m-%d %H-%M-%S", localtime(time())) outDir = LineConfig(self.config['output.setup'])['-dir'] fileName = self.config[ 'recommender'] + '@' + currentTime + '-' + str( k) + '-fold-cv' + '.txt' FileIO.writeFile(outDir, fileName, res) print('The result of %d-fold cross validation:\n%s' % (k, ''.join(res))) else: if self.config.contains('social'): recommender = self.config[ 'recommender'] + '(self.config,self.trainingData,self.testData,self.relation)' else: recommender = self.config[ 'recommender'] + '(self.config,self.trainingData,self.testData)' eval(recommender).execute()
def __init__(self): import os, sys MoleculeName = '' if len(sys.argv) < 2: print "Ideally you give me a molecule name, Defaulting to /Integrals" else: MoleculeName = sys.argv[1] self.MoleculeName = MoleculeName # Defined Globally in Main.py self.nocc = 4 self.nvirt = 4 self.nmo = 8 # These will be determined on-the-fly reading from disk anyways. self.occ = [] self.virt = [] self.all = [] self.alpha = [] self.beta = [] # Try to get a faster timestep by freezing the CoreOrbitals. # if it's not None, then freeze that many orbitals (evenly alpha and beta.) self.FreezeCore = 8 self.AvailablePropagators = ["phTDA2TCL","Whole2TCL", "AllPH"] self.Propagator = "phTDA2TCL" self.Correlated = True self.SecularApproximation = 0 # 0 => Nonsecular 1=> Secular self.ImaginaryTime = True self.Temperature = 303.15*2 self.TMax = 250.0 self.TStep = 0.01 self.tol = 1e-9 self.Safety = .9 #Ensures that if Error = Emax, the step size decreases slightly self.RK45 = True self.LoadPrev = False # self.MarkovEta = 0.05 self.MarkovEta = 0.001 self.Tc = 3000.0 self.Compiler = "gcc" self.Flags = ["-mtune=native", "-O3"] self.latex = True self.fluorescence = True # If true, this performs the fluorescence calculations in SpectralAnalysis # above a certain energy # no relaxation is included and the denominator is the bare electronic # denominator to avoid integration issues. self.UVCutoff = 300.0/27.2113 # No Relaxation above 1eV try: mkl.set_num_threads(8) except NameError: print "No MKL so I can't set the number of threads" # This is a hack switch to shut off the Boson Correlation Function # ------------------------------------------------------------------------ # Adiabatic = 0 # Implies numerical integration of expdeltaS and bosons. # Adiabatic = 1 # Implies numerical integration of expdeltaS no bosons. # Adiabatic = 2 # Implies analytical integration of expdeltaS, no bosons. # Adiabatic = 3 # Implies analytical integration of expdeltaS, no bosons, and the perturbative terms are forced to be anti-hermitian. # Adiabatic = 4 # Implies Markov Approximation. self.Adiabatic = 0 self.Inhomogeneous = False self.InhomogeneousTerms = self.Inhomogeneous self.Inhomogenous = False self.Undressed = True self.ContBath = True # Whether the bath is continuous/There is a continuous bath. self.ReOrg = True self.FiniteDifferenceBosons = False self.DipoleGuess = True # if False, a superposition of bright states will be used. self.AllDirections = True # Will initalize three concurrent propagations. self.DirectionSpecific = True self.InitialDirection = -1 # 0 = x etc. Only excites in the x direction -1=isotropic self.BeginWithStatesOfEnergy = None # self.BeginWithStatesOfEnergy = 18.7/27.2113 # self.BeginWithStatesOfEnergy = 18.2288355687/27.2113 self.PulseWidth = 1.7/27.2113 self.Plotting = True self.DoCisDecomposition = True self.DoBCT = True # plot and fourier transform the bath correlation tensor. self.DoEntropies = True if (self.Undressed): self.DoEntropies = False self.FieldThreshold = pow(10.0,-7.0) self.ExponentialStep = False #This will be set automatically if a matrix is made. self.LegendFontSize = 14 self.LabelFontSize = 16 print "--------------------------------------------" print "Running With Overall Parameters: " print "--------------------------------------------" print "self.MoleculeName", self.MoleculeName print "self.AllDirections", self.AllDirections print "self.Propagator", self.Propagator print "self.Temperature", self.Temperature print "self.TMax", self.TMax print "self.TStep", self.TStep print "self.MarkovEta", self.MarkovEta print "self.Adiabatic", self.Adiabatic print "self.Inhomogeneous", self.Inhomogeneous print "self.Undressed", self.Undressed print "self.Correlated", self.Correlated print "self.SecularApproximation", self.SecularApproximation print "self.DipoleGuess", self.DipoleGuess print "self.BeginWithStatesOfEnergy ", self.BeginWithStatesOfEnergy print "self.DoCisDecomposition", self.DoCisDecomposition print "self.DoBCT", self.DoBCT print "self.DoEntropies", self.DoEntropies print "self.FieldThreshold", self.FieldThreshold return
#!/usr/bin/env python import os, sys from os import path sys.path.append("/home2/data/Projects/CWAS/pyClusterROI") if len(sys.argv) != 2: sys.exit("Usage: %s num-threads" % sys.argv[0]) # control mkl import mkl mkl.set_num_threads(int(sys.argv[1])) ### # 1. SETUP ### print "1. Setup" obase = "/home2/data/Projects/CWAS/development+motion/spatial_cluster" # functions for connectivity metric from make_local_connectivity_ones import * # name of the maskfile that we will be using roidir = "/home2/data/Projects/CWAS/share/development+motion/rois" maskfile = path.join(roidir, "mask_gray_4mm.nii.gz") ###
def parmap(f,args,workers=None): ''' evaluates [f(a) for a in args] in parallel if workers is 0 then the built-in map is used. If workers is greater than one then the parent process spawns that many worker processes to evaluate the map. ''' starting_threads = mkl.get_max_threads() if workers is None: # starting_threads is a good estimate for the number of processes # that can be simultaneously running workers = starting_threads if workers < 0: raise ValueError('number of worker processes must be 0 or greater') if workers == 0: # use the built-in sequential map return map(f,args) # make sure that lower level functions are not running in parallel mkl.set_num_threads(1) # q_in has a max size of 1 so that args is not copied over to # the next process until absolutely necessary q_in = Queue(1) q_out = Queue() # any exceptions found by the child processes are put in this queue # and then raised by the parent q_err = Queue() # spawn worker processes procs = [] for i in range(workers): p = Process(target=_f,args=(f,q_in,q_out,q_err)) # process is starting and waiting for something to be put on q_in p.start() procs += [p] submitted_tasks = 0 for a in args: q_in.put((submitted_tasks,a)) submitted_tasks += 1 # indicate that nothing else will be added for i in range(workers): q_in.put(('DONE',None)) # allocate list of Nones and then fill it in with the results val_list = [None for i in range(submitted_tasks)] err_list = [None for i in range(submitted_tasks)] for i in range(submitted_tasks): idx,err = q_err.get() err_list[idx] = err idx,val = q_out.get() val_list[idx] = val # terminate all processes for p in procs: p.join() # close queues q_in.close() q_out.close() q_err.close() # raise an error if any were found if any([e is not None for e in err_list]): raise ParmapError(err_list) # reset the number of threads to its original value mkl.set_num_threads(starting_threads) return val_list
def parmap(f,args,workers=None): ''' evaluates [f(a) for a in args] in parallel if workers is 0 then the built-in map is used. If workers is greater than one then the parent process spawns that many worker processes to evaluate the map. Parameters ---------- f : callable a : list list of arguments to *f* workers : int, optional number of subprocess to spawn. Defaults to half the available cores plus one NOTES ----- If the *mkl* package is installed then this function first sets the maximum number of allowed threads per process to 1. This is to help prevents spawned subprocesses from using multiple cores. The number of allowed threads is reset after all subprocesses have finished. ''' if workers is None: # default number of processes to have simultaneously running workers = cpu_count()//2 + 1 if workers < 0: raise ValueError('number of worker processes must be 0 or greater') if workers == 0: # perform the map on the parent process return [f(i) for i in args] # attempt to prevent lower level functions from running in parallel if _HAS_MKL: starting_threads = mkl.get_max_threads() mkl.set_num_threads(1) # q_in has a max size of 1 so that args is not copied over to # the next process until absolutely necessary q_in = Queue(1) q_out = Queue() # any exceptions found by the child processes are put in this queue # and then raised by the parent q_err = Queue() # spawn worker processes procs = [] for i in range(workers): p = Process(target=_f,args=(f,q_in,q_out,q_err)) # process is starting and waiting for something to be put on q_in p.start() procs += [p] submitted_tasks = 0 for a in args: q_in.put((submitted_tasks,a)) submitted_tasks += 1 # indicate that nothing else will be added for i in range(workers): q_in.put(('DONE',None)) # allocate list of Nones and then fill it in with the results val_list = [None for i in range(submitted_tasks)] err_list = [None for i in range(submitted_tasks)] for i in range(submitted_tasks): idx,err = q_err.get() err_list[idx] = err idx,val = q_out.get() val_list[idx] = val # terminate all processes for p in procs: p.join() # close queues q_in.close() q_out.close() q_err.close() # raise an error if any were found if any([e is not None for e in err_list]): raise ParmapError(err_list) # reset the number of threads to its original value if _HAS_MKL: mkl.set_num_threads(starting_threads) return val_list
# How to use mkl # mkl module exposes functions which are declared in mkl_service.h. # とりあえず,mkl.set_num_threadsだけ覚えておく # mklのインポート import mkl # スレッド数の設定 # マルチコアCPUでは効果を発揮する mkl.set_num_threads(2)#CPUコア数に合わせておくと良い
help="Number of threads to parallel processes for Intel MKL") # Output parser.add_argument('-o', '--outdir', default=os.getcwd(), help="Output directory") ### # Parse and Read User Args ### args = parser.parse_args() try: import mkl mkl.set_num_threads(args.nthreads) except ImportError: pass if not args.degree and not args.eigen: raise SystemExit("--degree and/or --eigen must be specified") method_options = [args.degree, args.eigen] if not args.binarize and not args.weighted: raise SystemExit("--binarize and/or --weighted must be specified") weight_options = [args.binarize, args.weighted] if args.pvalue is not None: option = 0 threshold = args.pvalue elif args.sparsity is not None:
import numpy import numpy.fft as fft numpy.use_fastnumpy = True import time #from scipy.fftpack import fft import mkl print 'Intel MKL version:', mkl.get_version_string() print 'Intel cpu_clocks:', mkl.get_cpu_clocks() print 'Intel cpu_frequency:', mkl.get_cpu_frequency() #print 'Intel MKL, freeing buffer memory:', mkl.thread_free_buffers() print 'max Intel threads:', mkl.get_max_threads() mkl.set_num_threads(2) N = 2**16 print 'using numpy', numpy.__version__ a = numpy.random.rand(2, N) print a.shape, 'items' t0 = time.clock() for i in range(100): continue base = time.clock()-t0 fftn = fft.fftn t0 = time.clock() for i in range(100): r = fftn(a, (N,), (1,)) print 'simple loop', time.clock()-t0-base
def main(date, takeSubset=False): """ Reduces the dimensionality of the training data to 3 dimensions, plots the transformed data in 3d space. The idea is to bring out separability between the resistance classes which may be hidden in the dimensionality of the data. :param date: (string) Data collection date YYYY_MMDD :param takeSubset: (boolean) Transform and plot a random subset of the trainng data? :return: (None) """ mkl.set_num_threads(8) # Load the training and testing data into memory trainX, trainY = FileIO.loadTrainingData(date) if takeSubset: indices = np.random.choice(range(0, len(trainY)), size=NUM_SAMPLES, replace=False) X = trainX[indices,:] y = trainY[indices] else: X = trainX y = trainY X = np.nan_to_num(X) # Break the data into resistance classes susIndex = Constants.LABEL_TO_INDEX[Constants.SUSCEPTIBLE] drIndex = Constants.LABEL_TO_INDEX[Constants.DR_RESISTANT] grIndex = Constants.LABEL_TO_INDEX[Constants.GR_RESISTANT] susX = X[y==susIndex, :] drX = X[y==drIndex, :] grX = X[y==grIndex, :] # Transform the data using PCA pca = IncrementalPCA(n_components=6) pointsSUS = pca.fit_transform(susX) pointsGR= pca.fit_transform(grX) pointsDR = pca.fit_transform(drX) # Plot the transformed data in 3D space traceSUS = go.Scatter3d( x=pointsSUS[:, 0], y=pointsSUS[:, 1], z=pointsSUS[:, 2], mode='markers', marker=dict( size=5, line=dict( color='rgba(255, 0, 0, 0)', width=0.1 ), opacity=0 ) ) traceDR = go.Scatter3d( x=pointsDR[:, 0], y=pointsDR[:, 1], z=pointsDR[:, 2], mode='markers', marker=dict( size=5, line=dict( color='rgba(0, 255, 0, 0)', width=0.1 ), opacity=0 ) ) traceGR = go.Scatter3d( x=pointsGR[:, 0], y=pointsGR[:, 1], z=pointsGR[:, 2], mode='markers', marker=dict( size=5, line=dict( color='rgba(0, 0, 255, 0)', width=0.1 ), opacity=0 ) ) data = [traceSUS, traceDR, traceGR] fig = go.Figure(data=data) py.iplot(fig, filename='3D PCA Wavelength Plot') # Plot the principle components eigenSpectra = pca.components_ plt.subplot(3,1,1) plt.plot(Constants.WAVELENGTHS, eigenSpectra[0, :]) plt.title("Principle Components 1 - 3") plt.subplot(3,1,2) plt.plot(Constants.WAVELENGTHS, eigenSpectra[1, :]) plt.subplot(3,1,3) plt.plot(Constants.WAVELENGTHS, eigenSpectra[2, :]) plt.xlabel("Wavelength (nm)") plt.show() plt.clf() plt.subplot(3,1,1) plt.plot(Constants.WAVELENGTHS, eigenSpectra[3, :]) plt.title("Principle Components 4 - 6") plt.subplot(3,1,2) plt.plot(Constants.WAVELENGTHS, eigenSpectra[4, :]) plt.subplot(3,1,3) plt.plot(Constants.WAVELENGTHS, eigenSpectra[5, :]) plt.xlabel("Wavelength (nm)") plt.show()
def parallel_loop(args): import numpy as np import scipy.linalg as la import time import pysparsefht from utils import random_k_sparse try: import mkl as mkl_service # for such parallel processing, it is better # to deactivate multithreading in mkl mkl_service.set_num_threads(1) except ImportError: pass K = int(np.round(2**(args[0]*params['n']))) B = int(2**params['b']) C = int(params['C']) algo_name = params['algo_name'] seed = args[1] if algo_name == 'RANDOM': algo = pysparsefht.ALGO_RANDOM elif algo_name == 'DETERMINISTIC': algo = pysparsefht.ALGO_OPTIMIZED else: ValueError('No such algorithm.') # initialize rng np.random.seed(seed) # a list for return values ret = [] # Run the inner loops for i in range(params['inner_loops']): # generate a seed for the C RNG C_seed = np.random.randint(4294967295, dtype=np.uint32) # create sparse vector x_hat, y_hat, supp_hat = random_k_sparse(params['N'], K, params['sigma2']) # compute WHT x = pysparsefht.fht(x_hat) # Now apply the SparseFHT y_hat2, supp_hat2, unsat, loops = pysparsefht.sparse_fht(x, int(K), int(B), int(C), max_iter=params['max_iter'], algo=algo, req_loops=True, req_unsat=True, seed=C_seed) supp_r = supp_hat2[supp_hat2 > 0] y_r = y_hat2[supp_hat2 > 0] # reconstructed vector x_hat2 = np.zeros(params['N']) x_hat2[supp_r] = y_r mse = la.norm(x_hat - x_hat2)**2 supp_size = supp_r.shape[0] bit_error = len(set(supp_r).symmetric_difference(set(supp_hat))) success = (unsat[0] == 0) ret.append([mse, supp_size, bit_error, success, unsat, loops]) return ret
def parallel_loop(filename, algo_names, pmt): ''' This is one loop of the computation extracted for parallelization ''' # We need to do a bunch of imports import pyroomacoustics as pra import os import numpy as np from scipy.io import wavfile import mkl as mkl_service import copy import doa from tools import rfft # for such parallel processing, it is better # to deactivate multithreading in mkl mkl_service.set_num_threads(1) # exctract the speaker names from filename name = os.path.splitext(os.path.basename(filename))[0] sources = name.split('-') # number of sources K = len(sources) # Import speech signal fs_file, rec_signals = wavfile.read(filename) # sanity check if pmt['fs'] != fs_file: raise ValueError('The sampling frequency of the files doesn''t match that of the script') speech_signals = np.array(rec_signals[:,pmt['mic_select']], dtype=np.float32) # Remove the DC bias for s in speech_signals.T: s[:] = pra.highpass(s, pmt['fs'], 100.) if pmt['stft_win']: stft_win = np.hanning(pmt['nfft']) else: stft_win = None # Normalize the amplitude speech_signals *= pmt['scaling'] # Compute STFT of signal # ------------------------- y_mic_stft = [] for k in range(speech_signals.shape[1]): y_stft = pra.stft(speech_signals[:, k], pmt['nfft'], pmt['stft_hop'], transform=rfft, win=stft_win).T / np.sqrt(pmt['nfft']) y_mic_stft.append(y_stft) y_mic_stft = np.array(y_mic_stft) # estimate SNR in dB (on 1st microphone) sig_var = np.var(speech_signals) SNR = 10*np.log10( (sig_var - pmt['noise_var']) / pmt['noise_var'] ) freq_bins = copy.copy(pmt['freq_bins'][K-1]) # dict for output phi_recon = {} for alg in algo_names: # Use the convenient dictionary of algorithms defined d = doa.algos[alg]( L=pmt['mic_array'], fs=pmt['fs'], nfft=pmt['nfft'], num_src=K, c=pmt['c'], theta=pmt['phi_grid'], max_four=pmt['M'], num_iter=pmt['num_iter'], G_iter = pmt['G_iter'] ) # perform localization d.locate_sources(y_mic_stft, freq_bins=freq_bins[alg]) # store result phi_recon[alg] = d.phi_recon return SNR, sources, phi_recon
import os, sys from os import path as op import scipy import nibabel as nb import numpy as np from pandas import read_table, read_csv from patsy import dmatrices, dmatrix from CPAC.cwas import cwas from CPAC.cwas.utils import calc_subdists, calc_mdmrs from CPAC.cwas.subdist import * from CPAC.cwas.mdmr import mdmr, gen_perms import mkl mkl.set_num_threads(8) #### # Cool Functions #### def load_subject(filepath, dtype='float64'): return nb.load(filepath).get_data().astype(dtype) def load_subjects(filepaths, dtype='float64'): print "Loading Subjects" funcs = [ load_subject(fp, dtype) for fp in filepaths ] return funcs def rois2voxels(dat, rois):
from trainer import Trainer from utils import (update_task, get_max_of_db_column, get_a_task, ExploitationNeeded, LossIsNaN, get_task_ids_and_scores, PopulationFinished, get_col_from_populations, RemainingTasksTaken, print_with_time, ExploitationOcurring, create_new_population) from config import (get_optimizer, DATA_DIR, MODEL_CLASS, LOSS_FN, HYPERPARAM_NAMES, EPOCHS, BATCH_SIZE, POPULATION_SIZE, EXPLOIT_INTERVAL, USE_SQLITE) if __name__ == "__main__": # TODO: Does this help? nproc = mkl.get_max_threads() # e.g. 12 mkl.set_num_threads(nproc) parser = argparse.ArgumentParser(description="Population Based Training") parser.add_argument("-g", "--gpu", type=int, default=0, help="Selects GPU with the given ID. IDs are those shown in nvidia-smi.") # noqa parser.add_argument("-p", "--population_id", type=int, default=None, help="Resumes work on the population with the given ID. Use -1 to select the most recently created population. Without this flag, a new population will be created.") # noqa parser.add_argument("-e", "--exploiter", action="store_true", help="Set this process as the exploiter. It will be responsible for running the exploit step over the entire population at the end of each interval.") # noqa args = parser.parse_args() gpu = args.gpu population_id = args.population_id exploiter = args.exploiter inputs = bcolz.open(osp.join(DATA_DIR, "trn_inputs.bcolz"), 'r') targets = bcolz.open(osp.join(DATA_DIR, "trn_targets.bcolz"), 'r') pathlib.Path('checkpoints').mkdir(exist_ok=True) checkpoint_str = "checkpoints/pop-%03d_task-%03d.pth" interval_limit = int(np.ceil(EPOCHS / EXPLOIT_INTERVAL))
#!/usr/bin/env python # Create spatially constrained ROIs using Cameron's pyClusterROI import os, sys from os import path sys.path.append("/home2/data/Projects/CWAS/pyClusterROI") # control mkl import mkl mkl.set_num_threads(4) ### # 1. SETUP ### obase = "/home2/data/Projects/CWAS/age+gender/01_resolution/spatial_cluster" rbase = "/home2/data/Projects/CWAS/share/age+gender/analysis/01_resolution/rois" # functions to save to nifti from make_image_from_bin_renum import * ### # 2. Generate Individual Connectivity Matrices ### # Done in 04*
mute_error=False) ########################################################################################### ## Remove the fake datasets because we don't need them anymore ########################################################################################### spdc_glob = glob(dir_fakes + os.path.sep + "S*_spdc_distorcorr_{0}_ROC.fits".format(fakes_spectrum)) for filename in spdc_glob: print("Removing {0}".format(filename)) os.remove(filename) if __name__ == "__main__": try: import mkl mkl.set_num_threads(1) except: pass print(platform.system()) OS = platform.system() if OS == "Windows": print("Using WINDOWS!!") else: print("I hope you are using a UNIX OS") print('Number of arguments:', len(sys.argv), 'arguments.') print('Argument List:', str(sys.argv)) print("CPU COUNT: {0}".format(mp.cpu_count()))
def get_projection_operator(self, sampling, scene, i_band=None, verbose=True): """ Return the peak sampling operator. Parameters ---------- sampling : QubicSampling The pointing information. scene : QubicScene The observed scene. verbose : bool, optional If true, display information about the memory allocation. """ if not isinstance(scene.nu, float) and i_band == None: mask = [self.detector.index < 2295//2, self.detector.index > 2295//2] dtype = self.synthetic_beam.dtype if scene.nside > 8192: dtype_index = np.dtype(np.int64) else: dtype_index = np.dtype(np.int32) theta0, phi, vals = _peak_angles_fraction(self, scene, self.synthetic_beam.fraction, 0) theta1, phi, vals = _peak_angles_fraction(self, scene, self.synthetic_beam.fraction, 1) ncolmax = max(theta0.shape[-1], theta1.shape[-1]) cls = {'I' : FSRMatrix, 'QU' : FSRRotation2dMatrix, 'IQU': FSRRotation3dMatrix}[scene.kind] ndims = len(scene.kind) s = cls((len(self) * len(sampling) * ndims, 12 * scene.nside**2 * ndims), ncolmax=ncolmax, dtype=dtype, dtype_index=dtype_index, verbose=verbose) for i in [0, 1]: focal_plane = self[mask[i]] p = focal_plane.get_projection_operator(sampling, scene, i_band=i) smask = np.multiply.outer(mask[i], np.ones(len(sampling)), dtype=bool) smask = np.asarray(smask).reshape(-1) s.data[smask, :p.matrix.data.shape[-1]] = p.matrix.data # set_trace() # sh = s.data[mask[i], p.matrix.data.shape[-1]:].shape # tp = {'I' : "int, float", # 'IQ' : "int, float, float", # 'IQU' : "int, float, float, float"}[scene.kind] # s.data[mask[i], p.matrix.data.shape[-1]:] = np.ones(sh, dtype=tp) shapeout = {'I' : (len(self), len(sampling)), 'IQ' : (len(self), len(sampling), 2), 'IQU' : (len(self), len(sampling), 3)}[scene.kind] return ProjectionOperator(s, shapeout=shapeout) else: rotation = sampling.cartesian_galactic2instrument dtype = self.synthetic_beam.dtype fraction = self.synthetic_beam.fraction ndetectors = len(self) ntimes = len(sampling) nside = scene.nside theta, phi, vals = _peak_angles_fraction(self, scene, fraction, i_band) ncolmax = theta.shape[-1] thetaphi = _pack_vector(theta, phi) # (ndetectors, ncolmax, 2) direction = Spherical2CartesianOperator('zenith,azimuth')(thetaphi) e_nf = direction[:, None, :, :] if nside > 8192: dtype_index = np.dtype(np.int64) else: dtype_index = np.dtype(np.int32) cls = {'I': FSRMatrix, 'QU': FSRRotation2dMatrix, 'IQU': FSRRotation3dMatrix}[scene.kind] ndims = len(scene.kind) s = cls((ndetectors * ntimes * ndims, 12 * nside**2 * ndims), ncolmax=ncolmax, dtype=dtype, dtype_index=dtype_index, verbose=verbose) index = s.data.index.reshape((ndetectors, ntimes, ncolmax)) nthreads = openmp_num_threads() try: import mkl mkl.set_num_threads(1) except: pass def func_thread(i): # e_nf[i] shape: (1, ncolmax, 3) # e_ni shape: (ntimes, ncolmax, 3) e_ni = rotation.T(e_nf[i].swapaxes(0, 1)).swapaxes(0, 1) index[i] = Cartesian2HealpixOperator(nside)(e_ni) pool = Pool(nthreads) pool.map(func_thread, xrange(ndetectors)) pool.close() pool.join() try: mkl.set_num_threads(nthreads) except: pass if scene.kind == 'I': value = s.data.value.reshape(ndetectors, ntimes, ncolmax) value[...] = vals[:, None, :] shapeout = (ndetectors, ntimes) else: func = 'pointing_matrix_rot{0}d_i{1}_m{2}'.format( ndims, dtype_index.itemsize, dtype.itemsize) try: getattr(flib.polarization, func)( rotation.data.T, direction.T, s.data.ravel().view(np.int8), vals.T) except AttributeError: raise TypeError( 'The projection matrix cannot be created with types: {0} a' 'nd {1}.'.format(dtype, dtype_index)) if scene.kind == 'QU': shapeout = (ndetectors, ntimes, 2) else: shapeout = (ndetectors, ntimes, 3) return ProjectionOperator(s, shapeout=shapeout)