def mcmc(data, uncert=None, func=None, indparams=[], params=None, pmin=None, pmax=None, stepsize=None, prior=None, priorlow=None, priorup=None, nchains=10, nproc=None, nsamples=10, walk='demc', wlike=False, leastsq=True, lm=False, chisqscale=False, grtest=True, grbreak=0.01, grnmin=0.5, burnin=0, thinning=1, fgamma=1.0, fepsilon=0.0, hsize=1, kickoff='normal', plots=False, ioff=False, showbp=True, savefile=None, savemodel=None, resume=False, rms=False, log=None, pnames=None, texnames=None, full_output=False, chireturn=False, parname=None): """ This beautiful piece of code runs a Markov-chain Monte Carlo algorithm. Parameters ---------- data: 1D ndarray Dependent data fitted by func. uncert: 1D ndarray Uncertainty of data. func: callable or string-iterable The callable function that models data as: model = func(params, *indparams) Or an iterable (list, tuple, or ndarray) of 3 strings: (funcname, modulename, path) that specify the function name, function module, and module path. If the module is already in the python-path scope, path can be omitted. indparams: tuple Additional arguments required by func. params: 1D or 2D ndarray Set of initial fitting parameters for func. If 2D, of shape (nparams, nchains), it is assumed that it is one set for each chain. pmin: 1D ndarray Lower boundaries of the posteriors. pmax: 1D ndarray Upper boundaries of the posteriors. stepsize: 1D ndarray Proposal jump scale. If a values is 0, keep the parameter fixed. Negative values indicate a shared parameter (See Note 1). prior: 1D ndarray Parameter prior distribution means (See Note 2). priorlow: 1D ndarray Lower prior uncertainty values (See Note 2). priorup: 1D ndarray Upper prior uncertainty values (See Note 2). nchains: Scalar Number of simultaneous chains to run. nproc: Integer The number of processors for the MCMC chains (consider that MC3 uses one other CPU for the central hub). nsamples: Scalar Total number of samples. walk: String Random walk algorithm: - 'mrw': Metropolis random walk. - 'demc': Differential Evolution Markov chain. - 'snooker': DEMC-z with snooker update. wlike: Boolean If True, calculate the likelihood in a wavelet-base. This requires three additional parameters (See Note 3). leastsq: Boolean Perform a least-square minimization before the MCMC run. lm: Boolean If True use the Levenberg-Marquardt algorithm for the optimization. If False, use the Trust Region Reflective algorithm. chisqscale: Boolean Scale the data uncertainties such that the reduced chi-squared = 1. grtest: Boolean Run Gelman & Rubin test. grbreak: Float Gelman-Rubin convergence threshold to stop the MCMC (I'd suggest grbreak ~ 1.001--1.005). Do not break if grbreak=0.0 (default). grnmin: Integer or float Minimum number of samples required for grbreak to stop the MCMC. If grnmin > 1: grnmin sets the minimum required number of samples. If 0 < grnmin < 1: grnmin sets the minimum required nsamples fraction. burnin: Scalar Burned-in (discarded) number of iterations at the beginning of the chains. thinning: Integer Thinning factor of the chains (use every thinning-th iteration) used in the GR test and plots. fgamma: Float Proposals jump scale factor for DEMC's gamma. The code computes: gamma = fgamma * 2.38 / sqrt(2*Nfree) fepsilon: Float Jump scale factor for DEMC's support distribution. The code computes: e = fepsilon * Normal(0, stepsize) hsize: Integer Number of initial samples per chain. kickoff: String Flag to indicate how to start the chains: 'normal' for normal distribution around initial guess, or 'uniform' for uniform distribution withing the given boundaries. plots: Bool If True plot parameter traces, pairwise-posteriors, and posterior histograms. ioff: Bool If True, set plt.ioff(), i.e., do not display figures on screen. showbp: Bool If True, show best-fitting values in histogram and pairwise plots. savefile: String If not None, filename to store allparams and other MCMC results. savemodel: String If not None, filename to store the values of the evaluated function (with np.save). resume: Boolean If True resume a previous run. rms: Boolean If True, calculate the RMS of the residuals: data - bestmodel. log: String or FILE pointer Filename or File object to write log. pnames: 1D string iterable List of parameter names (including fixed and shared parameters) to display on output screen and figures. See also texnames. Screen output trims up to the 11th character. If not defined, default to texnames. texnames: 1D string iterable Parameter names for figures, which may use latex syntax. If not defined, default to pnames. full_output: Bool If True, return the full posterior sample, including the burned-in iterations. chireturn: Bool If True, include chi-squared statistics in the return. parname: 1D string ndarray Deprecated, use pnames. Returns ------- bestp: 1D ndarray Array of the best-fitting parameters (including fixed and shared). CRlo: 1D ndarray The lower boundary of the marginal 68%-highest posterior density (the credible region) for each parameter, with respect to bestp. CRhi: 1D ndarray The upper boundary of the marginal 68%-highest posterior density (the credible region) for each parameter, with respect to bestp. stdp: 1D ndarray Array of the best-fitting parameter uncertainties, calculated as the standard deviation of the marginalized, thinned, burned-in posterior. posterior: 2D float ndarray An array of shape (Nfreepars, Nsamples) with the thinned MCMC posterior distribution of the fitting parameters (excluding fixed and shared). If full_output is True, the posterior includes the burnin samples. Zchain: 1D integer ndarray Index of the chain for each sample in posterior. M0 samples have chain index of -1. chiout: 4-elements tuple Tuple containing the best-fit chi-square, reduced chi-square, scale factor to enforce redchisq=1, and the Bayesian information criterion (BIC). Note: Returned only if chireturn=True. Notes ----- 1.- To set one parameter equal to another, set its stepsize to the negative index in params (Starting the count from 1); e.g.: to set the second parameter equal to the first one, do: stepsize[1] = -1. 2.- If any of the fitting parameters has a prior estimate, e.g., param[i] = p0 +up/-low, with up and low the 1sigma uncertainties. This information can be considered in the MCMC run by setting: prior[i] = p0 priorup[i] = up priorlow[i] = low All three: prior, priorup, and priorlow must be set and, furthermore, priorup and priorlow must be > 0 to be considered as prior. 3.- FINDME: WAVELET LIKELIHOOD Examples -------- >>> # See https://github.com/pcubillos/MCcubed/tree/master/examples Uncredited developers --------------------- Kevin Stevenson (UCF) """ if ioff: plt.ioff() # Open log file if input is a filename: if isinstance(log, str): log = mu.Log(log, append=resume) closelog = True else: closelog = False if log is None: log = mu.Log(logname=None) if parname is not None: log.error("'parname' argument is deprecated. Use 'pnames' instead.") if resume: log.msg("\n\n{:s}\n{:s} Resuming previous MCMC run.\n\n".format( log.sep, log.sep)) log.msg( "\n{:s}\n" " Multi-core Markov-chain Monte Carlo (MC3).\n" " Version {:d}.{:d}.{:d}.\n" " Copyright (c) 2015-{:d} Patricio Cubillos and collaborators.\n" " MC3 is open-source software under the MIT license (see LICENSE).\n" "{:s}\n\n".format(log.sep, ver.MC3_VER, ver.MC3_MIN, ver.MC3_REV, date.today().year, log.sep)) # Import the model function: if type(func) in [list, tuple, np.ndarray]: if len(func) == 3: sys.path.append(func[2]) fmodule = importlib.import_module(func[1]) func = getattr(fmodule, func[0]) elif not callable(func): log.error("'func' must be either a callable or an iterable of strings " "with the model function, file, and path names.") if nproc is None: # Default to Nproc = Nchains: nproc = nchains # Cap the number of processors: if nproc >= mpr.cpu_count(): log.warning( "The number of requested CPUs ({:d}) is >= than the number " "of available CPUs ({:d}). Enforced nproc to {:d}.".format( nproc, mpr.cpu_count(), mpr.cpu_count() - 1)) nproc = mpr.cpu_count() - 1 nparams = len(params) # Number of model params ndata = len(data) # Number of data values # Set default uncertainties: if uncert is None: uncert = np.ones(ndata) # Setup array of parameter names: if pnames is None and texnames is not None: pnames = texnames elif pnames is not None and texnames is None: texnames = pnames elif pnames is None and texnames is None: pnames = texnames = mu.default_parnames(nparams) pnames = np.asarray(pnames) texnames = np.asarray(texnames) # Set uncert as shared-memory object: sm_uncert = mpr.Array(ctypes.c_double, uncert) uncert = np.ctypeslib.as_array(sm_uncert.get_obj()) # Set default boundaries: if pmin is None: pmin = np.tile(-np.inf, nparams) if pmax is None: pmax = np.tile(np.inf, nparams) # Set default stepsize: if stepsize is None: stepsize = 0.1 * np.abs(params) stepsize = np.asarray(stepsize) # Set prior parameter indices: if (prior is None) or (priorup is None) or (priorlow is None): prior = priorup = priorlow = np.zeros(nparams) # Zero arrays # Check that initial values lie within the boundaries: if (np.any(np.asarray(params) < pmin) or np.any(np.asarray(params) > pmax)): pout = "" for (pname, par, minp, maxp) in zip(pnames, params, pmin, pmax): if par < minp: pout += "\n{:11s} {: 12.5e} < {: 12.5e}".format( pname[:11], minp, par) if par > maxp: pout += "\n{:26s} {: 12.5e} > {: 12.5e}".format( pname[:11], par, maxp) log.error("Some initial-guess values are out of bounds:\n" "Param name pmin value pmax\n" "----------- ------------ ------------ ------------" "{:s}".format(pout)) nfree = int(np.sum(stepsize > 0)) # Number of free parameters ifree = np.where(stepsize > 0)[0] # Free parameter indices ishare = np.where(stepsize < 0)[0] # Shared parameter indices # Initial number of samples: M0 = hsize * nchains # Number of Z samples per chain: nZchain = int(np.ceil(nsamples / nchains / thinning)) # Number of iterations per chain: niter = nZchain * thinning # Total number of Z samples (initial + chains): Zlen = M0 + nZchain * nchains # Initialize shared-memory free params array: sm_freepars = mpr.Array(ctypes.c_double, nchains * nfree) freepars = np.ctypeslib.as_array(sm_freepars.get_obj()) freepars = freepars.reshape((nchains, nfree)) # Get lowest chi-square and best fitting parameters: bestchisq = mpr.Value(ctypes.c_double, np.inf) sm_bestp = mpr.Array(ctypes.c_double, np.copy(params)) bestp = np.ctypeslib.as_array(sm_bestp.get_obj()) # There seems to be a strange behavior with np.ctypeslib.as_array() # when the argument is a single-element array. In this case, the # returned value is a two-dimensional array, instead of 1D. The # following line fixes(?) that behavior: if np.ndim(bestp) > 1: bestp = bestp.flatten() if not resume and niter < burnin: log.error("The number of burned-in samples ({:d}) is greater than " "the number of iterations per chain ({:d}).".format( burnin, niter)) # Check that output path exists: if savefile is not None: fpath, fname = os.path.split(os.path.realpath(savefile)) if not os.path.exists(fpath): log.warning("Output folder path: '{:s}' does not exist. " "Creating new folder.".format(fpath)) os.makedirs(fpath) # Intermediate steps to run GR test and print progress report: intsteps = (nZchain * nchains) / 10 report = intsteps # Initial size of posterior (prior to this MCMC sample): size0 = M0 if resume: oldrun = np.load(savefile) Zold = oldrun["Z"] Zlen_old = np.shape(Zold)[0] # Previous MCMC Zchain_old = oldrun["Zchain"] # Redefine Zlen to include the previous runs: Zlen = Zlen_old + nZchain * nchains size0 = Zlen_old # Allocate arrays with variables: numaccept = mpr.Value(ctypes.c_int, 0) outbounds = mpr.Array(ctypes.c_int, nfree) # Out of bounds proposals #if savemodel is not None: # allmodel = np.zeros((nchains, ndata, niter)) # Fit model # Z array with the chains history: sm_Z = mpr.Array(ctypes.c_double, Zlen * nfree) Z = np.ctypeslib.as_array(sm_Z.get_obj()) Z = Z.reshape((Zlen, nfree)) # Chi-square value of Z: sm_Zchisq = mpr.Array(ctypes.c_double, Zlen) Zchisq = np.ctypeslib.as_array(sm_Zchisq.get_obj()) # Chain index for given state in the Z array: sm_Zchain = mpr.Array(ctypes.c_int, -np.ones(Zlen, np.int)) Zchain = np.ctypeslib.as_array(sm_Zchain.get_obj()) # Current number of samples in the Z array: Zsize = mpr.Value(ctypes.c_int, M0) # Burned samples in the Z array per chain: Zburn = int(burnin / thinning) # Include values from previous run: if resume: Z[0:Zlen_old, :] = Zold Zchisq[0:Zlen_old] = oldrun["Zchisq"] Zchain[0:Zlen_old] = oldrun["Zchain"] # Redefine Zsize: Zsize.value = Zlen_old numaccept.value = int(oldrun["numaccept"]) # Set GR N-min: if grnmin > 0 and grnmin < 1: # As a fraction: grnmin = int(grnmin * (Zlen - M0 - Zburn * nchains)) elif grnmin > 1: # As the number of iterations: pass else: log.error( "Invalid 'grnmin' argument (minimum number of samples to stop" "the MCMC under GR convergence), must either be grnmin > 1" "to set the minimum number of samples, or 0 < grnmin < 1" "to set the fraction of samples required to evaluate.") # Add these to compare grnmin to Zsize (which also include them): grnmin += int(M0 + Zburn * nchains) # Current length of each chain: sm_chainsize = mpr.Array(ctypes.c_int, np.tile(hsize, nchains)) chainsize = np.ctypeslib.as_array(sm_chainsize.get_obj()) # Number of chains per processor: ncpp = np.tile(int(nchains / nproc), nproc) ncpp[0:nchains % nproc] += 1 # Launch Chains: pipes = [] chains = [] for i in range(nproc): p = mpr.Pipe() pipes.append(p[0]) chains.append( ch.Chain(func, indparams, p[1], data, uncert, params, freepars, stepsize, pmin, pmax, walk, wlike, prior, priorlow, priorup, thinning, fgamma, fepsilon, Z, Zsize, Zchisq, Zchain, M0, numaccept, outbounds, ncpp[i], chainsize, bestp, bestchisq, i, nproc)) if resume: # Set bestp and bestchisq: bestp = oldrun["bestp"] bestchisq.value = oldrun["bestchisq"] for c in range(nchains): chainsize[c] = np.sum(Zchain_old == c) chifactor = float(oldrun['chifactor']) uncert *= chifactor else: fitpars = np.asarray(params) # Least-squares minimization: if leastsq: fitchisq, fitbestp, dummy, dummy = mf.modelfit( fitpars, func, data, uncert, indparams, stepsize, pmin, pmax, prior, priorlow, priorup, lm) # Store best-fitting parameters: bestp[ifree] = np.copy(fitbestp[ifree]) # Store minimum chisq: bestchisq.value = fitchisq log.msg( "Least-squares best-fitting parameters:\n {:s}\n\n".format( str(fitbestp[ifree])), si=2) # Populate the M0 initial samples of Z: Z[0] = np.clip(bestp[ifree], pmin[ifree], pmax[ifree]) for j in range(nfree): idx = ifree[j] if kickoff == "normal": # Start with a normal distribution vals = np.random.normal(params[idx], stepsize[idx], M0 - 1) # Stay within pmin and pmax boundaries: vals[np.where(vals < pmin[idx])] = pmin[idx] vals[np.where(vals > pmax[idx])] = pmax[idx] Z[1:M0, j] = vals elif kickoff == "uniform": # Start with a uniform distribution Z[1:M0, j] = np.random.uniform(pmin[idx], pmax[idx], M0 - 1) # Evaluate models for initial sample of Z: for i in range(M0): fitpars[ifree] = Z[i] # Update shared parameters: for s in ishare: fitpars[s] = fitpars[-int(stepsize[s]) - 1] Zchisq[i] = chains[0].eval_model(fitpars, ret="chisq") # Best-fitting values (so far): Zibest = np.argmin(Zchisq[0:M0]) bestchisq.value = Zchisq[Zibest] bestp[ifree] = np.copy(Z[Zibest]) # FINDME: think what to do with this: #models = np.zeros((nchains, ndata)) # Scale data-uncertainties such that reduced chisq = 1: chifactor = 1.0 if chisqscale: chifactor = np.sqrt(bestchisq.value / (ndata - nfree)) uncert *= chifactor # Re-calculate chisq with the new uncertainties: for i in range(M0): fitpars[ifree] = Z[i] for s in ishare: fitpars[s] = fitpars[-int(stepsize[s]) - 1] Zchisq[i] = chains[0].eval_model(fitpars, ret="chisq") # Re-calculate best-fitting parameters with new uncertainties: if leastsq: fitchisq, fitbestp, dummy, dummy = mf.modelfit( fitpars, func, data, uncert, indparams, stepsize, pmin, pmax, prior, priorlow, priorup, lm) bestp[ifree] = np.copy(fitbestp[ifree]) bestchisq.value = fitchisq log.msg( "Least-squares best-fitting parameters (rescaled chisq):\n" " {:s}\n\n".format(str(fitbestp[ifree])), si=2) #if savemodel is not None: # allmodel[:,:,0] = models # :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: # Start loop: print("Yippee Ki Yay Monte Carlo!") log.msg("Start MCMC chains ({:s})".format(time.ctime())) for chain in chains: chain.start() bit = bool(1) # Dummy variable to send through pipe for DEMC while True: # Proposal jump: if walk == "demc": # Send and receive bit for synchronization: for pipe in pipes: pipe.send(bit) for pipe in pipes: b = pipe.recv() # Print intermediate info: if (Zsize.value - size0 >= report) or (Zsize.value == Zlen): report += intsteps log.progressbar((Zsize.value + 1.0 - size0) / (nZchain * nchains)) log.msg("Out-of-bound Trials:\n{:s}".format( str(np.asarray(outbounds[:]))), width=80) log.msg("Best Parameters: (chisq={:.4f})\n{:s}".format( bestchisq.value, str(bestp[ifree])), width=80) # Save current results: if savefile is not None: np.savez(savefile, Z=Z, Zchain=Zchain) #if savemodel is not None: # np.save(savemodel, allmodel) # Gelman-Rubin statistics: if grtest and np.all(chainsize > (Zburn + hsize)): psrf = gr.gelmanrubin(Z, Zchain, Zburn) log.msg("Gelman-Rubin statistics for free parameters:\n{:s}". format(str(psrf)), width=80) if np.all(psrf < 1.01): log.msg( "All parameters have converged to within 1% of unity.") if (grbreak > 0.0 and np.all(psrf < grbreak) and Zsize.value > grnmin): with Zsize.get_lock(): Zsize.value = Zlen log.msg( "\nAll parameters satisfy the GR convergence threshold " "of {:g}, stopping the MCMC.".format(grbreak)) break if Zsize.value == Zlen: break for chain in chains: # Make sure to terminate the subprocesses chain.terminate() #if savemodel is not None: # modelstack = allmodel[0,:,burnin:] # for c in range(1, nchains): # modelstack = np.hstack((modelstack, allmodel[c, :, burnin:])) # Print out Summary: log.msg("\nFin, MCMC Summary:\n------------------") # Evaluate model for best fitting parameters: fitpars = np.asarray(params) fitpars[ifree] = np.copy(bestp[ifree]) for s in ishare: fitpars[s] = fitpars[-int(stepsize[s]) - 1] bestmodel = chains[0].eval_model(fitpars) # Truncate sample (if necessary): Ztotal = M0 + np.sum(Zchain >= 0) Zchain = Zchain[:Ztotal] Zchisq = Zchisq[:Ztotal] Z = Z[:Ztotal] # Get indices for samples considered in final analysis: good = np.zeros(len(Zchain), bool) for c in range(nchains): good[np.where(Zchain == c)[0][Zburn:]] = True # Values accepted for posterior stats: posterior = Z[good] pchain = Zchain[good] # Sort the posterior by chain: zsort = np.lexsort([pchain]) posterior = posterior[zsort] pchain = pchain[zsort] # Get some stats: nsample = np.sum(Zchain >= 0) * thinning # Total samples run nZsample = len(posterior) # Valid samples (after thinning and burning) BIC = bestchisq.value + nfree * np.log(ndata) if ndata > nfree: redchisq = bestchisq.value / (ndata - nfree) else: redchisq = np.nan sdr = np.std(bestmodel - data) fmt = len(str(nsample)) log.msg("Total number of samples: {:{}d}".format(nsample, fmt), indent=2) log.msg("Number of parallel chains: {:{}d}".format(nchains, fmt), indent=2) log.msg("Average iterations per chain: {:{}d}".format( nsample // nchains, fmt), indent=2) log.msg("Burned-in iterations per chain: {:{}d}".format(burnin, fmt), indent=2) log.msg("Thinning factor: {:{}d}".format(thinning, fmt), indent=2) log.msg("MCMC sample size (thinned, burned): {:{}d}".format(nZsample, fmt), indent=2) log.msg("Acceptance rate: {:.2f}%\n".format(numaccept.value * 100.0 / nsample), indent=2) # Compute the credible region for each parameter: CRlo = np.zeros(nparams) CRhi = np.zeros(nparams) pdf = [] xpdf = [] for i in range(nfree): PDF, Xpdf, HPDmin = mu.credregion(posterior[:, i]) pdf.append(PDF) xpdf.append(Xpdf) CRlo[ifree[i]] = np.amin(Xpdf[PDF > HPDmin]) CRhi[ifree[i]] = np.amax(Xpdf[PDF > HPDmin]) # CR relative to the best-fitting value: CRlo[ifree] -= bestp[ifree] CRhi[ifree] -= bestp[ifree] # Get the mean and standard deviation from the posterior: meanp = np.zeros(nparams, np.double) # Parameters mean stdp = np.zeros(nparams, np.double) # Parameter standard deviation meanp[ifree] = np.mean(posterior, axis=0) stdp[ifree] = np.std(posterior, axis=0) for s in ishare: bestp[s] = bestp[-int(stepsize[s]) - 1] meanp[s] = meanp[-int(stepsize[s]) - 1] stdp[s] = stdp[-int(stepsize[s]) - 1] CRlo[s] = CRlo[-int(stepsize[s]) - 1] CRhi[s] = CRhi[-int(stepsize[s]) - 1] log.msg( "\nParam name Best fit Lo HPD CR Hi HPD CR Mean Std dev S/N" "\n----------- ----------------------------------- ---------------------- ---------", width=80) for i in range(nparams): snr = "{:.1f}".format(np.abs(bestp[i]) / stdp[i]) mean = "{: 11.4e}".format(meanp[i]) lo = "{: 11.4e}".format(CRlo[i]) hi = "{: 11.4e}".format(CRhi[i]) if i in ifree: # Free-fitting value pass elif i in ishare: # Shared value snr = "[share{:02d}]".format(-int(stepsize[i])) else: # Fixed value snr = "[fixed]" mean = "{: 11.4e}".format(bestp[i]) log.msg( "{:<11s} {:11.4e} {:>11s} {:>11s} {:>11s} {:10.4e} {:>9s}".format( pnames[i][0:11], bestp[i], lo, hi, mean, stdp[i], snr), width=160) if leastsq and bestchisq.value - fitchisq < -3e-8: np.set_printoptions(precision=8) log.warning( "MCMC found a better fit than the minimizer:\n" "MCMC best-fitting parameters: (chisq={:.8g})\n{:s}\n" "Minimizer best-fitting parameters: (chisq={:.8g})\n" "{:s}".format(bestchisq.value, str(bestp[ifree]), fitchisq, str(fitbestp[ifree]))) fmt = len("{:.4f}".format(BIC)) # Length of string formatting log.msg(" ") if chisqscale: log.msg("sqrt(reduced chi-squared) factor: {:{}.4f}".format( chifactor, fmt), indent=2) log.msg("Best-parameter's chi-squared: {:{}.4f}".format( bestchisq.value, fmt), indent=2) log.msg("Bayesian Information Criterion: {:{}.4f}".format(BIC, fmt), indent=2) log.msg("Reduced chi-squared: {:{}.4f}".format(redchisq, fmt), indent=2) log.msg("Standard deviation of residuals: {:.6g}\n".format(sdr), indent=2) # Save definitive results: if savefile is not None: np.savez(savefile, bestp=bestp, Z=Z, Zchain=Zchain, Zchisq=Zchisq, CRlo=CRlo, CRhi=CRhi, stdp=stdp, meanp=meanp, bestchisq=bestchisq.value, redchisq=redchisq, chifactor=chifactor, BIC=BIC, sdr=sdr, numaccept=numaccept.value) #if savemodel is not None: # np.save(savemodel, allmodel) if rms: RMS, RMSlo, RMShi, stderr, bs = ta.binrms(bestmodel - data) if plots: print("Plotting figures.") # Extract filename from savefile: if savefile is not None: if savefile.rfind(".") == -1: fname = savefile else: # Cut out file extention. fname = savefile[:savefile.rfind(".")] else: fname = "MCMC" # Include bestp in posterior plots: if showbp: bestfreepars = bestp[ifree] else: bestfreepars = None # Trace plot: mp.trace(Z, Zchain=Zchain, burnin=Zburn, pnames=texnames[ifree], savefile=fname + "_trace.png") # Pairwise posteriors: mp.pairwise(posterior, pnames=texnames[ifree], bestp=bestfreepars, savefile=fname + "_pairwise.png") # Histograms: mp.histogram(posterior, pnames=texnames[ifree], bestp=bestfreepars, savefile=fname + "_posterior.png", percentile=0.683, pdf=pdf, xpdf=xpdf) # RMS vs bin size: if rms: mp.RMS(bs, RMS, stderr, RMSlo, RMShi, binstep=len(bs) // 500 + 1, savefile=fname + "_RMS.png") # Sort of guessing that indparams[0] is the X array for data as in y=y(x): if (indparams != [] and isinstance(indparams[0], (list, tuple, np.ndarray)) and np.size(indparams[0]) == ndata): try: mp.modelfit(data, uncert, indparams[0], bestmodel, savefile=fname + "_model.png") except: pass # Close the log file if necessary: if closelog: log.close() # Build the output tuple: output = bestp, CRlo, CRhi, stdp if full_output: output += (Z, Zchain) else: output += (posterior, pchain) chiout = (bestchisq.value, redchisq, chifactor, BIC) if chireturn: output += (chiout, ) return output
def main(): parser = argparse.ArgumentParser( description="making feature file argsurations.") parser.add_argument("--log_dir", required=True, type=str, help="directory to save the log") parser.add_argument("--wav_dir", default=None, help="directory of input wavfile") parser.add_argument("--hdf5dir", default=None, help="directory to save hdf5") parser.add_argument("--conf_path", default=None, type=str, help="Path to the config file") parser.add_argument("--fs", default=None, type=int, help="Sample rate.") parser.add_argument("--shiftms", default=None, type=int, help="Shift ms.") parser.add_argument("--mcep_dim", default=None, type=int, help="Dimension of mel cepstrum") parser.add_argument("--fftl", default=None, type=int, help="FFT length") parser.add_argument("--n_jobs", default=1, type=int, help="number of parallel jobs") args = parser.parse_args() # set log level logging.basicConfig( level=logging.INFO, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s', datefmt='%m/%d/%Y %I:%M:%S', filename=args.log_dir + "/feature_extract.log") logging.getLogger().addHandler(logging.StreamHandler()) # mcep_alpha if args.fs == 16000: mcep_alpha = 0.41 elif args.fs == 22050: mcep_alpha = 0.455 elif args.fs == 24000: mcep_alpha = 0.466 elif args.fs == 44100: mcep_alpha = 0.544 elif args.fs == 48000: mcep_alpha = 0.554 else: raise ValueError('sampling rate should be one of \ 16000, 22050, 24000, 44100, 48000') # read list file_list = sorted(glob.glob(os.path.join(args.wav_dir, "*.wav"))) # load config with open(args.conf_path, 'r', encoding='utf-8') as f: t = f.readlines() minf0 = int(t[0]) maxf0 = int(t[1]) thpow = float(t[2]) def feature_extract(wav_list, arr): n_sample = 0 n_frame = 0 max_frame = 0 count = 1 coeff = np.array([-0.5, 0.5, 0.0]) for wav_name in wav_list: # load wavfile and apply low cut filter fs, x = read_wav(wav_name, cutoff=70) n_sample += x.shape[0] logging.info(wav_name + " " + str(x.shape[0]) + " " + str(n_sample) + " " + str(count)) # check sampling frequency if not fs == args.fs: logging.debug("ERROR: sampling frequency is not matched.") sys.exit(1) hdf5name = args.hdf5dir + "/" + os.path.basename(wav_name).replace( ".wav", ".h5") # extimate f0 and ap time_axis, f0, spc, ap = analyze_range(x, fs=args.fs, minf0=minf0, maxf0=maxf0, fperiod=args.shiftms, fftl=args.fftl) write_hdf5(hdf5name, '/ap', ap) write_hdf5(hdf5name, "/f0", f0) # convert to continuous f0 and low-pass filter uv, cont_f0 = convert_continuos_f0(np.array(f0)) cont_f0_lpf = low_pass_filter(cont_f0, int(1.0 / (args.shiftms * 0.001)), cutoff=20) cont_f0_lpf = np.expand_dims(cont_f0_lpf, axis=-1) uv = np.expand_dims(uv, axis=-1) write_hdf5(hdf5name, "/lcf0", np.log(cont_f0_lpf)) write_hdf5(hdf5name, "/uv", uv) # extimate codeap codeap = pw.code_aperiodicity(ap, args.fs) if codeap.ndim == 1: # when fs == 16000 codeap = np.expand_dims(codeap, axis=-1) write_hdf5(hdf5name, "/codeap", codeap) # mcep mcep = ps.sp2mc(spc, args.mcep_dim, mcep_alpha) write_hdf5(hdf5name, "/mcep", mcep) # divie list file_lists = np.array_split(file_list, args.n_jobs) file_lists = [f_list.tolist() for f_list in file_lists] # multi processing processes = [] arr = mp.Array('d', 3) for f in file_lists: p = mp.Process(target=feature_extract, args=(f, arr)) p.start() processes.append(p) # wait for all process for p in processes: p.join()
#!/usr/bin/python import multiprocessing as mp import numpy as np # choose number of processes num_procs = 4 num_things_per_proc = 10 # create a list of things from which to operate mpVals = mp.Array('i', num_procs * num_things_per_proc, lock=False) # populate the array in serial, cause this is cheap for i in range(num_procs * num_things_per_proc): mpVals[i] = i # create a lock object for atomicity array_lock = mp.Lock() # define work that each process does atomically def print_segment(id, segment_start_index, segment_end_index): #with array_lock: for i in range(segment_start_index, segment_end_index): print("Process: " + str(id) + ", value: " + str(mpVals[i])) # create an empty processes array processes = []
if __name__ == "__main__": job_list = [] MC = 2 * 10**4 step = 0.5 start = 0 end = 2 EbNs = np.arange(start, (end + step), step) ArraySize = EbNs.size Error_shape = (ArraySize, MC) ErrorUncoded_shared = multiprocessing.Array('i', Error_shape[0] * Error_shape[1], lock=True) ErrorUncoded = np.frombuffer(ErrorUncoded_shared.get_obj(), dtype=np.int32).reshape(Error_shape) np.copyto(ErrorUncoded, np.zeros(Error_shape, dtype=int)) ErrorMSA_shared = multiprocessing.Array('i', Error_shape[0] * Error_shape[1], lock=True) ErrorMSA = np.frombuffer(ErrorMSA_shared.get_obj(), dtype=np.int32).reshape(Error_shape) np.copyto(ErrorMSA, np.zeros(Error_shape, dtype=int)) IterBF = np.zeros(Error_shape, dtype=int) IterSBF = np.zeros(Error_shape, dtype=int)
def __init__(self, _graph, _machines, order_dict, *args): # initialize the tk.Tk class portion tk.Tk.__init__(self, _graph['gui_title']) tk.Tk.wm_title(self, _graph['gui_title']) tk.Tk.columnconfigure(self, 0, weight=1) tk.Tk.rowconfigure(self, 0, weight=1) # dictionaries of measurement settings, graph settings and results self.settings = args self.graph = _graph self.machines = _machines self.results = { 'x_data': mp.Array('d', 1500), 'y_data': mp.Array('d', 1500), # for cases where Gaussmeter data is recorded 'x2_data': mp.Array('d', 1500), # counts the number of data points to graph 'counter': mp.Value('i', 0), # int value of the percentage done the total measurement loop is 'progress': mp.Value('i', 0), 'time': mp.Value('i', 0) } self.queue = mp.Queue() # directions for measure module import (since can't be pickled) and loop direction commands self.order = order_dict # make sure that there is a Measurements folder that exists if os.path.isdir( os.path.expanduser('~\\Documents') + '\\Measurements'): pass else: os.mkdir(os.path.expanduser('~\\Documents') + '\\Measurements') # save parameters self.directory = os.path.expanduser('~\\Documents\\Measurements') # loop and file name initial values, later changed to widgets self.file_name = 'test_file' self.loop = ['low-high', 'zero-zero', 'high-low', 'standard'] # multiprocessing.Process controls, for quit and pause functions self.measure_process = '' # initialized when measurement button is pressed # flag which will pause process at the top of the measurement loop self.pause_flag = mp.Event() # flag that kills the measurement process after safely releasing resources self.quit_flag = mp.Event() # initialize the plot figure and axes necessary for drawing self.fig = plt.Figure(figsize=(6, 5), dpi=100) self.ax = self.fig.add_subplot(111) # Main frame that will hold the frames for each specific group of widgets base_frame = ttk.Frame(self) base_frame.grid(row=0, column=0, sticky='nsew') base_frame.columnconfigure(0, weight=1) base_frame.rowconfigure(0, weight=1) # The four frames that house each different section of widgets plot_frame = ttk.Frame(base_frame, borderwidth=5) settings_frame = ttk.Frame(base_frame, borderwidth=5) info_frame = ttk.Frame(base_frame, borderwidth=5) buttons_frame = ttk.Frame(base_frame, borderwidth=5) progress_frame = ttk.Frame(base_frame, borderwidth=5) # Use grid to place all frames on the base_frame plot_frame.grid(row=0, column=0, sticky='nsew') settings_frame.grid(row=0, column=1, sticky='nsew') progress_frame.grid(row=1, column=0, sticky='nsew') info_frame.grid(row=2, column=0, sticky='nsew') buttons_frame.grid(row=1, column=1, rowspan=2, sticky='nsew') # creates and grids the listbox and scroll bar self.datalog = tk.Listbox(info_frame, height=5) self.y_scroll = ttk.Scrollbar(info_frame, command=self.datalog.yview) self.datalog['yscrollcommand'] = self.y_scroll.set self.datalog.grid(column=0, row=0, sticky='nsew') self.y_scroll.grid(column=1, row=0, sticky='ns') # test that title is indeed string try: if type(self.graph['graph_title'] + "\n" + self.graph['fixed_param_1'] + " " + self.graph['fixed_param_2']) is str: self.ax.set_title(self.graph['graph_title'] + "\n" + self.graph['fixed_param_1'] + " " + self.graph['fixed_param_2']) else: raise Exception( 'Graph Title or Fixed Parameters not a string.') except Exception as err: print('An exception occured for graph titles: ' + str(err)) self.datalog.insert( 'end', 'An exception occured for graph titles: ' + str(err)) self.datalog.see('end') # test that x and y axis labels are indeed strings try: if type(self.graph['x_title'] + self.graph['y_title']) is str: self.ax.set_xlabel(self.graph['x_title']) self.ax.set_ylabel(self.graph['y_title']) else: raise Exception('X or Y Axis label not a string.') except Exception as err2: print('An exception occured for axis labels: ' + str(err2)) self.datalog.insert( 'end', 'An exception occured for axis labels: ' + str(err2)) self.datalog.see('end') # Draw the plot on canvas plot_canvas = FigureCanvasTkAgg(self.fig, plot_frame) plot_canvas.draw() plot_canvas.get_tk_widget().grid(row=0, column=0, pady=0, padx=0, sticky='nsew') # variable that updates in animation, letting user how long left there is self.time_var = tk.StringVar() self.time_var.set('Time Remaining') # creates and grids a progress bar and estimated time remaining label self.progress_bar = ttk.Progressbar(progress_frame, orient='horizontal', length=500, mode='determinate', maximum=100, value=0) self.progress_bar.grid(column=0, row=0, sticky='nsew') self.time_label = ttk.Label(progress_frame, textvar=self.time_var) self.time_label.grid(column=1, row=0, sticky='nsew') # make test settings for dic in self.settings: try: if type(dic) is dict: # for each dictionary in the list, build items self.make_form(settings_frame, dic) else: raise Exception('List element (' + str(dic) + ') must be a dictionary') except Exception as err3: print( 'An exception occured with the list of measurement parameters: ' + str(err3)) self.datalog.insert( 'end', 'An exception occured with the list of measurement parameters: ' + str(err3)) self.datalog.see('end') # make loop parameters buttons/labels/entries (the misc commands) misc_lf = ttk.LabelFrame(settings_frame, text='Measurement Options') misc_lf.grid(ipadx=2, ipady=5, sticky='nsew') file_name_lbl = ttk.Label(misc_lf, width=20, text='File Name:', anchor='w') file_name_ent = ttk.Entry(misc_lf, width=15) file_name_ent.insert(0, self.file_name) # update dictionary value to the entry self.file_name = file_name_ent file_name_lbl.grid(row=0, column=0, sticky='nsew') file_name_ent.grid(row=0, column=1, sticky='nsew') loop_lbl = ttk.Label(misc_lf, width=20, text='Loop Type:', anchor='w') loop_box = ttk.Combobox(misc_lf, width=10, state='readonly', values=self.loop) loop_box.set('low-high') self.loop = loop_box # update dictionary value to the box loop_lbl.grid(row=1, column=0, sticky='nsew') loop_box.grid(row=1, column=1, sticky='nsew') # make the buttons self.measure_button = ttk.Button(buttons_frame, text='Measure', command=lambda: self.measure_method()) self.output_button = ttk.Button(buttons_frame, text='Output', command=lambda: self.output_method()) self.dir_button = ttk.Button( buttons_frame, text='Change Directory', command=lambda: self.change_directory_method()) self.stop_button = ttk.Button(buttons_frame, text='Stop', command=lambda: self.stop_method(), state='disabled') self.quit_button = ttk.Button(buttons_frame, text='Quit', command=lambda: self.quit_method()) # grid buttons self.measure_button.grid(row=0, column=0, sticky='nsew') self.output_button.grid(row=1, column=0, sticky='nsew') self.dir_button.grid(row=2, column=0, sticky='nsew') self.stop_button.grid(row=3, column=0, sticky='nsew') self.quit_button.grid(row=4, column=0, sticky='nsew') # weights columns/rows that are of weight 1, frames with specific weights are written explicitly self.weight_col(buttons_frame) self.weight_col(settings_frame) self.weight_col(plot_frame) self.weight_row(buttons_frame) self.weight_row(plot_frame) self.weight_row(info_frame) # necessary to keep the scroll bar tiny info_frame.columnconfigure(0, weight=1) info_frame.columnconfigure(1, weight=0) # necessary to keep the label tiny progress_frame.columnconfigure(0, weight=1) progress_frame.columnconfigure(1, weight=0) # test that all instruments are on and the gpib addresses are correct self.check_resources()
def test_make_file_index(self): from mock_config import mock_batches, mock_expected_malloc_requests, \ mock_class_index_map, mock_file_index_malloc, mock_file_index_list mock_batches = copy.deepcopy(mock_batches) mock_class_index_map = copy.deepcopy(mock_class_index_map) mock_file_index_malloc = copy.deepcopy(mock_file_index_malloc) mock_expected_malloc_requests = copy.deepcopy( mock_expected_malloc_requests) mock_expected_malloc_requests.extend(mock_file_index_malloc) max_batches = len(mock_batches) batch_size = 500 read_size = 2 * batch_size bucket_length = 10 comm_driver = QueueCommDriver({'ctl': 10, 'in': 100, 'out': 100}) # building queue up with read requests for batch in mock_batches: success = comm_driver.write('in', batch) self.assertTrue(success) reader_id = 0 shared_data_pointer = range(bucket_length) for bucket in shared_data_pointer: data_sets = [] for request in mock_expected_malloc_requests: # TODO requests are not ordered!!! # TODO not sure it matters as long as its consistent # reshape the requested shape to match the batch_size shape = (read_size, ) + request[1] shared_array_base = multiprocessing.Array(ctypes.c_double, np.prod(shape), lock=False) shared_array = np.ctypeslib.as_array(shared_array_base) shared_array = shared_array.reshape(shape) data_sets.append(shared_array) state = multiprocessing.Value('i', 0) generator_start_counter = multiprocessing.Value('i', 0) generator_end_counter = multiprocessing.Value('i', 0) shared_data_pointer[bucket] = [ state, data_sets, generator_start_counter, generator_end_counter ] reader = FileReader(worker_id=reader_id, comm_driver=comm_driver, shared_memory_pointer=shared_data_pointer, max_batches=max_batches, read_size=read_size, class_index_map=mock_class_index_map, make_one_hot=True, make_class_index=True, make_file_index=True, file_index_list=mock_file_index_list, io_ctlr=IOController() # io_driver=H5DataIODriver() ) reader.read() out = list() start_time = datetime.datetime.utcnow() end_time = datetime.timedelta(seconds=10) + start_time while datetime.datetime.utcnow() < end_time and len( out) != max_batches: batch = comm_driver.read('out', block=False) if batch is not None: out.append(batch) self.assertEquals( len(out), max_batches, "test consumed {0} of {1} expected batches from the in_queue". format(len(out), max_batches))
num_proc = 6 threads_arr = {} for i in range(num_proc): threads_arr[i] = [] pi_id = 0 for i in range(len(y_pred)): if y_pred[i] == 0: for j in way_dict[i]: if y_pred[j] == 1: t_id = j if find else i threads_arr[pi_id % num_proc].append(list(X[t_id])) pi_id += 1 result_dis_workers = multiprocessing.Array('d', num_proc) result_ids_workers = multiprocessing.Array('i', num_proc) for i in range(num_proc): result_dis_workers[i] = 0 ps = [] for pi in range(num_proc): ps.append( multiprocessing.Process(target=calc_distance, args=(pi, threads[0], threads_arr[pi], result_dis_workers, result_ids_workers))) for p in ps: p.start()
""" created by dyx on 2021/7/16. """ import multiprocessing def func(num): num[2] = 999 # 子进程改变数组,主进程跟着改变 if __name__ == '__main__': num = multiprocessing.Array("i", [1, 2, 3, 4, 5]) # 主进程和子进程共享这个数组 print(num[:]) p = multiprocessing.Process(target=func, args=(num, )) p.start() p.join() print(num[:])
def calculo_raices(lista_numeros, lista_raices, suma_total): for i, numero in enumerate(lista_numeros): lista_raices[i] = math.sqrt(numero) suma_total.value = sum(lista_raices) print('ID del proceso en ejecución: {}'.format(os.getpid())) print("Las raices de las lista son: {}".format(lista_raices[:])) print("La suma de las raices de la lista es: {} \n".format( suma_total.value)) if __name__ == "__main__": lista_numeros = [2, 4, 9, 16, 25, 36, 49, 64, 81, 100] lista_raices = multiprocessing.Array('d', 10) suma_total = multiprocessing.Value('d') p1 = multiprocessing.Process(target=calculo_raices, args=( lista_numeros, lista_raices, suma_total, )) p2 = multiprocessing.Process(target=calculo_raices, args=( lista_numeros, lista_raices, suma_total, ))
def denseampcor(self, slcImage1=None, slcImage2=None): if not (slcImage1 == None): self.slcImage1 = slcImage1 if (self.slcImage1 == None): logger.error("Error. reference slc image not set.") raise Exception if not (slcImage2 == None): self.slcImage2 = slcImage2 if (self.slcImage2 == None): logger.error("Error. secondary slc image not set.") raise Exception self.fileLength1 = self.slcImage1.getLength() self.lineLength1 = self.slcImage1.getWidth() self.fileLength2 = self.slcImage2.getLength() self.lineLength2 = self.slcImage2.getWidth() ####Run checks self.checkTypes() self.checkWindows() ####Actual processing coarseAcross = self.acrossGrossOffset coarseDown = self.downGrossOffset xMargin = 2 * self.searchWindowSizeWidth + self.windowSizeWidth yMargin = 2 * self.searchWindowSizeHeight + self.windowSizeHeight #####Set image limits for search offAc = max(self.margin, -coarseAcross) + xMargin if offAc % self.skipSampleAcross != 0: leftlim = offAc offAc = self.skipSampleAcross * ( 1 + int(offAc / self.skipSampleAcross)) - self.pixLocOffAc while offAc < leftlim: offAc += self.skipSampleAcross offDn = max(self.margin, -coarseDown) + yMargin if offDn % self.skipSampleDown != 0: toplim = offDn offDn = self.skipSampleDown * ( 1 + int(offDn / self.skipSampleDown)) - self.pixLocOffDn while offDn < toplim: offDn += self.skipSampleDown offAcmax = int(coarseAcross + ( (self.rangeSpacing1 / self.rangeSpacing2) - 1) * self.lineLength1) lastAc = int( min(self.lineLength1, self.lineLength2 - offAcmax) - xMargin - 1 - self.margin) offDnmax = int(coarseDown + ((self.prf2 / self.prf1) - 1) * self.fileLength1) lastDn = int( min(self.fileLength1, self.fileLength2 - offDnmax) - yMargin - 1 - self.margin) self.gridLocAcross = range(offAc + self.pixLocOffAc, lastAc - self.pixLocOffAc, self.skipSampleAcross) self.gridLocDown = range(offDn + self.pixLocOffDn, lastDn - self.pixLocOffDn, self.skipSampleDown) startAc, endAc = offAc, self.gridLocAcross[-1] - self.pixLocOffAc self.numLocationAcross = int((endAc - startAc) / self.skipSampleAcross + 1) self.numLocationDown = len(self.gridLocDown) self.offsetCols, self.offsetLines = self.numLocationAcross, self.numLocationDown print('Pixels: ', self.lineLength1, self.lineLength2) print('Lines: ', self.fileLength1, self.fileLength2) print('Wins : ', self.windowSizeWidth, self.windowSizeHeight) print('Srch: ', self.searchWindowSizeWidth, self.searchWindowSizeHeight) #####Create shared memory objects numlen = self.numLocationAcross * self.numLocationDown self.locationDown = np.frombuffer(mp.Array('i', numlen).get_obj(), dtype='i') self.locationDownOffset = np.frombuffer(mp.Array('f', numlen).get_obj(), dtype='f') self.locationAcross = np.frombuffer(mp.Array('i', numlen).get_obj(), dtype='i') self.locationAcrossOffset = np.frombuffer(mp.Array('f', numlen).get_obj(), dtype='f') self.snr = np.frombuffer(mp.Array('f', numlen).get_obj(), dtype='f') self.cov1 = np.frombuffer(mp.Array('f', numlen).get_obj(), dtype='f') self.cov2 = np.frombuffer(mp.Array('f', numlen).get_obj(), dtype='f') self.cov3 = np.frombuffer(mp.Array('f', numlen).get_obj(), dtype='f') self.locationDownOffset[:] = -10000.0 self.locationAcrossOffset[:] = -10000.0 self.snr[:] = 0.0 self.cov1[:] = 999.0 self.cov2[:] = 999.0 self.cov3[:] = 999.0 ###run ampcor on parallel processes threads = [] nominal_load = self.numLocationDown // self.numberThreads flat_indices = np.arange(numlen).reshape( (self.numLocationDown, self.numLocationAcross)) ofmt = 'Thread %d: %7d%7d%7d%7d%7d%7d' for thrd in range(self.numberThreads): # Determine location down grid indices for thread if thrd == self.numberThreads - 1: proc_num_grid = self.numLocationDown - thrd * nominal_load else: proc_num_grid = nominal_load istart = thrd * nominal_load iend = istart + proc_num_grid # Compute corresponding global line/down indices proc_loc_down = self.gridLocDown[istart:iend] startDown, endDown = proc_loc_down[0], proc_loc_down[-1] numDown = int((endDown - startDown) // self.skipSampleDown + 1) # Get flattened grid indices firstind = flat_indices[istart:iend, :].ravel()[0] lastind = flat_indices[istart:iend, :].ravel()[-1] print( ofmt % (thrd, firstind, lastind, startAc, endAc, startDown, endDown)) # Launch job args = (startAc, endAc, startDown, endDown, self.numLocationAcross, numDown, firstind, lastind) threads.append(mp.Process(target=self._run_ampcor, args=args)) threads[-1].start() # Wait for all threads to finish for thread in threads: thread.join() self.firstSampAc, self.firstSampDown = self.locationAcross[ 0], self.locationDown[0] self.lastSampAc, self.lastSampDown = self.locationAcross[ -1], self.locationDown[-1] #### Scale images (default is 1.0 to keep as pixel) self.locationDownOffset *= self.scaling_factor self.locationAcrossOffset *= self.scaling_factor self.write_slantrange_images()
def main(): hashes = [] if len(sys.argv) <= 1: print("Usage: python3 main.py <hashfile> <outfile>") return elif len(sys.argv) == 3: hashfile = sys.argv[1] outfile = sys.argv[2] try: hashes = load_hashes(hashfile) except IOError: print("Error: Unable to read hash file (%s)" % hashfile) return try: outStream = open(outfile, "w+") outStream.close() except IOError: print("Error: Unable to create output file (%s)" % outfile) return lines = 0 with open(wordlist, "r") as file: for line in file: lines += 1 hashLock = multiprocessing.Lock() hashStatus = multiprocessing.Array('i', [0] * len(hashes)) print("Wordlist length: " + str(lines)) print("Available CPUs: %d" % CPUCOUNT) print("Creating %d processes..." % CPUCOUNT) start = time.time() procs = [] for i in range(1, CPUCOUNT + 1): p = multiprocessing.Process(target=crack, args=(hashes, hashStatus, hashLock, CPUCOUNT, i, outfile)) p.start() procs.append(p) for p in procs: p.join() end = time.time() print("All processes have completed.") # Aggregate data and print summary info outHashes = [] data = None with open(outfile, 'r') as file: data = file.readlines() count = 0 for line in data: line = line.split(":") h = Hash(line[0], count, "") h.password = line[1].replace('\n', '') outHashes.append(h) count += 1 hashCount = len(hashStatus) crackedCount = 0 for h in hashStatus: if h == 1: crackedCount += 1 for h in hashes: if hashStatus[h.id] == 1: h.cracked = True for o in outHashes: if h.value == o.value: o.cracked = h.cracked o.username = h.username break print("------------------Summary------------------") print("Cracked %d/%d hashes in %.5f seconds" % (crackedCount, hashCount, (end - start))) for o in outHashes: if o.cracked: print("%s:%s (%s)" % (o.value, o.password, o.username))
else: new_model = False # prevent `main` from being set to `eval32` at loading save_d = np.load(sdir + save_nm, allow_pickle=True).item() for key in save_vars + state_vars + training_ex_vars: if key == 'save_nm': continue exec('%s = save_d["%s"]' % (key, key)) print save_nm ################### shared memory variables ###### self play from `eval` model used for training `eval32`: s_board = mp.Array('h', board.ravel( )) # shape: (BUFFER_SZ, gv.n_rows, gv.n_cols, gv.n_input_channels) s_winner = mp.Array( 'b', winner.ravel()) # (N_BATCH_SETS_TOTAL, N_TURNS, 2, gv.BATCH_SZ) s_tree_probs = mp.Array('f', tree_probs.ravel()) # (BUFFER_SZ, gv.map_szt) # indices, counters, & flags buffer_loc = mp.Value('i', buffer_loc) # index into above ^ training vars weights_changed = mp.Value('i', 0) # 0 = no change, 1 = changed batch_sets_created = mp.Value('i', batch_sets_created) batch_sets_created_total = mp.Value('i', batch_sets_created_total) batch_set = mp.Value('i', batch_set) # evaluation (`eval` vs `main` benchmark testing to see when to update `main` to the current `eval32` backprop weights) scope_next = mp.Value( 'i', 0 ) # alternates between 0,1 during model evaluation to dictate if `eval` or `main` starts 1st
# Get the maximum size for all images. max_w = 0 max_h = 0 for f in args.filespec: im = Image.open(f) (w, h) = im.size max_w = max(max_w, w) max_h = max(max_h, h) # Create an array to store the numeric totals for each pixel. # This is a special multithreading thing--this Array will allow # us to share memory between the threads. Results are "rolled up" # into a one-dimenisonal array. totals = multiprocessing.Array('i', [0 for _ in range(3 * max_h * max_w)]) last_im = None cutoff_squared = args.cutoff[0] * args.cutoff[0] # Square for square norm. # Process the antiphoto. # This can take a long time so we can employ multiple threads. num_files = len(args.filespec) if num_files - 1 < args.threads[0]: args.threads[0] = num_files - 1 num_files_per_thread = math.ceil(num_files / args.threads[0]) for t in range(args.threads[0]): # We also need to do the first file in the next group, or we will have
def run(self): """ run execution loop """ # ignore signals being used by main program signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) # function local variables mint = mp.cpu_count() # min/initial # of threshers maxt = self._ps['max'] # maximum # of threshers lSta = mp.Lock() # sta lock qT = mp.Queue() # thresher task queue rmap = {} # maps callsigns (vnic) to hwaddr fmap = {} # maps src (hwaddr) to frames smap = {} # maps src (hwaddr) to sensor params threshers = {} # pid->connection tst = 0 # timestamp of last thresher creation stop = False # poison pill has been received poisoned = False # poison pill has been sent (to threshers) df = rf = 0 # number dropped frames & read frames # create our frame buffer m = DIM_M * 5 # set max # of frames n = DIM_N # and keep n as is _ix = 0 # current 'pointer' cb = memoryview(mp.Array('B', m * n, lock=False)) # send sensor up notification, platform details & create threshers try: self._submitsensor(isots(), True) for _ in xrange(mint): try: (pid, send) = self._newthresher(qT, lSta, cb, smap) threshers[pid] = send except RuntimeError as e: self._cI.send((IYRI_WARN, 'collator', 'Thresher', e)) tst = time() self._submitplatform() except psql.Error as e: if e.pgcode == '23P01': err = "Last session closed incorrectly" else: err = "{0}->{1}".format(e.pgcode, e.pgerror) self._conn.rollback() self._cI.send((IYRI_ERR, 'collator', 'Thresher', err)) # notify iyri of startup parameters msg = "{0} threshers (min={1},max={2}) & a {3} x {4} internal buffer" msg = msg.format(format(len(threshers)), mint, maxt, m, n) self._cI.send((IYRI_INFO, 'collator', 'Startup', msg)) # execution loop - NOTE: if submitsensor failed, there will be no # threshers and we will immediately fall through the exection loop ins = [self._cI, self._icomms._reader] while mp.active_children(): # if we have stop, send out the poison pills once all frames have been # processed (or at least pulled of the buffer) if stop: if not [i for i in fmap if fmap[i]] and not poisoned: msg = "{0} outstanding frames".format(len(fmap)) for pid in threshers: try: threshers[pid].send((POISON, None, None)) except Exception as e: msg = "Error stopping Thresher-{0}: {1}".format( pid, e) self._cI.send( (IYRI_WARN, 'collator', 'Thresher', e)) poisoned = True # continue with processing try: # block longer if we havent gotten poison pill (rs, _, _) = select.select(ins, [], [], 0.5 if stop else 5) # convert tkns from iyri to icomms format # NOTE: all Thresh events (exluding THRESH_THRESH) will return # data as a tuple t = (string message,buffer index) where any # individual item in the tuple could be None for r in rs: if r == self._cI: (cs, ts, ev, d) = ('', '', self._cI.recv(), '') else: (cs, ts, ev, d) = self._icomms.get_nowait() if ev == POISON: stop = True elif ev == THRESH_THRESH: # thresher process up/down message if d is not None: # thresher up msg = "{0} up".format(cs) self._cI.send( (IYRI_INFO, 'collator', 'Thresher', msg)) else: # thresher down pid = int(cs.split('-')[1]) threshers[pid].close() del threshers[pid] msg = "{0} down".format(cs) self._cI.send( (IYRI_INFO, 'collator', 'Thresher', msg)) elif ev == THRESH_DQ: # frame has been pulled off buffer (_, idx) = d fmap[idx] = 0 elif ev == THRESH_WARN or ev == THRESH_ERR or ev == THRESH_DONE: # frame processing finished msg, idx = d try: del fmap[idx] except: pass # handle individual events below if ev == THRESH_WARN: self._cI.send( (IYRI_WARN, 'collator', 'Thresher', msg)) elif ev == THRESH_ERR: pid = int(cs.split('-')[1]) msg = "{0}->{1}".format(cs, msg[0]) self._cI.send( (IYRI_WARN, 'collator', 'Thresher', msg)) threshers[pid].send((POISON, None, None)) # make a new thresher (to replace this one) try: (pid, send) = self._newthresher(qT, lSta, cb, smap) threshers[pid] = send except RuntimeError as e: self._cI.send( (IYRI_WARN, 'Collator', Thresher, e)) # determine workload percentage if stop: continue if len(fmap) / (m * 1.0) <= 0.1 and len(threshers) > mint: msg = "Workload falling below 10%, removing a threseher" self._cI.send((IYRI_INFO, 'collator', 'load', msg)) pid = threshers.keys()[0] threshers[pid].send((POISON, None, None)) elif ev == GPS_GPSD: # gpsd up/down self._submitgpsd(ts, d) if d: self._cI.send( (IYRI_INFO, 'collator', 'GPSD', 'initiated')) elif ev == GPS_FLT: self._submitflt(ts, d) elif ev == RDO_RADIO: # up/down message from radio(s). if d is not None: # radio up (mac, role) = d['mac'], d['role'] # shouldn't happen but make sure: assert (cs not in rmap) # reinitialization assert (role in ['abad', 'shama']) # role is valid # notify iyri radio is up & add to maps msg = "{0}:{1} initiated".format(role, cs) self._cI.send( (IYRI_INFO, 'collator', 'Radio', msg)) rmap[cs] = mac smap[mac] = {'cb': None} if role == 'abad': smap[mac]['cb'] = self._ab else: smap[mac]['cb'] = self._sb # update threshers for pid in threshers: threshers[pid].send((COL_RDO, ts, [mac])) # submit radio & antenna(s) self._submitradio(ts, mac, d) for i in xrange(d['nA']): self._submitantenna( ts, { 'mac': mac, 'idx': i, 'type': d['type'][i], 'gain': d['gain'][i], 'loss': d['loss'][i], 'x': d['x'][i], 'y': d['y'][i], 'z': d['z'][i] }) else: # radio down msg = "{0} shutdown".format(cs) self._cI.send( (IYRI_INFO, 'collator', 'Radio', msg)) self._submitradio(ts, rmap[cs], None) # delete our maps (only delete tasks if its empty) del smap[rmap[cs]] del rmap[cs] elif ev == RDO_FAIL: msg = "Deleting radio {0}: {1}".format(cs, d) self._cI.send((IYRI_ERR, 'collator', 'Radio', msg)) self._submitradioevent(ts, [rmap[cs], 'fail', d]) # delete our maps del smap[rmap[cs]] del rmap[cs] elif RDO_SCAN <= ev <= RDO_STATE: if ev == RDO_SCAN: # need to format the channel list d = ','.join( ["{0}:{1}".format(c, w) for (c, w) in d]) self._cI.send((IYRI_INFO, cs, RDO_DESC[ev], d)) self._submitradioevent(ts, [rmap[cs], RDO_DESC[ev], d]) elif ev == RDO_FRAME: # get the index,length of the frame & src hwaddr (i, l) = d hw = rmap[cs] # about to overwrite a frame? if _ix in fmap and fmap[_ix] == 1: df += 1 print 'dropping' continue # move to our buffer, put on threshers queue & update index cb[(_ix * n):(_ix * n) + l] = smap[hw]['cb'][(i * n):(i * n) + l] fmap[_ix] = 1 qT.put((COL_FRAME, ts, (hw, _ix, l))) _ix = (_ix + 1) % m rf += 1 # if the buffer reaches 25% capacity, add a thresher # NOTE: consider all outstanding frames even those that # have been pulled of the buffer but are still being # processed) if len(fmap) / (m * 1.0) > 0.25 and len(threshers) < maxt: # create at most mint new threshers w/out adding # more than allowed. Give time for previously # created threshers to start working tsnow = time() if tsnow - tst < 3: continue x = min(mint, maxt - len(threshers)) msg = "Buffer at 25%, adding {0} threshers".format( x) self._cI.send((IYRI_INFO, 'collator', 'load', msg)) for _ in xrange(x): try: (pid, send) = self._newthresher( qT, lSta, cb, smap) threshers[pid] = send except RuntimeError as e: self._cI.send( (IYRI_WARN, 'Collator', Thresher, e)) tst = time() else: # unidentified event type, notify iyri msg = "unknown event. {0}->{1}".format(ev, cs) self._cI.send((IYRI_WARN, 'collator', 'icomms', msg)) except Empty: continue except (EOFError, IOError): pass except psql.OperationalError as e: # unrecoverable rmsg = "{0}: {1}".format(e.pgcode, e.pgerror) self._cI.send((IYRI_ERR, 'collator', 'DB', rmsg)) except psql.Error as e: # possible incorrect semantics/syntax self._conn.rollback() rmsg = "{0}: {1}".format(e.pgcode, e.pgerror) self._cI.send((IYRI_WARN, 'collator', 'SQL', rmsg)) except IndexError as e: rmsg = "Bad index {0} in event {1}".format(e, ev) self._cI.send((IYRI_ERR, 'collator', "Index", rmsg)) except KeyError as e: rmsg = "Bad key {0} in event {1}".format(e, ev) self._cI.send((IYRI_ERR, 'collator', 'Key', rmsg)) except select.error as e: # hide (4,'Interupted system call') errors if e[0] != 4: self._cI.send((IYRI_WARN, 'collator', 'select', e)) except Exception as e: # handle catchall error self._cI.send((IYRI_WARN, 'collator', type(e).__name__, e)) # clean up & shut down try: try: # notify Nidus of sensor - check if radio(s), gpsd closed out for cs in rmap: self._submitradio(isots(), rmap[cs], None) if self._gid: self._submitgpsd(isots(), None) if self._sid: self._submitsensor(isots(), False) except psql.Error: self._conn.rollback() self._cI.send( (IYRI_WARN, 'collator', 'shutdown', "DB is corrupted")) if self._curs: self._curs.close() if self._conn: self._conn.close() if rf or df: msg = "{0} read frames, {1} dropped frames".format(rf, df) self._cI.send((IYRI_DONE, 'collator', 'Complete', msg)) self._cI.close() except (EOFError, IOError): pass except Exception as e: if self._cI and not self._cI.closed: msg = "Failed to clean up: {0}".format(e) self._cI.send((IYRI_WARN, 'collator', 'shutdown', msg))
def main(): # step 1: # obtain host name, gpuID # create the corresponding folder hostName = socket.gethostname() foldName = str(hostName) + '-gpu' + str(gpu2Use) path = os.getcwd() targetDir = path + "/" + foldName if not os.path.exists(targetDir): try: os.mkdir(targetDir) except OSError: print("creation of %s failed!" % targetDir) MAXCORUN = int(args.maxCoRun) # max jobs per gpu RANDSEED = int(args.seed) gpuNum = 1 #logger.debug("MaxCoRun={}\trandseed={}\tsaveFile={}".format(MAXCORUN, RANDSEED, args.ofile)) #---------------------------------------------------------------------- # 1) application status table : 5 columns #---------------------------------------------------------------------- # # jobid gpu status starT endT # 0 0 1 1 2 # 1 1 1 1.3 2.4 # 2 0 0 - - # ... #---------------------------------------------------------------------- maxJobs = 10000 rows, cols = maxJobs, 5 # note: init with a large prefixed table d_arr = mp.Array(ctypes.c_double, rows * cols) arr = np.frombuffer(d_arr.get_obj()) AppStat = arr.reshape((rows, cols)) id2name = {} #---------------------------------------------------------------------- # #---------------------------------------------------------------------- #-------------------------------------------------------------------------- # input: app, app2dir_dd in app_info.py #-------------------------------------------------------------------------- if len(app) <> len(app2dir_dd): print "Error: app number wrong, check ../prepare/app_info.py!" sys.exit(1) # three random sequences #app_s1 = genRandSeq(app, seed=RANDSEED) # pi apps_num = len(app) logger.debug("Total GPU Applications = {}.".format(apps_num)) #-------------------------------------------------------------------------- # interference analysis : by run 2 apps concurrently #-------------------------------------------------------------------------- appPool = app for idx, app1 in enumerate(appPool): ofile = targetDir + '/' + app1 + ".npy" #print ofile # exclude: rodinia-heartwall, lonestar_sssp, dmr if idx >= 0: # NOTE: modify if program hangs #if idx == 0: # NOTE: modify if program hangs app_run2_dd = {} for a2_idx, app2 in enumerate(appPool): if a2_idx >= 0: print("idx = {}({}) : a2_idx={}({})".format( idx, app1, a2_idx, app2)) app1_runtime, app2_runtime = [], [] # run 3 times, get the fastest for ii in xrange(2): workers = [] # for mp processes # run 2 processes concurrently jobID = 0 id2name[jobID] = app1 process = Process(target=run_work, args=(jobID, AppStat, app2dir_dd[app1])) process.daemon = False workers.append(process) process.start() jobID = 1 id2name[jobID] = app2 process = Process(target=run_work, args=(jobID, AppStat, app2dir_dd[app2])) process.daemon = False workers.append(process) process.start() for p in workers: p.join() total_jobs = 2 appRT_dd = getGpuJobTiming(AppStat, total_jobs, id2name) #print appRT_dd app1_runtime.append(appRT_dd[app1]) app2_runtime.append(appRT_dd[app2]) #print app1_runtime app1_best = min(app1_runtime) # NOTE: only remember the app1 runtime when running with app2 app_run2_dd[app2] = app1_best
import talib as tb import matplotlib.pyplot as plt import matplotlib.dates as mdates import matplotlib.ticker as mticker from mpl_finance import candlestick_ohlc import datetime as dt from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg from matplotlib.figure import Figure import matplotlib.pyplot as plt from matplotlib.dates import DateFormatter, WeekdayLocator,\ DayLocator, MONDAY globalz = multiprocessing.Array('i', 32) varz = [] API_KEY = "" API_SECRET = "" class Application(Frame): def create_widgets(self): self.quit_button = Button(self) self.quit_button['text'] = 'STOP' self.quit_button['fg'] = 'red' self.quit_button['command'] = self.stop #self.quit self.quit_button.pack({'side': 'bottom'}) self.launch_button = Button(self)
def test_file_caching(self): """ max_batches and batch_size are directly correlated to test input data make sure gen_rand_data has the correct shape :return: """ from mock_config import mock_batches, mock_expected_malloc_requests, \ mock_class_index_map, mock_file_index_list mock_batches = copy.deepcopy(mock_batches) mock_expected_malloc_requests = copy.deepcopy( mock_expected_malloc_requests) mock_class_index_map = copy.deepcopy(mock_class_index_map) max_batches = len(mock_batches) batch_size = 500 read_size = 2 * batch_size bucket_length = 10 # in_queue = multiprocessing.Queue(maxsize=max_batches) # out_queue = multiprocessing.Queue(maxsize=max_batches) comm_driver = QueueCommDriver({'ctl': 10, 'in': 100, 'out': 100}) # building queue up with read requests for batch in mock_batches: success = comm_driver.write('in', batch) self.assertTrue(success) reader_id = 0 shared_data_pointer = range(bucket_length) for bucket in shared_data_pointer: data_sets = [] for request in mock_expected_malloc_requests: # reshape the requested shape to match the batch_size shape = (read_size, ) + request[1] shared_array_base = multiprocessing.Array(ctypes.c_double, np.prod(shape), lock=False) shared_array = np.ctypeslib.as_array(shared_array_base) shared_array = shared_array.reshape(shape) data_sets.append(shared_array) state = multiprocessing.Value('i', 0) generator_start_counter = multiprocessing.Value('i', 0) generator_end_counter = multiprocessing.Value('i', 0) shared_data_pointer[bucket] = [ state, data_sets, generator_start_counter, generator_end_counter ] reader = FileReader(worker_id=reader_id, comm_driver=comm_driver, shared_memory_pointer=shared_data_pointer, max_batches=max_batches, read_size=read_size, class_index_map=mock_class_index_map, file_index_list=mock_file_index_list, io_ctlr=IOController() # io_driver=H5DataIODriver({"cache_handles": True}) ) reader.read() out = list() start_time = datetime.datetime.utcnow() end_time = datetime.timedelta(seconds=10) + start_time while datetime.datetime.utcnow() < end_time and len( out) != max_batches: batch = comm_driver.read('out', block=False) if batch is not None: out.append(batch) self.assertEquals( len(out), max_batches, "test consumed {0} of {1} expected batches from the in_queue". format(len(out), max_batches)) target = 0 self.assertGreaterEqual(len(reader.io_ctlr('').file_handle_holder), 0) for handle in reader.io_ctlr('').file_handle_holder: # Here we try to close the file again, this will raise an Exception and thus means the caching cleaned up # successfully. if 0 then we closed everything successfully with the __exit__ function target += 1 try: handle.close() except Exception: target -= 1 self.assertEqual(target, 0)
# -*- coding: utf-8 -*- # @Time : 2018/4/26 17:00 # @Author : Inkky # @Email : [email protected] ''' ''' import multiprocessing as mp import time # 其中d和i参数用来设置数据类型的, # d表示一个双精浮点类型,i表示一个带符号的整型。 value1 = mp.Value('i', 0) value2 = mp.Value('d', 3.14) array = mp.Array('i', [1, 2, 3, 4]) # 这里的Array和numpy中的不同,它只能是一维的,不能是多维的。 def job(v, num,l): l.acquire() for _ in range(5): time.sleep(0.2) v.value += num # v.value获取共享变量值 print('\n',v.value, end="") l.release() def multicore(): l=mp.Lock() v = mp.Value('i', 0) # 定义共享变量 p1 = mp.Process(target=job, args=(v, 1,l)) p2 = mp.Process(target=job, args=(v, 3,l)) # 设定不同的number看如何抢夺内存 p1.start()
dic = mp.Manager().dict()#创建字典的时候不能用{}创建 lis = mp.Manager().list([1,2,3,4]) plist = [] for i in range(3): p = mp.Process(target = job3, args = (dic,lis,i)) plist.append(p) p.start() for process in plist: process.join() print(dic) print(lis) for process in plist: process.close() #共享内存shared memory: shared_value = mp.Value('i',-1)#i表示带符号的整型 shared_array = mp.Array('I',[1,2,3,4,5])#I表示不带符号的整型 plist = [] for i in range(5): p = mp.Process(target = job4, args = (shared_value,i)) plist.append(p) p.start() for process in plist: p.join() print(shared_value.value) for process in plist: p.close() #进程锁 lock = mp.Lock()#和线程锁使用方法一样 plist = [] v = mp.Value('i',0)
def __createArray(self, dimensions, dtype=np.float64, syncAccess=False): """ Create an array """ # convert to dtype type (in case short code) dtype = np.dtype(dtype) if (dtype == np.float32): ctype = ctypes.c_float elif (dtype == np.float64): ctype = ctypes.c_double elif (dtype == np.int32): ctype = ctypes.c_int32 elif (dtype == np.int64): ctype = ctypes.c_int64 elif (dtype == np.int16): ctype = ctypes.c_int16 elif (dtype == bool): ctype = ctypes.c_bool else: raise ValueError("Unsupported dtype") # This lock is only needed when creating the array, which is fast # It is not used when accessing data self.lock.acquire() # double size if necessary if (self.cnt >= len(self.sharedArrays)): self.sharedArrays = self.sharedArrays + [None] * len(self.sharedArrays) self.sharedArrayBases = self.sharedArrayBases + [None] * len(self.sharedArrayBases) # next handle self.__getNextFreeHdl() # create array in shared memory segment if (syncAccess): try: self.sharedArrayBases[self.cur] = multiprocessing.Array(ctype, int(np.prod(dimensions))) except: raise MemoryError("Failed to allocate memory for shared array. Note that $TMPDIR must have more space than allocated shared memory.") self.sharedArrays[self.cur] = np.frombuffer(self.sharedArrayBases[self.cur].get_obj(),dtype=dtype) else: try: self.sharedArrayBases[self.cur] = multiprocessing.RawArray(ctype, int(np.prod(dimensions))) except: raise MemoryError("Failed to allocate memory for shared array. Note that $TMPDIR must have more space than allocated shared memory.") self.sharedArrays[self.cur] = np.frombuffer(self.sharedArrayBases[self.cur],dtype=dtype) # do a reshape for correct dimensions # Returns a masked array containing the same data, but with a new shape. # The result is a view on the original array self.sharedArrays[self.cur] = self.sharedArrays[self.cur].reshape(dimensions) # update cnt self.cnt += 1 # Release the creation lock self.lock.release() # return handle to the shared memory numpy array return self.cur
| `'H'` | unsigned short | int | 2 | | `'i'` | signed int | int | 2 | | `'I'` | unsigned int | int | 2 | | `'l'` | signed long | int | 4 | | `'L'` | unsigned long | int | 4 | | `'q'` | signed long long | int | 8 | | `'Q'` | unsigned long long | int | 8 | | `'f'` | float | float | 4 | | `'d'` | double | float | 8 | 参考地址:https://docs.python.org/3/library/array.html 注意:Value和Array只适用于Process类。 ''' import multiprocessing def f(n, a): n.value = 3.14 a[0] = 5 if __name__ == '__main__': num = multiprocessing.Value('d', 0.0) arr = multiprocessing.Array('i', range(10)) p = multiprocessing.Process(target=f, args=(num, arr)) p.start() p.join() print(num.value) print(arr[:])
Shared Memory process 1 (Main process) ------> SHARED MEMORY <-------- process 2 (child process (calc_square)) ''' import multiprocessing def calc_square(numbers, result, v): v.value = 5.56 for index, num in enumerate(numbers): result[index] = num * num if __name__ == '__main__': number = [1, 2, 3] result = multiprocessing.Array( 'i', 3 ) #Result is the shared Array variable where "i" is to denote integer and 3 is the size v = multiprocessing.Value('d', 0.0) p = multiprocessing.Process(target=calc_square, args=(number, result, v)) p.start() p.join() print("The Result is : {}".format(result[:])) print("The Value is : {}".format(v.value))
def main(): #global app2dir #global app2cmd global app2metric global app2trace #-------------------------------------------------------------------------# # GPU Job Table #-------------------------------------------------------------------------# # jobid starT endT status # 0 1 2 # 1 1.3 2.4 # 2 - - # ... #---------------------------------------------------------------------- maxJobs = 10000 rows, cols = maxJobs, 4 # note: init with a large prefixed table d_arr = mp.Array(ctypes.c_double, rows * cols) arr = np.frombuffer(d_arr.get_obj()) GpuJobTable = arr.reshape((rows, cols)) #===================# # 1) read app info #===================# #app2dir = np.load('../07_sim_devid/similarity/app2dir_dd.npy').item() #app2cmd = np.load('../07_sim_devid/similarity/app2cmd_dd.npy').item() #app2metric = np.load('../07_sim_devid/similarity/app2metric_dd.npy').item() #app2trace = np.load('../07_sim_devid/perfmodel/app2trace_dd.npy').item() #print len(app2dir), len(app2cmd), len(app2metric), len(app2trace) #app_iobound_dd = np.load('./case_studies/app_iobound_dd.npy').item() #=========================================================================# # set up the launch order list #=========================================================================# appsList = get_appinfo('./prepare/app_info_79.bin') #print appsList app2dir_dd = {} for v in appsList: app2dir_dd[v[0]] = v[1] #print app2dir_dd #launch_list = ['cudasdk_concurrentKernels', 'poly_correlation'] #==========# # test 1 #==========# #launch_list = ['cudasdk_concurrentKernels', 'poly_3mm', 'poly_gemm', 'cudasdk_segmentationTreeThrust'] # sim #launch_list = ['cudasdk_concurrentKernels', 'cudasdk_segmentationTreeThrust', 'poly_3mm', 'poly_gemm'] # job size #launch_list = ['cudasdk_concurrentKernels', 'poly_3mm', 'cudasdk_segmentationTreeThrust', 'cudasdk_MCSingleAsianOptionP'] # sim #launch_list = ['cudasdk_concurrentKernels', 'cudasdk_segmentationTreeThrust', 'poly_3mm', 'cudasdk_MCSingleAsianOptionP'] # job size #launch_list = ['cudasdk_concurrentKernels', 'cudasdk_segmentationTreeThrust', 'cudasdk_MCSingleAsianOptionP', 'poly_3mm'] # job size + iobound #launch_list = ['cudasdk_concurrentKernels', 'poly_3mm', 'cudasdk_segmentationTreeThrust'] # sim #launch_list = ['cudasdk_concurrentKernels', 'cudasdk_segmentationTreeThrust', 'poly_3mm'] # job size #for x in launch_list: # print x, app2dir_dd[x] #==========# # test 2 #==========# #launch_list = ['shoc_lev1reduction', 'poly_3mm', 'poly_gemm', 'cudasdk_segmentationTreeThrust'] # sim #launch_list = ['shoc_lev1reduction', 'cudasdk_segmentationTreeThrust', 'poly_3mm', 'poly_gemm'] # jobsize + sim #==========# # test 3 #==========# #launch_list = ['cudasdk_scan', 'cudasdk_interval', 'cudasdk_MCEstimatePiQ', 'cudasdk_concurrentKernels'] # sim #launch_list = ['cudasdk_scan', 'cudasdk_concurrentKernels', 'cudasdk_MCEstimatePiQ','cudasdk_interval'] # threads + sim #launch_list = ['cudasdk_scan', 'cudasdk_concurrentKernels', 'cudasdk_interval', 'cudasdk_MCEstimatePiQ'] # threads + sim apps_num = len(launch_list) logger.debug("Total GPU Applications = {}.".format(apps_num)) appQueList = copy.deepcopy(launch_list) # application running queue workers = [] # for mp processes #==================================# # run the apps in the queue #==================================# MAXCORUN = 2 activeJobs = 0 jobID = -1 current_jobid_list = [] for i in xrange(apps_num): Dispatch = False if activeJobs < MAXCORUN: Dispatch = True #print("iter {} dispatch={}".format(i, Dispatch)) if Dispatch: activeJobs += 1 jobID += 1 current_jobid_list.append(jobID) appName = appQueList[i] process = Process(target=run_work, args=(jobID, GpuJobTable, appName, app2dir_dd)) process.daemon = False #logger.debug("Start %r", process) workers.append(process) process.start() else: # spin while True: break_loop = False current_running_jobs = 0 jobs2del = [] for jid in current_jobid_list: if GpuJobTable[jid, 3] == 1: # check the status, if one is done jobs2del.append(jid) break_loop = True if break_loop: activeJobs -= 1 # update if jobs2del: for id2del in jobs2del: del_idx = current_jobid_list.index(id2del) del current_jobid_list[del_idx] break #------------------------------------ # after spinning, schedule the work #------------------------------------ #print("iter {}: activeJobs = {}".format(i, activeJobs)) activeJobs += 1 jobID += 1 current_jobid_list.append(jobID) #print("iter {}: activeJobs = {}".format(i, activeJobs)) appName = appQueList[i] process = Process(target=run_work, args=(jobID, GpuJobTable, appName, app2dir_dd)) process.daemon = False #logger.debug("Start %r", process) workers.append(process) process.start() #=========================================================================# # end of running all the jobs #=========================================================================# for p in workers: p.join() total_jobs = jobID + 1 PrintGpuJobTable(GpuJobTable, total_jobs) if total_jobs <> apps_num: logger.debug("[Warning] job number doesn't match.")
def init_shm_ndarray(shape,dtype=np.float64): '''Initialize a shared memory array of the supplied shape and dtype''' shm = mp.Array(_numpy_to_ctypes[np.dtype(dtype)],int(np.prod(shape)),lock=False) return shm
print datetime.datetime.now().time(), '- Allocating memory...' syn_images_num = len( [line.strip() for line in open(g_syn_images_train_val_split, 'r')]) env = lmdb.open(g_pool5_lmdb, readonly=True) datum_array = np.empty(0) with env.begin() as txn: cursor = txn.cursor() for key, value in cursor: datum = caffe_pb2.Datum() datum.ParseFromString(value) datum_array = caffe.io.datum_to_array(datum) break print datetime.datetime.now().time( ), '- the shape of the datum is:', datum_array.shape shared_array_base = multiprocessing.Array(ctypes.c_double, syn_images_num * datum_array.size) shared_array = np.ctypeslib.as_array(shared_array_base.get_obj()) shared_array = shared_array.reshape([syn_images_num] + list(datum_array.shape)) def image_pair_to_string(pair_info, def_param=shared_array): array_1_id = pair_info[0] array_2_id = pair_info[1] similar_or_not = pair_info[2] array_1 = shared_array[array_1_id, :] array_2 = shared_array[array_2_id, :] array = np.concatenate((array_1, array_2), axis=0) datum = caffe.io.array_to_datum(array) datum.label = similar_or_not return datum.SerializeToString()
loop = asyncio.get_event_loop() loop.create_task( selfbot.start( 'NzE2NzE2NTIzMjc1ODEyOTE0.XvqgaQ.6qyKKBfFOcLdAL2i0LAl1FWUZUw', bot=False)) loop.run_forever() if __name__ == '__main__': # running bot and selfbot in separate OS processes # shared event for embed update update_event = multiprocessing.Event() # shared array with answer results answer_scores = multiprocessing.Array(typecode_or_type='i', size_or_initializer=3) p_bot = multiprocessing.Process(target=bot_with_cyclic_update_process, args=(update_event, answer_scores)) p_selfbot = multiprocessing.Process(target=selfbot_process, args=(update_event, answer_scores)) p_bot.start() p_selfbot.start() p_bot.join() p_selfbot.join()
def solveNbodiesVerletMultiprocessing(masses, init_pos, init_vel, dt, iterations): def solveForOneBody(q, q_out, init_vel, shared_pos, body, events1, events2): dimension = len(shared_pos) // len(masses) my_cur_pos = np.array(shared_pos[body * dimension:(body + 1) * dimension]) cur_acceleration = acceleration_no_np(body, masses, shared_pos) result = np.empty((iterations, dimension * 2)) result[0, :] = np.concatenate( (my_cur_pos, init_vel[body * dimension:(body + 1) * dimension])) for j in np.arange(iterations - 1) + 1: result[j, :dimension] = \ (my_cur_pos + result[j-1, dimension:] * dt + 0.5 * cur_acceleration * dt**2) q.put([body, result[j, :dimension]]) events1[body].set() if body == 0: for i in range(len(masses)): events1[i].wait() events1[i].clear() for i in range(len(masses)): tmp = q.get() shared_pos[tmp[0] * dimension:(tmp[0] + 1) * dimension] = tmp[1] for i in range(len(masses)): events2[i].set() else: events2[body].wait() events2[body].clear() my_cur_pos = np.array(shared_pos[body * dimension:(body + 1) * dimension]) next_acceleration = acceleration_no_np(body, masses, shared_pos) result[j, dimension:] = \ (result[j-1, dimension:] + 0.5 * (cur_acceleration + next_acceleration) * dt) cur_acceleration = next_acceleration q_out.put([body, result]) if __name__ == '__main__': times = np.arange(iterations) * dt posvel = np.empty((times.size, init_pos.size * 2)) shared_pos = mp.Array('d', init_pos) events1 = [] events2 = [] for body in masses: events1.append(mp.Event()) events2.append(mp.Event()) events1[-1].clear() events2[-1].clear() q = mp.Queue() q_out = mp.Queue() processes = [] for body in range(masses.size): processes.append( mp.Process(target=solveForOneBody, args=(q, q_out, init_vel, shared_pos, body, events1, events2))) processes[-1].start() dim = init_pos.size // len(masses) for i in range(len(masses)): tmp = q_out.get() posvel[:, tmp[0] * dim:(tmp[0] + 1) * dim] = tmp[1][:, :dim] posvel[:, init_pos.size + tmp[0] * dim:init_pos.size + (tmp[0] + 1) * dim] = tmp[1][:, dim:] for process in processes: process.join() return posvel, times
for i in range(10): array[i]=random.randrange(0,100) # randomly pick the number in range 1~100. return def sorting_function(array) : i=9 for i in range(9,0,-1): j=0 for j in range(i): time.sleep(0.01) # in order to slow down. if array[j] > array[j+1] : # swap the bigger one and smaller one. tmp = array[j+1] array[j+1]=array[j] array[j]=tmp return def print_array(array, pid): print"[%d] --> " %pid,; # print the process's id. for i in range(10): print"%d " %array[i],; # print sorted numbers. if __name__ =='__main__' : array=multiprocessing.Array('i',[0]*10) print('############Start#############') lock = multiprocessing.Lock() # avoid the "race condition" lock the thread. for i in range(10) : p = multiprocessing.Process(target=sorting_thread, args=(array, i,lock)) p.start() # start the work. p.join() # stop the work. print('#############End###############')
counter = 0 print('Parallel Programming Assignment 2') print('*********************************') def increase_counter(number): global primes global prime_object if prime_object.is_prime(number): primes[number-1] = True running = True table = [] while running: primes = multiprocessing.Array('b', primes_till) primes_till = eval(input("Enter max number: ")) processors_count = multiprocessing.cpu_count() chosen_processors = eval(input(f'Chose number of processors from 1 to {processors_count}: ')) if chosen_processors not in range(1, processors_count + 1): print(f'You chose {chosen_processors}, ' f'that is not included in the expected range! We are proceeding with the maximum power.') chosen_processors = processors_count print(f'Calculating with {chosen_processors} processors...') pool = multiprocessing.Pool(processes=chosen_processors) start_time = datetime.datetime.now() pool.map(increase_counter, range(1, primes_till + 1)) prime_count = primes[:].count(1)
np.clip(f0_map_list.reshape(n_hop, n_band).T, 1e-5, 1.0)), cmap='plasma', interpolation='nearest', aspect='auto', origin='lower') pl.plot(np.argmax(f0_map_list, axis=1)) pl.show() else: print("* Spawn data process...") ctx = mp.get_context('spawn') memory_pool = [] data_queue = mp.Queue(n_data_process * 2) memory_queue = mp.Queue(n_data_process * 2) command_queue = mp.Queue(n_data_process) for i in range(n_data_process * 2): memory_pool.append((mp.Array(ctypes.c_float, batch_size * n_band), mp.Array(ctypes.c_float, batch_size * n_feature))) memory_queue.put(i) del i for _ in range(n_data_process): ctx.Process(target=data_fetch_process_main, args=(data_queue, memory_queue, command_queue, memory_pool)).start() def fetch_mem_data(): i_memory = data_queue.get() (memory_f0, memory_feature) = memory_pool[i_memory] with memory_f0.get_lock(): with memory_feature.get_lock(): out = np.frombuffer(memory_f0.get_obj(),