def mcmc(data,
uncert=None,
func=None,
indparams=[],
params=None,
pmin=None,
pmax=None,
stepsize=None,
prior=None,
priorlow=None,
priorup=None,
nchains=10,
nproc=None,
nsamples=10,
walk='demc',
wlike=False,
leastsq=True,
lm=False,
chisqscale=False,
grtest=True,
grbreak=0.01,
grnmin=0.5,
burnin=0,
thinning=1,
fgamma=1.0,
fepsilon=0.0,
hsize=1,
kickoff='normal',
plots=False,
ioff=False,
showbp=True,
savefile=None,
savemodel=None,
resume=False,
rms=False,
log=None,
pnames=None,
texnames=None,
full_output=False,
chireturn=False,
parname=None):
"""
This beautiful piece of code runs a Markov-chain Monte Carlo algorithm.
Parameters
----------
data: 1D ndarray
Dependent data fitted by func.
uncert: 1D ndarray
Uncertainty of data.
func: callable or string-iterable
The callable function that models data as:
model = func(params, *indparams)
Or an iterable (list, tuple, or ndarray) of 3 strings:
(funcname, modulename, path)
that specify the function name, function module, and module path.
If the module is already in the python-path scope, path can be omitted.
indparams: tuple
Additional arguments required by func.
params: 1D or 2D ndarray
Set of initial fitting parameters for func. If 2D, of shape
(nparams, nchains), it is assumed that it is one set for each chain.
pmin: 1D ndarray
Lower boundaries of the posteriors.
pmax: 1D ndarray
Upper boundaries of the posteriors.
stepsize: 1D ndarray
Proposal jump scale. If a values is 0, keep the parameter fixed.
Negative values indicate a shared parameter (See Note 1).
prior: 1D ndarray
Parameter prior distribution means (See Note 2).
priorlow: 1D ndarray
Lower prior uncertainty values (See Note 2).
priorup: 1D ndarray
Upper prior uncertainty values (See Note 2).
nchains: Scalar
Number of simultaneous chains to run.
nproc: Integer
The number of processors for the MCMC chains (consider that MC3 uses
one other CPU for the central hub).
nsamples: Scalar
Total number of samples.
walk: String
Random walk algorithm:
- 'mrw': Metropolis random walk.
- 'demc': Differential Evolution Markov chain.
- 'snooker': DEMC-z with snooker update.
wlike: Boolean
If True, calculate the likelihood in a wavelet-base. This requires
three additional parameters (See Note 3).
leastsq: Boolean
Perform a least-square minimization before the MCMC run.
lm: Boolean
If True use the Levenberg-Marquardt algorithm for the optimization.
If False, use the Trust Region Reflective algorithm.
chisqscale: Boolean
Scale the data uncertainties such that the reduced chi-squared = 1.
grtest: Boolean
Run Gelman & Rubin test.
grbreak: Float
Gelman-Rubin convergence threshold to stop the MCMC (I'd suggest
grbreak ~ 1.001--1.005). Do not break if grbreak=0.0 (default).
grnmin: Integer or float
Minimum number of samples required for grbreak to stop the MCMC.
If grnmin > 1: grnmin sets the minimum required number of samples.
If 0 < grnmin < 1: grnmin sets the minimum required nsamples fraction.
burnin: Scalar
Burned-in (discarded) number of iterations at the beginning
of the chains.
thinning: Integer
Thinning factor of the chains (use every thinning-th iteration) used
in the GR test and plots.
fgamma: Float
Proposals jump scale factor for DEMC's gamma.
The code computes: gamma = fgamma * 2.38 / sqrt(2*Nfree)
fepsilon: Float
Jump scale factor for DEMC's support distribution.
The code computes: e = fepsilon * Normal(0, stepsize)
hsize: Integer
Number of initial samples per chain.
kickoff: String
Flag to indicate how to start the chains:
'normal' for normal distribution around initial guess, or
'uniform' for uniform distribution withing the given boundaries.
plots: Bool
If True plot parameter traces, pairwise-posteriors, and posterior
histograms.
ioff: Bool
If True, set plt.ioff(), i.e., do not display figures on screen.
showbp: Bool
If True, show best-fitting values in histogram and pairwise plots.
savefile: String
If not None, filename to store allparams and other MCMC results.
savemodel: String
If not None, filename to store the values of the evaluated function
(with np.save).
resume: Boolean
If True resume a previous run.
rms: Boolean
If True, calculate the RMS of the residuals: data - bestmodel.
log: String or FILE pointer
Filename or File object to write log.
pnames: 1D string iterable
List of parameter names (including fixed and shared parameters)
to display on output screen and figures. See also texnames.
Screen output trims up to the 11th character.
If not defined, default to texnames.
texnames: 1D string iterable
Parameter names for figures, which may use latex syntax.
If not defined, default to pnames.
full_output: Bool
If True, return the full posterior sample, including the burned-in
iterations.
chireturn: Bool
If True, include chi-squared statistics in the return.
parname: 1D string ndarray
Deprecated, use pnames.
Returns
-------
bestp: 1D ndarray
Array of the best-fitting parameters (including fixed and shared).
CRlo: 1D ndarray
The lower boundary of the marginal 68%-highest posterior density
(the credible region) for each parameter, with respect to bestp.
CRhi: 1D ndarray
The upper boundary of the marginal 68%-highest posterior density
(the credible region) for each parameter, with respect to bestp.
stdp: 1D ndarray
Array of the best-fitting parameter uncertainties, calculated as the
standard deviation of the marginalized, thinned, burned-in posterior.
posterior: 2D float ndarray
An array of shape (Nfreepars, Nsamples) with the thinned MCMC posterior
distribution of the fitting parameters (excluding fixed and shared).
If full_output is True, the posterior includes the burnin samples.
Zchain: 1D integer ndarray
Index of the chain for each sample in posterior. M0 samples have chain
index of -1.
chiout: 4-elements tuple
Tuple containing the best-fit chi-square, reduced chi-square, scale
factor to enforce redchisq=1, and the Bayesian information
criterion (BIC).
Note: Returned only if chireturn=True.
Notes
-----
1.- To set one parameter equal to another, set its stepsize to the
negative index in params (Starting the count from 1); e.g.: to set
the second parameter equal to the first one, do: stepsize[1] = -1.
2.- If any of the fitting parameters has a prior estimate, e.g.,
param[i] = p0 +up/-low,
with up and low the 1sigma uncertainties. This information can be
considered in the MCMC run by setting:
prior[i] = p0
priorup[i] = up
priorlow[i] = low
All three: prior, priorup, and priorlow must be set and, furthermore,
priorup and priorlow must be > 0 to be considered as prior.
3.- FINDME: WAVELET LIKELIHOOD
Examples
--------
>>> # See https://github.com/pcubillos/MCcubed/tree/master/examples
Uncredited developers
---------------------
Kevin Stevenson (UCF)
"""
if ioff:
plt.ioff()
# Open log file if input is a filename:
if isinstance(log, str):
log = mu.Log(log, append=resume)
closelog = True
else:
closelog = False
if log is None:
log = mu.Log(logname=None)
if parname is not None:
log.error("'parname' argument is deprecated. Use 'pnames' instead.")
if resume:
log.msg("\n\n{:s}\n{:s} Resuming previous MCMC run.\n\n".format(
log.sep, log.sep))
log.msg(
"\n{:s}\n"
" Multi-core Markov-chain Monte Carlo (MC3).\n"
" Version {:d}.{:d}.{:d}.\n"
" Copyright (c) 2015-{:d} Patricio Cubillos and collaborators.\n"
" MC3 is open-source software under the MIT license (see LICENSE).\n"
"{:s}\n\n".format(log.sep, ver.MC3_VER, ver.MC3_MIN, ver.MC3_REV,
date.today().year, log.sep))
# Import the model function:
if type(func) in [list, tuple, np.ndarray]:
if len(func) == 3:
sys.path.append(func[2])
fmodule = importlib.import_module(func[1])
func = getattr(fmodule, func[0])
elif not callable(func):
log.error("'func' must be either a callable or an iterable of strings "
"with the model function, file, and path names.")
if nproc is None: # Default to Nproc = Nchains:
nproc = nchains
# Cap the number of processors:
if nproc >= mpr.cpu_count():
log.warning(
"The number of requested CPUs ({:d}) is >= than the number "
"of available CPUs ({:d}). Enforced nproc to {:d}.".format(
nproc, mpr.cpu_count(),
mpr.cpu_count() - 1))
nproc = mpr.cpu_count() - 1
nparams = len(params) # Number of model params
ndata = len(data) # Number of data values
# Set default uncertainties:
if uncert is None:
uncert = np.ones(ndata)
# Setup array of parameter names:
if pnames is None and texnames is not None:
pnames = texnames
elif pnames is not None and texnames is None:
texnames = pnames
elif pnames is None and texnames is None:
pnames = texnames = mu.default_parnames(nparams)
pnames = np.asarray(pnames)
texnames = np.asarray(texnames)
# Set uncert as shared-memory object:
sm_uncert = mpr.Array(ctypes.c_double, uncert)
uncert = np.ctypeslib.as_array(sm_uncert.get_obj())
# Set default boundaries:
if pmin is None:
pmin = np.tile(-np.inf, nparams)
if pmax is None:
pmax = np.tile(np.inf, nparams)
# Set default stepsize:
if stepsize is None:
stepsize = 0.1 * np.abs(params)
stepsize = np.asarray(stepsize)
# Set prior parameter indices:
if (prior is None) or (priorup is None) or (priorlow is None):
prior = priorup = priorlow = np.zeros(nparams) # Zero arrays
# Check that initial values lie within the boundaries:
if (np.any(np.asarray(params) < pmin)
or np.any(np.asarray(params) > pmax)):
pout = ""
for (pname, par, minp, maxp) in zip(pnames, params, pmin, pmax):
if par < minp:
pout += "\n{:11s} {: 12.5e} < {: 12.5e}".format(
pname[:11], minp, par)
if par > maxp:
pout += "\n{:26s} {: 12.5e} > {: 12.5e}".format(
pname[:11], par, maxp)
log.error("Some initial-guess values are out of bounds:\n"
"Param name pmin value pmax\n"
"----------- ------------ ------------ ------------"
"{:s}".format(pout))
nfree = int(np.sum(stepsize > 0)) # Number of free parameters
ifree = np.where(stepsize > 0)[0] # Free parameter indices
ishare = np.where(stepsize < 0)[0] # Shared parameter indices
# Initial number of samples:
M0 = hsize * nchains
# Number of Z samples per chain:
nZchain = int(np.ceil(nsamples / nchains / thinning))
# Number of iterations per chain:
niter = nZchain * thinning
# Total number of Z samples (initial + chains):
Zlen = M0 + nZchain * nchains
# Initialize shared-memory free params array:
sm_freepars = mpr.Array(ctypes.c_double, nchains * nfree)
freepars = np.ctypeslib.as_array(sm_freepars.get_obj())
freepars = freepars.reshape((nchains, nfree))
# Get lowest chi-square and best fitting parameters:
bestchisq = mpr.Value(ctypes.c_double, np.inf)
sm_bestp = mpr.Array(ctypes.c_double, np.copy(params))
bestp = np.ctypeslib.as_array(sm_bestp.get_obj())
# There seems to be a strange behavior with np.ctypeslib.as_array()
# when the argument is a single-element array. In this case, the
# returned value is a two-dimensional array, instead of 1D. The
# following line fixes(?) that behavior:
if np.ndim(bestp) > 1:
bestp = bestp.flatten()
if not resume and niter < burnin:
log.error("The number of burned-in samples ({:d}) is greater than "
"the number of iterations per chain ({:d}).".format(
burnin, niter))
# Check that output path exists:
if savefile is not None:
fpath, fname = os.path.split(os.path.realpath(savefile))
if not os.path.exists(fpath):
log.warning("Output folder path: '{:s}' does not exist. "
"Creating new folder.".format(fpath))
os.makedirs(fpath)
# Intermediate steps to run GR test and print progress report:
intsteps = (nZchain * nchains) / 10
report = intsteps
# Initial size of posterior (prior to this MCMC sample):
size0 = M0
if resume:
oldrun = np.load(savefile)
Zold = oldrun["Z"]
Zlen_old = np.shape(Zold)[0] # Previous MCMC
Zchain_old = oldrun["Zchain"]
# Redefine Zlen to include the previous runs:
Zlen = Zlen_old + nZchain * nchains
size0 = Zlen_old
# Allocate arrays with variables:
numaccept = mpr.Value(ctypes.c_int, 0)
outbounds = mpr.Array(ctypes.c_int, nfree) # Out of bounds proposals
#if savemodel is not None:
# allmodel = np.zeros((nchains, ndata, niter)) # Fit model
# Z array with the chains history:
sm_Z = mpr.Array(ctypes.c_double, Zlen * nfree)
Z = np.ctypeslib.as_array(sm_Z.get_obj())
Z = Z.reshape((Zlen, nfree))
# Chi-square value of Z:
sm_Zchisq = mpr.Array(ctypes.c_double, Zlen)
Zchisq = np.ctypeslib.as_array(sm_Zchisq.get_obj())
# Chain index for given state in the Z array:
sm_Zchain = mpr.Array(ctypes.c_int, -np.ones(Zlen, np.int))
Zchain = np.ctypeslib.as_array(sm_Zchain.get_obj())
# Current number of samples in the Z array:
Zsize = mpr.Value(ctypes.c_int, M0)
# Burned samples in the Z array per chain:
Zburn = int(burnin / thinning)
# Include values from previous run:
if resume:
Z[0:Zlen_old, :] = Zold
Zchisq[0:Zlen_old] = oldrun["Zchisq"]
Zchain[0:Zlen_old] = oldrun["Zchain"]
# Redefine Zsize:
Zsize.value = Zlen_old
numaccept.value = int(oldrun["numaccept"])
# Set GR N-min:
if grnmin > 0 and grnmin < 1: # As a fraction:
grnmin = int(grnmin * (Zlen - M0 - Zburn * nchains))
elif grnmin > 1: # As the number of iterations:
pass
else:
log.error(
"Invalid 'grnmin' argument (minimum number of samples to stop"
"the MCMC under GR convergence), must either be grnmin > 1"
"to set the minimum number of samples, or 0 < grnmin < 1"
"to set the fraction of samples required to evaluate.")
# Add these to compare grnmin to Zsize (which also include them):
grnmin += int(M0 + Zburn * nchains)
# Current length of each chain:
sm_chainsize = mpr.Array(ctypes.c_int, np.tile(hsize, nchains))
chainsize = np.ctypeslib.as_array(sm_chainsize.get_obj())
# Number of chains per processor:
ncpp = np.tile(int(nchains / nproc), nproc)
ncpp[0:nchains % nproc] += 1
# Launch Chains:
pipes = []
chains = []
for i in range(nproc):
p = mpr.Pipe()
pipes.append(p[0])
chains.append(
ch.Chain(func, indparams, p[1], data, uncert, params, freepars,
stepsize, pmin, pmax, walk, wlike, prior, priorlow,
priorup, thinning, fgamma, fepsilon, Z, Zsize, Zchisq,
Zchain, M0, numaccept, outbounds, ncpp[i], chainsize,
bestp, bestchisq, i, nproc))
if resume:
# Set bestp and bestchisq:
bestp = oldrun["bestp"]
bestchisq.value = oldrun["bestchisq"]
for c in range(nchains):
chainsize[c] = np.sum(Zchain_old == c)
chifactor = float(oldrun['chifactor'])
uncert *= chifactor
else:
fitpars = np.asarray(params)
# Least-squares minimization:
if leastsq:
fitchisq, fitbestp, dummy, dummy = mf.modelfit(
fitpars, func, data, uncert, indparams, stepsize, pmin, pmax,
prior, priorlow, priorup, lm)
# Store best-fitting parameters:
bestp[ifree] = np.copy(fitbestp[ifree])
# Store minimum chisq:
bestchisq.value = fitchisq
log.msg(
"Least-squares best-fitting parameters:\n {:s}\n\n".format(
str(fitbestp[ifree])),
si=2)
# Populate the M0 initial samples of Z:
Z[0] = np.clip(bestp[ifree], pmin[ifree], pmax[ifree])
for j in range(nfree):
idx = ifree[j]
if kickoff == "normal": # Start with a normal distribution
vals = np.random.normal(params[idx], stepsize[idx], M0 - 1)
# Stay within pmin and pmax boundaries:
vals[np.where(vals < pmin[idx])] = pmin[idx]
vals[np.where(vals > pmax[idx])] = pmax[idx]
Z[1:M0, j] = vals
elif kickoff == "uniform": # Start with a uniform distribution
Z[1:M0, j] = np.random.uniform(pmin[idx], pmax[idx], M0 - 1)
# Evaluate models for initial sample of Z:
for i in range(M0):
fitpars[ifree] = Z[i]
# Update shared parameters:
for s in ishare:
fitpars[s] = fitpars[-int(stepsize[s]) - 1]
Zchisq[i] = chains[0].eval_model(fitpars, ret="chisq")
# Best-fitting values (so far):
Zibest = np.argmin(Zchisq[0:M0])
bestchisq.value = Zchisq[Zibest]
bestp[ifree] = np.copy(Z[Zibest])
# FINDME: think what to do with this:
#models = np.zeros((nchains, ndata))
# Scale data-uncertainties such that reduced chisq = 1:
chifactor = 1.0
if chisqscale:
chifactor = np.sqrt(bestchisq.value / (ndata - nfree))
uncert *= chifactor
# Re-calculate chisq with the new uncertainties:
for i in range(M0):
fitpars[ifree] = Z[i]
for s in ishare:
fitpars[s] = fitpars[-int(stepsize[s]) - 1]
Zchisq[i] = chains[0].eval_model(fitpars, ret="chisq")
# Re-calculate best-fitting parameters with new uncertainties:
if leastsq:
fitchisq, fitbestp, dummy, dummy = mf.modelfit(
fitpars, func, data, uncert, indparams, stepsize, pmin,
pmax, prior, priorlow, priorup, lm)
bestp[ifree] = np.copy(fitbestp[ifree])
bestchisq.value = fitchisq
log.msg(
"Least-squares best-fitting parameters (rescaled chisq):\n"
" {:s}\n\n".format(str(fitbestp[ifree])),
si=2)
#if savemodel is not None:
# allmodel[:,:,0] = models
# ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
# Start loop:
print("Yippee Ki Yay Monte Carlo!")
log.msg("Start MCMC chains ({:s})".format(time.ctime()))
for chain in chains:
chain.start()
bit = bool(1) # Dummy variable to send through pipe for DEMC
while True:
# Proposal jump:
if walk == "demc":
# Send and receive bit for synchronization:
for pipe in pipes:
pipe.send(bit)
for pipe in pipes:
b = pipe.recv()
# Print intermediate info:
if (Zsize.value - size0 >= report) or (Zsize.value == Zlen):
report += intsteps
log.progressbar((Zsize.value + 1.0 - size0) / (nZchain * nchains))
log.msg("Out-of-bound Trials:\n{:s}".format(
str(np.asarray(outbounds[:]))),
width=80)
log.msg("Best Parameters: (chisq={:.4f})\n{:s}".format(
bestchisq.value, str(bestp[ifree])),
width=80)
# Save current results:
if savefile is not None:
np.savez(savefile, Z=Z, Zchain=Zchain)
#if savemodel is not None:
# np.save(savemodel, allmodel)
# Gelman-Rubin statistics:
if grtest and np.all(chainsize > (Zburn + hsize)):
psrf = gr.gelmanrubin(Z, Zchain, Zburn)
log.msg("Gelman-Rubin statistics for free parameters:\n{:s}".
format(str(psrf)),
width=80)
if np.all(psrf < 1.01):
log.msg(
"All parameters have converged to within 1% of unity.")
if (grbreak > 0.0 and np.all(psrf < grbreak)
and Zsize.value > grnmin):
with Zsize.get_lock():
Zsize.value = Zlen
log.msg(
"\nAll parameters satisfy the GR convergence threshold "
"of {:g}, stopping the MCMC.".format(grbreak))
break
if Zsize.value == Zlen:
break
for chain in chains: # Make sure to terminate the subprocesses
chain.terminate()
#if savemodel is not None:
# modelstack = allmodel[0,:,burnin:]
# for c in range(1, nchains):
# modelstack = np.hstack((modelstack, allmodel[c, :, burnin:]))
# Print out Summary:
log.msg("\nFin, MCMC Summary:\n------------------")
# Evaluate model for best fitting parameters:
fitpars = np.asarray(params)
fitpars[ifree] = np.copy(bestp[ifree])
for s in ishare:
fitpars[s] = fitpars[-int(stepsize[s]) - 1]
bestmodel = chains[0].eval_model(fitpars)
# Truncate sample (if necessary):
Ztotal = M0 + np.sum(Zchain >= 0)
Zchain = Zchain[:Ztotal]
Zchisq = Zchisq[:Ztotal]
Z = Z[:Ztotal]
# Get indices for samples considered in final analysis:
good = np.zeros(len(Zchain), bool)
for c in range(nchains):
good[np.where(Zchain == c)[0][Zburn:]] = True
# Values accepted for posterior stats:
posterior = Z[good]
pchain = Zchain[good]
# Sort the posterior by chain:
zsort = np.lexsort([pchain])
posterior = posterior[zsort]
pchain = pchain[zsort]
# Get some stats:
nsample = np.sum(Zchain >= 0) * thinning # Total samples run
nZsample = len(posterior) # Valid samples (after thinning and burning)
BIC = bestchisq.value + nfree * np.log(ndata)
if ndata > nfree:
redchisq = bestchisq.value / (ndata - nfree)
else:
redchisq = np.nan
sdr = np.std(bestmodel - data)
fmt = len(str(nsample))
log.msg("Total number of samples: {:{}d}".format(nsample, fmt),
indent=2)
log.msg("Number of parallel chains: {:{}d}".format(nchains, fmt),
indent=2)
log.msg("Average iterations per chain: {:{}d}".format(
nsample // nchains, fmt),
indent=2)
log.msg("Burned-in iterations per chain: {:{}d}".format(burnin, fmt),
indent=2)
log.msg("Thinning factor: {:{}d}".format(thinning, fmt),
indent=2)
log.msg("MCMC sample size (thinned, burned): {:{}d}".format(nZsample, fmt),
indent=2)
log.msg("Acceptance rate: {:.2f}%\n".format(numaccept.value * 100.0 /
nsample),
indent=2)
# Compute the credible region for each parameter:
CRlo = np.zeros(nparams)
CRhi = np.zeros(nparams)
pdf = []
xpdf = []
for i in range(nfree):
PDF, Xpdf, HPDmin = mu.credregion(posterior[:, i])
pdf.append(PDF)
xpdf.append(Xpdf)
CRlo[ifree[i]] = np.amin(Xpdf[PDF > HPDmin])
CRhi[ifree[i]] = np.amax(Xpdf[PDF > HPDmin])
# CR relative to the best-fitting value:
CRlo[ifree] -= bestp[ifree]
CRhi[ifree] -= bestp[ifree]
# Get the mean and standard deviation from the posterior:
meanp = np.zeros(nparams, np.double) # Parameters mean
stdp = np.zeros(nparams, np.double) # Parameter standard deviation
meanp[ifree] = np.mean(posterior, axis=0)
stdp[ifree] = np.std(posterior, axis=0)
for s in ishare:
bestp[s] = bestp[-int(stepsize[s]) - 1]
meanp[s] = meanp[-int(stepsize[s]) - 1]
stdp[s] = stdp[-int(stepsize[s]) - 1]
CRlo[s] = CRlo[-int(stepsize[s]) - 1]
CRhi[s] = CRhi[-int(stepsize[s]) - 1]
log.msg(
"\nParam name Best fit Lo HPD CR Hi HPD CR Mean Std dev S/N"
"\n----------- ----------------------------------- ---------------------- ---------",
width=80)
for i in range(nparams):
snr = "{:.1f}".format(np.abs(bestp[i]) / stdp[i])
mean = "{: 11.4e}".format(meanp[i])
lo = "{: 11.4e}".format(CRlo[i])
hi = "{: 11.4e}".format(CRhi[i])
if i in ifree: # Free-fitting value
pass
elif i in ishare: # Shared value
snr = "[share{:02d}]".format(-int(stepsize[i]))
else: # Fixed value
snr = "[fixed]"
mean = "{: 11.4e}".format(bestp[i])
log.msg(
"{:<11s} {:11.4e} {:>11s} {:>11s} {:>11s} {:10.4e} {:>9s}".format(
pnames[i][0:11], bestp[i], lo, hi, mean, stdp[i], snr),
width=160)
if leastsq and bestchisq.value - fitchisq < -3e-8:
np.set_printoptions(precision=8)
log.warning(
"MCMC found a better fit than the minimizer:\n"
"MCMC best-fitting parameters: (chisq={:.8g})\n{:s}\n"
"Minimizer best-fitting parameters: (chisq={:.8g})\n"
"{:s}".format(bestchisq.value, str(bestp[ifree]), fitchisq,
str(fitbestp[ifree])))
fmt = len("{:.4f}".format(BIC)) # Length of string formatting
log.msg(" ")
if chisqscale:
log.msg("sqrt(reduced chi-squared) factor: {:{}.4f}".format(
chifactor, fmt),
indent=2)
log.msg("Best-parameter's chi-squared: {:{}.4f}".format(
bestchisq.value, fmt),
indent=2)
log.msg("Bayesian Information Criterion: {:{}.4f}".format(BIC, fmt),
indent=2)
log.msg("Reduced chi-squared: {:{}.4f}".format(redchisq, fmt),
indent=2)
log.msg("Standard deviation of residuals: {:.6g}\n".format(sdr), indent=2)
# Save definitive results:
if savefile is not None:
np.savez(savefile,
bestp=bestp,
Z=Z,
Zchain=Zchain,
Zchisq=Zchisq,
CRlo=CRlo,
CRhi=CRhi,
stdp=stdp,
meanp=meanp,
bestchisq=bestchisq.value,
redchisq=redchisq,
chifactor=chifactor,
BIC=BIC,
sdr=sdr,
numaccept=numaccept.value)
#if savemodel is not None:
# np.save(savemodel, allmodel)
if rms:
RMS, RMSlo, RMShi, stderr, bs = ta.binrms(bestmodel - data)
if plots:
print("Plotting figures.")
# Extract filename from savefile:
if savefile is not None:
if savefile.rfind(".") == -1:
fname = savefile
else:
# Cut out file extention.
fname = savefile[:savefile.rfind(".")]
else:
fname = "MCMC"
# Include bestp in posterior plots:
if showbp:
bestfreepars = bestp[ifree]
else:
bestfreepars = None
# Trace plot:
mp.trace(Z,
Zchain=Zchain,
burnin=Zburn,
pnames=texnames[ifree],
savefile=fname + "_trace.png")
# Pairwise posteriors:
mp.pairwise(posterior,
pnames=texnames[ifree],
bestp=bestfreepars,
savefile=fname + "_pairwise.png")
# Histograms:
mp.histogram(posterior,
pnames=texnames[ifree],
bestp=bestfreepars,
savefile=fname + "_posterior.png",
percentile=0.683,
pdf=pdf,
xpdf=xpdf)
# RMS vs bin size:
if rms:
mp.RMS(bs,
RMS,
stderr,
RMSlo,
RMShi,
binstep=len(bs) // 500 + 1,
savefile=fname + "_RMS.png")
# Sort of guessing that indparams[0] is the X array for data as in y=y(x):
if (indparams != [] and isinstance(indparams[0],
(list, tuple, np.ndarray))
and np.size(indparams[0]) == ndata):
try:
mp.modelfit(data,
uncert,
indparams[0],
bestmodel,
savefile=fname + "_model.png")
except:
pass
# Close the log file if necessary:
if closelog:
log.close()
# Build the output tuple:
output = bestp, CRlo, CRhi, stdp
if full_output:
output += (Z, Zchain)
else:
output += (posterior, pchain)
chiout = (bestchisq.value, redchisq, chifactor, BIC)
if chireturn:
output += (chiout, )
return output
def main():
parser = argparse.ArgumentParser(
description="making feature file argsurations.")
parser.add_argument("--log_dir",
required=True,
type=str,
help="directory to save the log")
parser.add_argument("--wav_dir",
default=None,
help="directory of input wavfile")
parser.add_argument("--hdf5dir",
default=None,
help="directory to save hdf5")
parser.add_argument("--conf_path",
default=None,
type=str,
help="Path to the config file")
parser.add_argument("--fs", default=None, type=int, help="Sample rate.")
parser.add_argument("--shiftms", default=None, type=int, help="Shift ms.")
parser.add_argument("--mcep_dim",
default=None,
type=int,
help="Dimension of mel cepstrum")
parser.add_argument("--fftl", default=None, type=int, help="FFT length")
parser.add_argument("--n_jobs",
default=1,
type=int,
help="number of parallel jobs")
args = parser.parse_args()
# set log level
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
filename=args.log_dir + "/feature_extract.log")
logging.getLogger().addHandler(logging.StreamHandler())
# mcep_alpha
if args.fs == 16000:
mcep_alpha = 0.41
elif args.fs == 22050:
mcep_alpha = 0.455
elif args.fs == 24000:
mcep_alpha = 0.466
elif args.fs == 44100:
mcep_alpha = 0.544
elif args.fs == 48000:
mcep_alpha = 0.554
else:
raise ValueError('sampling rate should be one of \
16000, 22050, 24000, 44100, 48000')
# read list
file_list = sorted(glob.glob(os.path.join(args.wav_dir, "*.wav")))
# load config
with open(args.conf_path, 'r', encoding='utf-8') as f:
t = f.readlines()
minf0 = int(t[0])
maxf0 = int(t[1])
thpow = float(t[2])
def feature_extract(wav_list, arr):
n_sample = 0
n_frame = 0
max_frame = 0
count = 1
coeff = np.array([-0.5, 0.5, 0.0])
for wav_name in wav_list:
# load wavfile and apply low cut filter
fs, x = read_wav(wav_name, cutoff=70)
n_sample += x.shape[0]
logging.info(wav_name + " " + str(x.shape[0]) + " " +
str(n_sample) + " " + str(count))
# check sampling frequency
if not fs == args.fs:
logging.debug("ERROR: sampling frequency is not matched.")
sys.exit(1)
hdf5name = args.hdf5dir + "/" + os.path.basename(wav_name).replace(
".wav", ".h5")
# extimate f0 and ap
time_axis, f0, spc, ap = analyze_range(x,
fs=args.fs,
minf0=minf0,
maxf0=maxf0,
fperiod=args.shiftms,
fftl=args.fftl)
write_hdf5(hdf5name, '/ap', ap)
write_hdf5(hdf5name, "/f0", f0)
# convert to continuous f0 and low-pass filter
uv, cont_f0 = convert_continuos_f0(np.array(f0))
cont_f0_lpf = low_pass_filter(cont_f0,
int(1.0 / (args.shiftms * 0.001)),
cutoff=20)
cont_f0_lpf = np.expand_dims(cont_f0_lpf, axis=-1)
uv = np.expand_dims(uv, axis=-1)
write_hdf5(hdf5name, "/lcf0", np.log(cont_f0_lpf))
write_hdf5(hdf5name, "/uv", uv)
# extimate codeap
codeap = pw.code_aperiodicity(ap, args.fs)
if codeap.ndim == 1:
# when fs == 16000
codeap = np.expand_dims(codeap, axis=-1)
write_hdf5(hdf5name, "/codeap", codeap)
# mcep
mcep = ps.sp2mc(spc, args.mcep_dim, mcep_alpha)
write_hdf5(hdf5name, "/mcep", mcep)
# divie list
file_lists = np.array_split(file_list, args.n_jobs)
file_lists = [f_list.tolist() for f_list in file_lists]
# multi processing
processes = []
arr = mp.Array('d', 3)
for f in file_lists:
p = mp.Process(target=feature_extract, args=(f, arr))
p.start()
processes.append(p)
# wait for all process
for p in processes:
p.join()
#!/usr/bin/python
import multiprocessing as mp
import numpy as np
# choose number of processes
num_procs = 4
num_things_per_proc = 10
# create a list of things from which to operate
mpVals = mp.Array('i', num_procs * num_things_per_proc, lock=False)
# populate the array in serial, cause this is cheap
for i in range(num_procs * num_things_per_proc):
mpVals[i] = i
# create a lock object for atomicity
array_lock = mp.Lock()
# define work that each process does atomically
def print_segment(id, segment_start_index, segment_end_index):
#with array_lock:
for i in range(segment_start_index, segment_end_index):
print("Process: " + str(id) + ", value: " + str(mpVals[i]))
# create an empty processes array
processes = []
if __name__ == "__main__":
job_list = []
MC = 2 * 10**4
step = 0.5
start = 0
end = 2
EbNs = np.arange(start, (end + step), step)
ArraySize = EbNs.size
Error_shape = (ArraySize, MC)
ErrorUncoded_shared = multiprocessing.Array('i',
Error_shape[0] *
Error_shape[1],
lock=True)
ErrorUncoded = np.frombuffer(ErrorUncoded_shared.get_obj(),
dtype=np.int32).reshape(Error_shape)
np.copyto(ErrorUncoded, np.zeros(Error_shape, dtype=int))
ErrorMSA_shared = multiprocessing.Array('i',
Error_shape[0] * Error_shape[1],
lock=True)
ErrorMSA = np.frombuffer(ErrorMSA_shared.get_obj(),
dtype=np.int32).reshape(Error_shape)
np.copyto(ErrorMSA, np.zeros(Error_shape, dtype=int))
IterBF = np.zeros(Error_shape, dtype=int)
IterSBF = np.zeros(Error_shape, dtype=int)
def __init__(self, _graph, _machines, order_dict, *args):
# initialize the tk.Tk class portion
tk.Tk.__init__(self, _graph['gui_title'])
tk.Tk.wm_title(self, _graph['gui_title'])
tk.Tk.columnconfigure(self, 0, weight=1)
tk.Tk.rowconfigure(self, 0, weight=1)
# dictionaries of measurement settings, graph settings and results
self.settings = args
self.graph = _graph
self.machines = _machines
self.results = {
'x_data': mp.Array('d', 1500),
'y_data': mp.Array('d', 1500),
# for cases where Gaussmeter data is recorded
'x2_data': mp.Array('d', 1500),
# counts the number of data points to graph
'counter': mp.Value('i', 0),
# int value of the percentage done the total measurement loop is
'progress': mp.Value('i', 0),
'time': mp.Value('i', 0)
}
self.queue = mp.Queue()
# directions for measure module import (since can't be pickled) and loop direction commands
self.order = order_dict
# make sure that there is a Measurements folder that exists
if os.path.isdir(
os.path.expanduser('~\\Documents') + '\\Measurements'):
pass
else:
os.mkdir(os.path.expanduser('~\\Documents') + '\\Measurements')
# save parameters
self.directory = os.path.expanduser('~\\Documents\\Measurements')
# loop and file name initial values, later changed to widgets
self.file_name = 'test_file'
self.loop = ['low-high', 'zero-zero', 'high-low', 'standard']
# multiprocessing.Process controls, for quit and pause functions
self.measure_process = '' # initialized when measurement button is pressed
# flag which will pause process at the top of the measurement loop
self.pause_flag = mp.Event()
# flag that kills the measurement process after safely releasing resources
self.quit_flag = mp.Event()
# initialize the plot figure and axes necessary for drawing
self.fig = plt.Figure(figsize=(6, 5), dpi=100)
self.ax = self.fig.add_subplot(111)
# Main frame that will hold the frames for each specific group of widgets
base_frame = ttk.Frame(self)
base_frame.grid(row=0, column=0, sticky='nsew')
base_frame.columnconfigure(0, weight=1)
base_frame.rowconfigure(0, weight=1)
# The four frames that house each different section of widgets
plot_frame = ttk.Frame(base_frame, borderwidth=5)
settings_frame = ttk.Frame(base_frame, borderwidth=5)
info_frame = ttk.Frame(base_frame, borderwidth=5)
buttons_frame = ttk.Frame(base_frame, borderwidth=5)
progress_frame = ttk.Frame(base_frame, borderwidth=5)
# Use grid to place all frames on the base_frame
plot_frame.grid(row=0, column=0, sticky='nsew')
settings_frame.grid(row=0, column=1, sticky='nsew')
progress_frame.grid(row=1, column=0, sticky='nsew')
info_frame.grid(row=2, column=0, sticky='nsew')
buttons_frame.grid(row=1, column=1, rowspan=2, sticky='nsew')
# creates and grids the listbox and scroll bar
self.datalog = tk.Listbox(info_frame, height=5)
self.y_scroll = ttk.Scrollbar(info_frame, command=self.datalog.yview)
self.datalog['yscrollcommand'] = self.y_scroll.set
self.datalog.grid(column=0, row=0, sticky='nsew')
self.y_scroll.grid(column=1, row=0, sticky='ns')
# test that title is indeed string
try:
if type(self.graph['graph_title'] + "\n" +
self.graph['fixed_param_1'] + " " +
self.graph['fixed_param_2']) is str:
self.ax.set_title(self.graph['graph_title'] + "\n" +
self.graph['fixed_param_1'] + " " +
self.graph['fixed_param_2'])
else:
raise Exception(
'Graph Title or Fixed Parameters not a string.')
except Exception as err:
print('An exception occured for graph titles: ' + str(err))
self.datalog.insert(
'end', 'An exception occured for graph titles: ' + str(err))
self.datalog.see('end')
# test that x and y axis labels are indeed strings
try:
if type(self.graph['x_title'] + self.graph['y_title']) is str:
self.ax.set_xlabel(self.graph['x_title'])
self.ax.set_ylabel(self.graph['y_title'])
else:
raise Exception('X or Y Axis label not a string.')
except Exception as err2:
print('An exception occured for axis labels: ' + str(err2))
self.datalog.insert(
'end', 'An exception occured for axis labels: ' + str(err2))
self.datalog.see('end')
# Draw the plot on canvas
plot_canvas = FigureCanvasTkAgg(self.fig, plot_frame)
plot_canvas.draw()
plot_canvas.get_tk_widget().grid(row=0,
column=0,
pady=0,
padx=0,
sticky='nsew')
# variable that updates in animation, letting user how long left there is
self.time_var = tk.StringVar()
self.time_var.set('Time Remaining')
# creates and grids a progress bar and estimated time remaining label
self.progress_bar = ttk.Progressbar(progress_frame,
orient='horizontal',
length=500,
mode='determinate',
maximum=100,
value=0)
self.progress_bar.grid(column=0, row=0, sticky='nsew')
self.time_label = ttk.Label(progress_frame, textvar=self.time_var)
self.time_label.grid(column=1, row=0, sticky='nsew')
# make test settings
for dic in self.settings:
try:
if type(dic) is dict:
# for each dictionary in the list, build items
self.make_form(settings_frame, dic)
else:
raise Exception('List element (' + str(dic) +
') must be a dictionary')
except Exception as err3:
print(
'An exception occured with the list of measurement parameters: '
+ str(err3))
self.datalog.insert(
'end',
'An exception occured with the list of measurement parameters: '
+ str(err3))
self.datalog.see('end')
# make loop parameters buttons/labels/entries (the misc commands)
misc_lf = ttk.LabelFrame(settings_frame, text='Measurement Options')
misc_lf.grid(ipadx=2, ipady=5, sticky='nsew')
file_name_lbl = ttk.Label(misc_lf,
width=20,
text='File Name:',
anchor='w')
file_name_ent = ttk.Entry(misc_lf, width=15)
file_name_ent.insert(0, self.file_name)
# update dictionary value to the entry
self.file_name = file_name_ent
file_name_lbl.grid(row=0, column=0, sticky='nsew')
file_name_ent.grid(row=0, column=1, sticky='nsew')
loop_lbl = ttk.Label(misc_lf, width=20, text='Loop Type:', anchor='w')
loop_box = ttk.Combobox(misc_lf,
width=10,
state='readonly',
values=self.loop)
loop_box.set('low-high')
self.loop = loop_box # update dictionary value to the box
loop_lbl.grid(row=1, column=0, sticky='nsew')
loop_box.grid(row=1, column=1, sticky='nsew')
# make the buttons
self.measure_button = ttk.Button(buttons_frame,
text='Measure',
command=lambda: self.measure_method())
self.output_button = ttk.Button(buttons_frame,
text='Output',
command=lambda: self.output_method())
self.dir_button = ttk.Button(
buttons_frame,
text='Change Directory',
command=lambda: self.change_directory_method())
self.stop_button = ttk.Button(buttons_frame,
text='Stop',
command=lambda: self.stop_method(),
state='disabled')
self.quit_button = ttk.Button(buttons_frame,
text='Quit',
command=lambda: self.quit_method())
# grid buttons
self.measure_button.grid(row=0, column=0, sticky='nsew')
self.output_button.grid(row=1, column=0, sticky='nsew')
self.dir_button.grid(row=2, column=0, sticky='nsew')
self.stop_button.grid(row=3, column=0, sticky='nsew')
self.quit_button.grid(row=4, column=0, sticky='nsew')
# weights columns/rows that are of weight 1, frames with specific weights are written explicitly
self.weight_col(buttons_frame)
self.weight_col(settings_frame)
self.weight_col(plot_frame)
self.weight_row(buttons_frame)
self.weight_row(plot_frame)
self.weight_row(info_frame)
# necessary to keep the scroll bar tiny
info_frame.columnconfigure(0, weight=1)
info_frame.columnconfigure(1, weight=0)
# necessary to keep the label tiny
progress_frame.columnconfigure(0, weight=1)
progress_frame.columnconfigure(1, weight=0)
# test that all instruments are on and the gpib addresses are correct
self.check_resources()
def test_make_file_index(self):
from mock_config import mock_batches, mock_expected_malloc_requests, \
mock_class_index_map, mock_file_index_malloc, mock_file_index_list
mock_batches = copy.deepcopy(mock_batches)
mock_class_index_map = copy.deepcopy(mock_class_index_map)
mock_file_index_malloc = copy.deepcopy(mock_file_index_malloc)
mock_expected_malloc_requests = copy.deepcopy(
mock_expected_malloc_requests)
mock_expected_malloc_requests.extend(mock_file_index_malloc)
max_batches = len(mock_batches)
batch_size = 500
read_size = 2 * batch_size
bucket_length = 10
comm_driver = QueueCommDriver({'ctl': 10, 'in': 100, 'out': 100})
# building queue up with read requests
for batch in mock_batches:
success = comm_driver.write('in', batch)
self.assertTrue(success)
reader_id = 0
shared_data_pointer = range(bucket_length)
for bucket in shared_data_pointer:
data_sets = []
for request in mock_expected_malloc_requests:
# TODO requests are not ordered!!!
# TODO not sure it matters as long as its consistent
# reshape the requested shape to match the batch_size
shape = (read_size, ) + request[1]
shared_array_base = multiprocessing.Array(ctypes.c_double,
np.prod(shape),
lock=False)
shared_array = np.ctypeslib.as_array(shared_array_base)
shared_array = shared_array.reshape(shape)
data_sets.append(shared_array)
state = multiprocessing.Value('i', 0)
generator_start_counter = multiprocessing.Value('i', 0)
generator_end_counter = multiprocessing.Value('i', 0)
shared_data_pointer[bucket] = [
state, data_sets, generator_start_counter,
generator_end_counter
]
reader = FileReader(worker_id=reader_id,
comm_driver=comm_driver,
shared_memory_pointer=shared_data_pointer,
max_batches=max_batches,
read_size=read_size,
class_index_map=mock_class_index_map,
make_one_hot=True,
make_class_index=True,
make_file_index=True,
file_index_list=mock_file_index_list,
io_ctlr=IOController()
# io_driver=H5DataIODriver()
)
reader.read()
out = list()
start_time = datetime.datetime.utcnow()
end_time = datetime.timedelta(seconds=10) + start_time
while datetime.datetime.utcnow() < end_time and len(
out) != max_batches:
batch = comm_driver.read('out', block=False)
if batch is not None:
out.append(batch)
self.assertEquals(
len(out), max_batches,
"test consumed {0} of {1} expected batches from the in_queue".
format(len(out), max_batches))
num_proc = 6
threads_arr = {}
for i in range(num_proc):
threads_arr[i] = []
pi_id = 0
for i in range(len(y_pred)):
if y_pred[i] == 0:
for j in way_dict[i]:
if y_pred[j] == 1:
t_id = j if find else i
threads_arr[pi_id % num_proc].append(list(X[t_id]))
pi_id += 1
result_dis_workers = multiprocessing.Array('d', num_proc)
result_ids_workers = multiprocessing.Array('i', num_proc)
for i in range(num_proc):
result_dis_workers[i] = 0
ps = []
for pi in range(num_proc):
ps.append(
multiprocessing.Process(target=calc_distance,
args=(pi, threads[0], threads_arr[pi],
result_dis_workers,
result_ids_workers)))
for p in ps:
p.start()
"""
created by dyx on 2021/7/16.
"""
import multiprocessing
def func(num):
num[2] = 999 # 子进程改变数组，主进程跟着改变
if __name__ == '__main__':
num = multiprocessing.Array("i", [1, 2, 3, 4, 5]) # 主进程和子进程共享这个数组
print(num[:])
p = multiprocessing.Process(target=func, args=(num, ))
p.start()
p.join()
print(num[:])
def calculo_raices(lista_numeros, lista_raices, suma_total):
for i, numero in enumerate(lista_numeros):
lista_raices[i] = math.sqrt(numero)
suma_total.value = sum(lista_raices)
print('ID del proceso en ejecución: {}'.format(os.getpid()))
print("Las raices de las lista son: {}".format(lista_raices[:]))
print("La suma de las raices de la lista es: {} \n".format(
suma_total.value))
if __name__ == "__main__":
lista_numeros = [2, 4, 9, 16, 25, 36, 49, 64, 81, 100]
lista_raices = multiprocessing.Array('d', 10)
suma_total = multiprocessing.Value('d')
p1 = multiprocessing.Process(target=calculo_raices,
args=(
lista_numeros,
lista_raices,
suma_total,
))
p2 = multiprocessing.Process(target=calculo_raices,
args=(
lista_numeros,
lista_raices,
suma_total,
))
def denseampcor(self, slcImage1=None, slcImage2=None):
if not (slcImage1 == None):
self.slcImage1 = slcImage1
if (self.slcImage1 == None):
logger.error("Error. reference slc image not set.")
raise Exception
if not (slcImage2 == None):
self.slcImage2 = slcImage2
if (self.slcImage2 == None):
logger.error("Error. secondary slc image not set.")
raise Exception
self.fileLength1 = self.slcImage1.getLength()
self.lineLength1 = self.slcImage1.getWidth()
self.fileLength2 = self.slcImage2.getLength()
self.lineLength2 = self.slcImage2.getWidth()
####Run checks
self.checkTypes()
self.checkWindows()
####Actual processing
coarseAcross = self.acrossGrossOffset
coarseDown = self.downGrossOffset
xMargin = 2 * self.searchWindowSizeWidth + self.windowSizeWidth
yMargin = 2 * self.searchWindowSizeHeight + self.windowSizeHeight
#####Set image limits for search
offAc = max(self.margin, -coarseAcross) + xMargin
if offAc % self.skipSampleAcross != 0:
leftlim = offAc
offAc = self.skipSampleAcross * (
1 + int(offAc / self.skipSampleAcross)) - self.pixLocOffAc
while offAc < leftlim:
offAc += self.skipSampleAcross
offDn = max(self.margin, -coarseDown) + yMargin
if offDn % self.skipSampleDown != 0:
toplim = offDn
offDn = self.skipSampleDown * (
1 + int(offDn / self.skipSampleDown)) - self.pixLocOffDn
while offDn < toplim:
offDn += self.skipSampleDown
offAcmax = int(coarseAcross + (
(self.rangeSpacing1 / self.rangeSpacing2) - 1) * self.lineLength1)
lastAc = int(
min(self.lineLength1, self.lineLength2 - offAcmax) - xMargin - 1 -
self.margin)
offDnmax = int(coarseDown +
((self.prf2 / self.prf1) - 1) * self.fileLength1)
lastDn = int(
min(self.fileLength1, self.fileLength2 - offDnmax) - yMargin - 1 -
self.margin)
self.gridLocAcross = range(offAc + self.pixLocOffAc,
lastAc - self.pixLocOffAc,
self.skipSampleAcross)
self.gridLocDown = range(offDn + self.pixLocOffDn,
lastDn - self.pixLocOffDn,
self.skipSampleDown)
startAc, endAc = offAc, self.gridLocAcross[-1] - self.pixLocOffAc
self.numLocationAcross = int((endAc - startAc) /
self.skipSampleAcross + 1)
self.numLocationDown = len(self.gridLocDown)
self.offsetCols, self.offsetLines = self.numLocationAcross, self.numLocationDown
print('Pixels: ', self.lineLength1, self.lineLength2)
print('Lines: ', self.fileLength1, self.fileLength2)
print('Wins : ', self.windowSizeWidth, self.windowSizeHeight)
print('Srch: ', self.searchWindowSizeWidth,
self.searchWindowSizeHeight)
#####Create shared memory objects
numlen = self.numLocationAcross * self.numLocationDown
self.locationDown = np.frombuffer(mp.Array('i', numlen).get_obj(),
dtype='i')
self.locationDownOffset = np.frombuffer(mp.Array('f',
numlen).get_obj(),
dtype='f')
self.locationAcross = np.frombuffer(mp.Array('i', numlen).get_obj(),
dtype='i')
self.locationAcrossOffset = np.frombuffer(mp.Array('f',
numlen).get_obj(),
dtype='f')
self.snr = np.frombuffer(mp.Array('f', numlen).get_obj(), dtype='f')
self.cov1 = np.frombuffer(mp.Array('f', numlen).get_obj(), dtype='f')
self.cov2 = np.frombuffer(mp.Array('f', numlen).get_obj(), dtype='f')
self.cov3 = np.frombuffer(mp.Array('f', numlen).get_obj(), dtype='f')
self.locationDownOffset[:] = -10000.0
self.locationAcrossOffset[:] = -10000.0
self.snr[:] = 0.0
self.cov1[:] = 999.0
self.cov2[:] = 999.0
self.cov3[:] = 999.0
###run ampcor on parallel processes
threads = []
nominal_load = self.numLocationDown // self.numberThreads
flat_indices = np.arange(numlen).reshape(
(self.numLocationDown, self.numLocationAcross))
ofmt = 'Thread %d: %7d%7d%7d%7d%7d%7d'
for thrd in range(self.numberThreads):
# Determine location down grid indices for thread
if thrd == self.numberThreads - 1:
proc_num_grid = self.numLocationDown - thrd * nominal_load
else:
proc_num_grid = nominal_load
istart = thrd * nominal_load
iend = istart + proc_num_grid
# Compute corresponding global line/down indices
proc_loc_down = self.gridLocDown[istart:iend]
startDown, endDown = proc_loc_down[0], proc_loc_down[-1]
numDown = int((endDown - startDown) // self.skipSampleDown + 1)
# Get flattened grid indices
firstind = flat_indices[istart:iend, :].ravel()[0]
lastind = flat_indices[istart:iend, :].ravel()[-1]
print(
ofmt %
(thrd, firstind, lastind, startAc, endAc, startDown, endDown))
# Launch job
args = (startAc, endAc, startDown, endDown, self.numLocationAcross,
numDown, firstind, lastind)
threads.append(mp.Process(target=self._run_ampcor, args=args))
threads[-1].start()
# Wait for all threads to finish
for thread in threads:
thread.join()
self.firstSampAc, self.firstSampDown = self.locationAcross[
0], self.locationDown[0]
self.lastSampAc, self.lastSampDown = self.locationAcross[
-1], self.locationDown[-1]
#### Scale images (default is 1.0 to keep as pixel)
self.locationDownOffset *= self.scaling_factor
self.locationAcrossOffset *= self.scaling_factor
self.write_slantrange_images()
def main():
hashes = []
if len(sys.argv) <= 1:
print("Usage: python3 main.py <hashfile> <outfile>")
return
elif len(sys.argv) == 3:
hashfile = sys.argv[1]
outfile = sys.argv[2]
try:
hashes = load_hashes(hashfile)
except IOError:
print("Error: Unable to read hash file (%s)" % hashfile)
return
try:
outStream = open(outfile, "w+")
outStream.close()
except IOError:
print("Error: Unable to create output file (%s)" % outfile)
return
lines = 0
with open(wordlist, "r") as file:
for line in file:
lines += 1
hashLock = multiprocessing.Lock()
hashStatus = multiprocessing.Array('i', [0] * len(hashes))
print("Wordlist length: " + str(lines))
print("Available CPUs: %d" % CPUCOUNT)
print("Creating %d processes..." % CPUCOUNT)
start = time.time()
procs = []
for i in range(1, CPUCOUNT + 1):
p = multiprocessing.Process(target=crack,
args=(hashes, hashStatus, hashLock,
CPUCOUNT, i, outfile))
p.start()
procs.append(p)
for p in procs:
p.join()
end = time.time()
print("All processes have completed.")
# Aggregate data and print summary info
outHashes = []
data = None
with open(outfile, 'r') as file:
data = file.readlines()
count = 0
for line in data:
line = line.split(":")
h = Hash(line[0], count, "")
h.password = line[1].replace('\n', '')
outHashes.append(h)
count += 1
hashCount = len(hashStatus)
crackedCount = 0
for h in hashStatus:
if h == 1:
crackedCount += 1
for h in hashes:
if hashStatus[h.id] == 1:
h.cracked = True
for o in outHashes:
if h.value == o.value:
o.cracked = h.cracked
o.username = h.username
break
print("------------------Summary------------------")
print("Cracked %d/%d hashes in %.5f seconds" % (crackedCount, hashCount,
(end - start)))
for o in outHashes:
if o.cracked:
print("%s:%s (%s)" % (o.value, o.password, o.username))
else:
new_model = False # prevent `main` from being set to `eval32` at loading
save_d = np.load(sdir + save_nm, allow_pickle=True).item()
for key in save_vars + state_vars + training_ex_vars:
if key == 'save_nm':
continue
exec('%s = save_d["%s"]' % (key, key))
print save_nm
################### shared memory variables
###### self play from `eval` model used for training `eval32`:
s_board = mp.Array('h', board.ravel(
)) # shape: (BUFFER_SZ, gv.n_rows, gv.n_cols, gv.n_input_channels)
s_winner = mp.Array(
'b', winner.ravel()) # (N_BATCH_SETS_TOTAL, N_TURNS, 2, gv.BATCH_SZ)
s_tree_probs = mp.Array('f', tree_probs.ravel()) # (BUFFER_SZ, gv.map_szt)
# indices, counters, & flags
buffer_loc = mp.Value('i', buffer_loc) # index into above ^ training vars
weights_changed = mp.Value('i', 0) # 0 = no change, 1 = changed
batch_sets_created = mp.Value('i', batch_sets_created)
batch_sets_created_total = mp.Value('i', batch_sets_created_total)
batch_set = mp.Value('i', batch_set)
# evaluation (`eval` vs `main` benchmark testing to see when to update `main` to the current `eval32` backprop weights)
scope_next = mp.Value(
'i', 0
) # alternates between 0,1 during model evaluation to dictate if `eval` or `main` starts 1st
# Get the maximum size for all images.
max_w = 0
max_h = 0
for f in args.filespec:
im = Image.open(f)
(w, h) = im.size
max_w = max(max_w, w)
max_h = max(max_h, h)
# Create an array to store the numeric totals for each pixel.
# This is a special multithreading thing--this Array will allow
# us to share memory between the threads. Results are "rolled up"
# into a one-dimenisonal array.
totals = multiprocessing.Array('i', [0 for _ in range(3 * max_h * max_w)])
last_im = None
cutoff_squared = args.cutoff[0] * args.cutoff[0] # Square for square norm.
# Process the antiphoto.
# This can take a long time so we can employ multiple threads.
num_files = len(args.filespec)
if num_files - 1 < args.threads[0]:
args.threads[0] = num_files - 1
num_files_per_thread = math.ceil(num_files / args.threads[0])
for t in range(args.threads[0]):
# We also need to do the first file in the next group, or we will have
def run(self):
""" run execution loop """
# ignore signals being used by main program
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
# function local variables
mint = mp.cpu_count() # min/initial # of threshers
maxt = self._ps['max'] # maximum # of threshers
lSta = mp.Lock() # sta lock
qT = mp.Queue() # thresher task queue
rmap = {} # maps callsigns (vnic) to hwaddr
fmap = {} # maps src (hwaddr) to frames
smap = {} # maps src (hwaddr) to sensor params
threshers = {} # pid->connection
tst = 0 # timestamp of last thresher creation
stop = False # poison pill has been received
poisoned = False # poison pill has been sent (to threshers)
df = rf = 0 # number dropped frames & read frames
# create our frame buffer
m = DIM_M * 5 # set max # of frames
n = DIM_N # and keep n as is
_ix = 0 # current 'pointer'
cb = memoryview(mp.Array('B', m * n, lock=False))
# send sensor up notification, platform details & create threshers
try:
self._submitsensor(isots(), True)
for _ in xrange(mint):
try:
(pid, send) = self._newthresher(qT, lSta, cb, smap)
threshers[pid] = send
except RuntimeError as e:
self._cI.send((IYRI_WARN, 'collator', 'Thresher', e))
tst = time()
self._submitplatform()
except psql.Error as e:
if e.pgcode == '23P01': err = "Last session closed incorrectly"
else: err = "{0}->{1}".format(e.pgcode, e.pgerror)
self._conn.rollback()
self._cI.send((IYRI_ERR, 'collator', 'Thresher', err))
# notify iyri of startup parameters
msg = "{0} threshers (min={1},max={2}) & a {3} x {4} internal buffer"
msg = msg.format(format(len(threshers)), mint, maxt, m, n)
self._cI.send((IYRI_INFO, 'collator', 'Startup', msg))
# execution loop - NOTE: if submitsensor failed, there will be no
# threshers and we will immediately fall through the exection loop
ins = [self._cI, self._icomms._reader]
while mp.active_children():
# if we have stop, send out the poison pills once all frames have been
# processed (or at least pulled of the buffer)
if stop:
if not [i for i in fmap if fmap[i]] and not poisoned:
msg = "{0} outstanding frames".format(len(fmap))
for pid in threshers:
try:
threshers[pid].send((POISON, None, None))
except Exception as e:
msg = "Error stopping Thresher-{0}: {1}".format(
pid, e)
self._cI.send(
(IYRI_WARN, 'collator', 'Thresher', e))
poisoned = True
# continue with processing
try:
# block longer if we havent gotten poison pill
(rs, _, _) = select.select(ins, [], [], 0.5 if stop else 5)
# convert tkns from iyri to icomms format
# NOTE: all Thresh events (exluding THRESH_THRESH) will return
# data as a tuple t = (string message,buffer index) where any
# individual item in the tuple could be None
for r in rs:
if r == self._cI:
(cs, ts, ev, d) = ('', '', self._cI.recv(), '')
else:
(cs, ts, ev, d) = self._icomms.get_nowait()
if ev == POISON: stop = True
elif ev == THRESH_THRESH: # thresher process up/down message
if d is not None: # thresher up
msg = "{0} up".format(cs)
self._cI.send(
(IYRI_INFO, 'collator', 'Thresher', msg))
else: # thresher down
pid = int(cs.split('-')[1])
threshers[pid].close()
del threshers[pid]
msg = "{0} down".format(cs)
self._cI.send(
(IYRI_INFO, 'collator', 'Thresher', msg))
elif ev == THRESH_DQ: # frame has been pulled off buffer
(_, idx) = d
fmap[idx] = 0
elif ev == THRESH_WARN or ev == THRESH_ERR or ev == THRESH_DONE:
# frame processing finished
msg, idx = d
try:
del fmap[idx]
except:
pass
# handle individual events below
if ev == THRESH_WARN:
self._cI.send(
(IYRI_WARN, 'collator', 'Thresher', msg))
elif ev == THRESH_ERR:
pid = int(cs.split('-')[1])
msg = "{0}->{1}".format(cs, msg[0])
self._cI.send(
(IYRI_WARN, 'collator', 'Thresher', msg))
threshers[pid].send((POISON, None, None))
# make a new thresher (to replace this one)
try:
(pid,
send) = self._newthresher(qT, lSta, cb, smap)
threshers[pid] = send
except RuntimeError as e:
self._cI.send(
(IYRI_WARN, 'Collator', Thresher, e))
# determine workload percentage
if stop: continue
if len(fmap) / (m *
1.0) <= 0.1 and len(threshers) > mint:
msg = "Workload falling below 10%, removing a threseher"
self._cI.send((IYRI_INFO, 'collator', 'load', msg))
pid = threshers.keys()[0]
threshers[pid].send((POISON, None, None))
elif ev == GPS_GPSD: # gpsd up/down
self._submitgpsd(ts, d)
if d:
self._cI.send(
(IYRI_INFO, 'collator', 'GPSD', 'initiated'))
elif ev == GPS_FLT:
self._submitflt(ts, d)
elif ev == RDO_RADIO: # up/down message from radio(s).
if d is not None: # radio up
(mac, role) = d['mac'], d['role']
# shouldn't happen but make sure:
assert (cs not in rmap) # reinitialization
assert (role in ['abad', 'shama']) # role is valid
# notify iyri radio is up & add to maps
msg = "{0}:{1} initiated".format(role, cs)
self._cI.send(
(IYRI_INFO, 'collator', 'Radio', msg))
rmap[cs] = mac
smap[mac] = {'cb': None}
if role == 'abad': smap[mac]['cb'] = self._ab
else: smap[mac]['cb'] = self._sb
# update threshers
for pid in threshers:
threshers[pid].send((COL_RDO, ts, [mac]))
# submit radio & antenna(s)
self._submitradio(ts, mac, d)
for i in xrange(d['nA']):
self._submitantenna(
ts, {
'mac': mac,
'idx': i,
'type': d['type'][i],
'gain': d['gain'][i],
'loss': d['loss'][i],
'x': d['x'][i],
'y': d['y'][i],
'z': d['z'][i]
})
else: # radio down
msg = "{0} shutdown".format(cs)
self._cI.send(
(IYRI_INFO, 'collator', 'Radio', msg))
self._submitradio(ts, rmap[cs], None)
# delete our maps (only delete tasks if its empty)
del smap[rmap[cs]]
del rmap[cs]
elif ev == RDO_FAIL:
msg = "Deleting radio {0}: {1}".format(cs, d)
self._cI.send((IYRI_ERR, 'collator', 'Radio', msg))
self._submitradioevent(ts, [rmap[cs], 'fail', d])
# delete our maps
del smap[rmap[cs]]
del rmap[cs]
elif RDO_SCAN <= ev <= RDO_STATE:
if ev == RDO_SCAN: # need to format the channel list
d = ','.join(
["{0}:{1}".format(c, w) for (c, w) in d])
self._cI.send((IYRI_INFO, cs, RDO_DESC[ev], d))
self._submitradioevent(ts, [rmap[cs], RDO_DESC[ev], d])
elif ev == RDO_FRAME:
# get the index,length of the frame & src hwaddr
(i, l) = d
hw = rmap[cs]
# about to overwrite a frame?
if _ix in fmap and fmap[_ix] == 1:
df += 1
print 'dropping'
continue
# move to our buffer, put on threshers queue & update index
cb[(_ix * n):(_ix * n) +
l] = smap[hw]['cb'][(i * n):(i * n) + l]
fmap[_ix] = 1
qT.put((COL_FRAME, ts, (hw, _ix, l)))
_ix = (_ix + 1) % m
rf += 1
# if the buffer reaches 25% capacity, add a thresher
# NOTE: consider all outstanding frames even those that
# have been pulled of the buffer but are still being
# processed)
if len(fmap) / (m *
1.0) > 0.25 and len(threshers) < maxt:
# create at most mint new threshers w/out adding
# more than allowed. Give time for previously
# created threshers to start working
tsnow = time()
if tsnow - tst < 3: continue
x = min(mint, maxt - len(threshers))
msg = "Buffer at 25%, adding {0} threshers".format(
x)
self._cI.send((IYRI_INFO, 'collator', 'load', msg))
for _ in xrange(x):
try:
(pid, send) = self._newthresher(
qT, lSta, cb, smap)
threshers[pid] = send
except RuntimeError as e:
self._cI.send(
(IYRI_WARN, 'Collator', Thresher, e))
tst = time()
else: # unidentified event type, notify iyri
msg = "unknown event. {0}->{1}".format(ev, cs)
self._cI.send((IYRI_WARN, 'collator', 'icomms', msg))
except Empty:
continue
except (EOFError, IOError):
pass
except psql.OperationalError as e: # unrecoverable
rmsg = "{0}: {1}".format(e.pgcode, e.pgerror)
self._cI.send((IYRI_ERR, 'collator', 'DB', rmsg))
except psql.Error as e: # possible incorrect semantics/syntax
self._conn.rollback()
rmsg = "{0}: {1}".format(e.pgcode, e.pgerror)
self._cI.send((IYRI_WARN, 'collator', 'SQL', rmsg))
except IndexError as e:
rmsg = "Bad index {0} in event {1}".format(e, ev)
self._cI.send((IYRI_ERR, 'collator', "Index", rmsg))
except KeyError as e:
rmsg = "Bad key {0} in event {1}".format(e, ev)
self._cI.send((IYRI_ERR, 'collator', 'Key', rmsg))
except select.error as e: # hide (4,'Interupted system call') errors
if e[0] != 4:
self._cI.send((IYRI_WARN, 'collator', 'select', e))
except Exception as e: # handle catchall error
self._cI.send((IYRI_WARN, 'collator', type(e).__name__, e))
# clean up & shut down
try:
try:
# notify Nidus of sensor - check if radio(s), gpsd closed out
for cs in rmap:
self._submitradio(isots(), rmap[cs], None)
if self._gid: self._submitgpsd(isots(), None)
if self._sid: self._submitsensor(isots(), False)
except psql.Error:
self._conn.rollback()
self._cI.send(
(IYRI_WARN, 'collator', 'shutdown', "DB is corrupted"))
if self._curs: self._curs.close()
if self._conn: self._conn.close()
if rf or df:
msg = "{0} read frames, {1} dropped frames".format(rf, df)
self._cI.send((IYRI_DONE, 'collator', 'Complete', msg))
self._cI.close()
except (EOFError, IOError):
pass
except Exception as e:
if self._cI and not self._cI.closed:
msg = "Failed to clean up: {0}".format(e)
self._cI.send((IYRI_WARN, 'collator', 'shutdown', msg))
def main():
# step 1:
# obtain host name, gpuID
# create the corresponding folder
hostName = socket.gethostname()
foldName = str(hostName) + '-gpu' + str(gpu2Use)
path = os.getcwd()
targetDir = path + "/" + foldName
if not os.path.exists(targetDir):
try:
os.mkdir(targetDir)
except OSError:
print("creation of %s failed!" % targetDir)
MAXCORUN = int(args.maxCoRun) # max jobs per gpu
RANDSEED = int(args.seed)
gpuNum = 1
#logger.debug("MaxCoRun={}\trandseed={}\tsaveFile={}".format(MAXCORUN, RANDSEED, args.ofile))
#----------------------------------------------------------------------
# 1) application status table : 5 columns
#----------------------------------------------------------------------
#
# jobid gpu status starT endT
# 0 0 1 1 2
# 1 1 1 1.3 2.4
# 2 0 0 - -
# ...
#----------------------------------------------------------------------
maxJobs = 10000
rows, cols = maxJobs, 5 # note: init with a large prefixed table
d_arr = mp.Array(ctypes.c_double, rows * cols)
arr = np.frombuffer(d_arr.get_obj())
AppStat = arr.reshape((rows, cols))
id2name = {}
#----------------------------------------------------------------------
#
#----------------------------------------------------------------------
#--------------------------------------------------------------------------
# input: app, app2dir_dd in app_info.py
#--------------------------------------------------------------------------
if len(app) <> len(app2dir_dd):
print "Error: app number wrong, check ../prepare/app_info.py!"
sys.exit(1)
# three random sequences
#app_s1 = genRandSeq(app, seed=RANDSEED) # pi
apps_num = len(app)
logger.debug("Total GPU Applications = {}.".format(apps_num))
#--------------------------------------------------------------------------
# interference analysis : by run 2 apps concurrently
#--------------------------------------------------------------------------
appPool = app
for idx, app1 in enumerate(appPool):
ofile = targetDir + '/' + app1 + ".npy"
#print ofile
# exclude: rodinia-heartwall, lonestar_sssp, dmr
if idx >= 0: # NOTE: modify if program hangs
#if idx == 0: # NOTE: modify if program hangs
app_run2_dd = {}
for a2_idx, app2 in enumerate(appPool):
if a2_idx >= 0:
print("idx = {}({}) : a2_idx={}({})".format(
idx, app1, a2_idx, app2))
app1_runtime, app2_runtime = [], []
# run 3 times, get the fastest
for ii in xrange(2):
workers = [] # for mp processes
# run 2 processes concurrently
jobID = 0
id2name[jobID] = app1
process = Process(target=run_work,
args=(jobID, AppStat,
app2dir_dd[app1]))
process.daemon = False
workers.append(process)
process.start()
jobID = 1
id2name[jobID] = app2
process = Process(target=run_work,
args=(jobID, AppStat,
app2dir_dd[app2]))
process.daemon = False
workers.append(process)
process.start()
for p in workers:
p.join()
total_jobs = 2
appRT_dd = getGpuJobTiming(AppStat, total_jobs,
id2name)
#print appRT_dd
app1_runtime.append(appRT_dd[app1])
app2_runtime.append(appRT_dd[app2])
#print app1_runtime
app1_best = min(app1_runtime)
# NOTE: only remember the app1 runtime when running with app2
app_run2_dd[app2] = app1_best
import talib as tb
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import matplotlib.ticker as mticker
from mpl_finance import candlestick_ohlc
import datetime as dt
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.figure import Figure
import matplotlib.pyplot as plt
from matplotlib.dates import DateFormatter, WeekdayLocator,\
DayLocator, MONDAY
globalz = multiprocessing.Array('i', 32)
varz = []
API_KEY = ""
API_SECRET = ""
class Application(Frame):
def create_widgets(self):
self.quit_button = Button(self)
self.quit_button['text'] = 'STOP'
self.quit_button['fg'] = 'red'
self.quit_button['command'] = self.stop #self.quit
self.quit_button.pack({'side': 'bottom'})
self.launch_button = Button(self)
def test_file_caching(self):
"""
max_batches and batch_size are directly correlated to test input data
make sure gen_rand_data has the correct shape
:return:
"""
from mock_config import mock_batches, mock_expected_malloc_requests, \
mock_class_index_map, mock_file_index_list
mock_batches = copy.deepcopy(mock_batches)
mock_expected_malloc_requests = copy.deepcopy(
mock_expected_malloc_requests)
mock_class_index_map = copy.deepcopy(mock_class_index_map)
max_batches = len(mock_batches)
batch_size = 500
read_size = 2 * batch_size
bucket_length = 10
# in_queue = multiprocessing.Queue(maxsize=max_batches)
# out_queue = multiprocessing.Queue(maxsize=max_batches)
comm_driver = QueueCommDriver({'ctl': 10, 'in': 100, 'out': 100})
# building queue up with read requests
for batch in mock_batches:
success = comm_driver.write('in', batch)
self.assertTrue(success)
reader_id = 0
shared_data_pointer = range(bucket_length)
for bucket in shared_data_pointer:
data_sets = []
for request in mock_expected_malloc_requests:
# reshape the requested shape to match the batch_size
shape = (read_size, ) + request[1]
shared_array_base = multiprocessing.Array(ctypes.c_double,
np.prod(shape),
lock=False)
shared_array = np.ctypeslib.as_array(shared_array_base)
shared_array = shared_array.reshape(shape)
data_sets.append(shared_array)
state = multiprocessing.Value('i', 0)
generator_start_counter = multiprocessing.Value('i', 0)
generator_end_counter = multiprocessing.Value('i', 0)
shared_data_pointer[bucket] = [
state, data_sets, generator_start_counter,
generator_end_counter
]
reader = FileReader(worker_id=reader_id,
comm_driver=comm_driver,
shared_memory_pointer=shared_data_pointer,
max_batches=max_batches,
read_size=read_size,
class_index_map=mock_class_index_map,
file_index_list=mock_file_index_list,
io_ctlr=IOController()
# io_driver=H5DataIODriver({"cache_handles": True})
)
reader.read()
out = list()
start_time = datetime.datetime.utcnow()
end_time = datetime.timedelta(seconds=10) + start_time
while datetime.datetime.utcnow() < end_time and len(
out) != max_batches:
batch = comm_driver.read('out', block=False)
if batch is not None:
out.append(batch)
self.assertEquals(
len(out), max_batches,
"test consumed {0} of {1} expected batches from the in_queue".
format(len(out), max_batches))
target = 0
self.assertGreaterEqual(len(reader.io_ctlr('').file_handle_holder), 0)
for handle in reader.io_ctlr('').file_handle_holder:
# Here we try to close the file again, this will raise an Exception and thus means the caching cleaned up
# successfully. if 0 then we closed everything successfully with the __exit__ function
target += 1
try:
handle.close()
except Exception:
target -= 1
self.assertEqual(target, 0)
# -*- coding: utf-8 -*- # @Time : 2018/4/26 17:00 # @Author : Inkky # @Email : [email protected] ''' ''' import multiprocessing as mp import time # 其中d和i参数用来设置数据类型的， # d表示一个双精浮点类型，i表示一个带符号的整型。 value1 = mp.Value('i', 0) value2 = mp.Value('d', 3.14) array = mp.Array('i', [1, 2, 3, 4]) # 这里的Array和numpy中的不同，它只能是一维的，不能是多维的。 def job(v, num,l): l.acquire() for _ in range(5): time.sleep(0.2) v.value += num # v.value获取共享变量值 print('\n',v.value, end="") l.release() def multicore(): l=mp.Lock() v = mp.Value('i', 0) # 定义共享变量 p1 = mp.Process(target=job, args=(v, 1,l)) p2 = mp.Process(target=job, args=(v, 3,l)) # 设定不同的number看如何抢夺内存 p1.start()
dic = mp.Manager().dict()#创建字典的时候不能用{}创建
lis = mp.Manager().list([1,2,3,4])
plist = []
for i in range(3):
p = mp.Process(target = job3, args = (dic,lis,i))
plist.append(p)
p.start()
for process in plist:
process.join()
print(dic)
print(lis)
for process in plist:
process.close()
#共享内存shared memory:
shared_value = mp.Value('i',-1)#i表示带符号的整型
shared_array = mp.Array('I',[1,2,3,4,5])#I表示不带符号的整型
plist = []
for i in range(5):
p = mp.Process(target = job4, args = (shared_value,i))
plist.append(p)
p.start()
for process in plist:
p.join()
print(shared_value.value)
for process in plist:
p.close()
#进程锁
lock = mp.Lock()#和线程锁使用方法一样
plist = []
v = mp.Value('i',0)
def __createArray(self, dimensions, dtype=np.float64, syncAccess=False):
"""
Create an array
"""
# convert to dtype type (in case short code)
dtype = np.dtype(dtype)
if (dtype == np.float32):
ctype = ctypes.c_float
elif (dtype == np.float64):
ctype = ctypes.c_double
elif (dtype == np.int32):
ctype = ctypes.c_int32
elif (dtype == np.int64):
ctype = ctypes.c_int64
elif (dtype == np.int16):
ctype = ctypes.c_int16
elif (dtype == bool):
ctype = ctypes.c_bool
else:
raise ValueError("Unsupported dtype")
# This lock is only needed when creating the array, which is fast
# It is not used when accessing data
self.lock.acquire()
# double size if necessary
if (self.cnt >= len(self.sharedArrays)):
self.sharedArrays = self.sharedArrays + [None] * len(self.sharedArrays)
self.sharedArrayBases = self.sharedArrayBases + [None] * len(self.sharedArrayBases)
# next handle
self.__getNextFreeHdl()
# create array in shared memory segment
if (syncAccess):
try:
self.sharedArrayBases[self.cur] = multiprocessing.Array(ctype,
int(np.prod(dimensions)))
except:
raise MemoryError("Failed to allocate memory for shared array. Note that $TMPDIR must have more space than allocated shared memory.")
self.sharedArrays[self.cur] = np.frombuffer(self.sharedArrayBases[self.cur].get_obj(),dtype=dtype)
else:
try:
self.sharedArrayBases[self.cur] = multiprocessing.RawArray(ctype,
int(np.prod(dimensions)))
except:
raise MemoryError("Failed to allocate memory for shared array. Note that $TMPDIR must have more space than allocated shared memory.")
self.sharedArrays[self.cur] = np.frombuffer(self.sharedArrayBases[self.cur],dtype=dtype)
# do a reshape for correct dimensions
# Returns a masked array containing the same data, but with a new shape.
# The result is a view on the original array
self.sharedArrays[self.cur] = self.sharedArrays[self.cur].reshape(dimensions)
# update cnt
self.cnt += 1
# Release the creation lock
self.lock.release()
# return handle to the shared memory numpy array
return self.cur
| `'H'` | unsigned short | int | 2 |
| `'i'` | signed int | int | 2 |
| `'I'` | unsigned int | int | 2 |
| `'l'` | signed long | int | 4 |
| `'L'` | unsigned long | int | 4 |
| `'q'` | signed long long | int | 8 |
| `'Q'` | unsigned long long | int | 8 |
| `'f'` | float | float | 4 |
| `'d'` | double | float | 8 |
参考地址：https://docs.python.org/3/library/array.html
注意：Value和Array只适用于Process类。
'''
import multiprocessing
def f(n, a):
n.value = 3.14
a[0] = 5
if __name__ == '__main__':
num = multiprocessing.Value('d', 0.0)
arr = multiprocessing.Array('i', range(10))
p = multiprocessing.Process(target=f, args=(num, arr))
p.start()
p.join()
print(num.value)
print(arr[:])
Shared Memory
process 1 (Main process) ------> SHARED MEMORY <-------- process 2 (child process (calc_square))
'''
import multiprocessing
def calc_square(numbers, result, v):
v.value = 5.56
for index, num in enumerate(numbers):
result[index] = num * num
if __name__ == '__main__':
number = [1, 2, 3]
result = multiprocessing.Array(
'i', 3
) #Result is the shared Array variable where "i" is to denote integer and 3 is the size
v = multiprocessing.Value('d', 0.0)
p = multiprocessing.Process(target=calc_square, args=(number, result, v))
p.start()
p.join()
print("The Result is : {}".format(result[:]))
print("The Value is : {}".format(v.value))
def main():
#global app2dir
#global app2cmd
global app2metric
global app2trace
#-------------------------------------------------------------------------#
# GPU Job Table
#-------------------------------------------------------------------------#
# jobid starT endT status
# 0 1 2
# 1 1.3 2.4
# 2 - -
# ...
#----------------------------------------------------------------------
maxJobs = 10000
rows, cols = maxJobs, 4 # note: init with a large prefixed table
d_arr = mp.Array(ctypes.c_double, rows * cols)
arr = np.frombuffer(d_arr.get_obj())
GpuJobTable = arr.reshape((rows, cols))
#===================#
# 1) read app info
#===================#
#app2dir = np.load('../07_sim_devid/similarity/app2dir_dd.npy').item()
#app2cmd = np.load('../07_sim_devid/similarity/app2cmd_dd.npy').item()
#app2metric = np.load('../07_sim_devid/similarity/app2metric_dd.npy').item()
#app2trace = np.load('../07_sim_devid/perfmodel/app2trace_dd.npy').item()
#print len(app2dir), len(app2cmd), len(app2metric), len(app2trace)
#app_iobound_dd = np.load('./case_studies/app_iobound_dd.npy').item()
#=========================================================================#
# set up the launch order list
#=========================================================================#
appsList = get_appinfo('./prepare/app_info_79.bin')
#print appsList
app2dir_dd = {}
for v in appsList:
app2dir_dd[v[0]] = v[1]
#print app2dir_dd
#launch_list = ['cudasdk_concurrentKernels', 'poly_correlation']
#==========#
# test 1
#==========#
#launch_list = ['cudasdk_concurrentKernels', 'poly_3mm', 'poly_gemm', 'cudasdk_segmentationTreeThrust'] # sim
#launch_list = ['cudasdk_concurrentKernels', 'cudasdk_segmentationTreeThrust', 'poly_3mm', 'poly_gemm'] # job size
#launch_list = ['cudasdk_concurrentKernels', 'poly_3mm', 'cudasdk_segmentationTreeThrust', 'cudasdk_MCSingleAsianOptionP'] # sim
#launch_list = ['cudasdk_concurrentKernels', 'cudasdk_segmentationTreeThrust', 'poly_3mm', 'cudasdk_MCSingleAsianOptionP'] # job size
#launch_list = ['cudasdk_concurrentKernels', 'cudasdk_segmentationTreeThrust', 'cudasdk_MCSingleAsianOptionP', 'poly_3mm'] # job size + iobound
#launch_list = ['cudasdk_concurrentKernels', 'poly_3mm', 'cudasdk_segmentationTreeThrust'] # sim
#launch_list = ['cudasdk_concurrentKernels', 'cudasdk_segmentationTreeThrust', 'poly_3mm'] # job size
#for x in launch_list:
# print x, app2dir_dd[x]
#==========#
# test 2
#==========#
#launch_list = ['shoc_lev1reduction', 'poly_3mm', 'poly_gemm', 'cudasdk_segmentationTreeThrust'] # sim
#launch_list = ['shoc_lev1reduction', 'cudasdk_segmentationTreeThrust', 'poly_3mm', 'poly_gemm'] # jobsize + sim
#==========#
# test 3
#==========#
#launch_list = ['cudasdk_scan', 'cudasdk_interval', 'cudasdk_MCEstimatePiQ', 'cudasdk_concurrentKernels'] # sim
#launch_list = ['cudasdk_scan', 'cudasdk_concurrentKernels', 'cudasdk_MCEstimatePiQ','cudasdk_interval'] # threads + sim
#launch_list = ['cudasdk_scan', 'cudasdk_concurrentKernels', 'cudasdk_interval', 'cudasdk_MCEstimatePiQ'] # threads + sim
apps_num = len(launch_list)
logger.debug("Total GPU Applications = {}.".format(apps_num))
appQueList = copy.deepcopy(launch_list) # application running queue
workers = [] # for mp processes
#==================================#
# run the apps in the queue
#==================================#
MAXCORUN = 2
activeJobs = 0
jobID = -1
current_jobid_list = []
for i in xrange(apps_num):
Dispatch = False
if activeJobs < MAXCORUN:
Dispatch = True
#print("iter {} dispatch={}".format(i, Dispatch))
if Dispatch:
activeJobs += 1
jobID += 1
current_jobid_list.append(jobID)
appName = appQueList[i]
process = Process(target=run_work,
args=(jobID, GpuJobTable, appName, app2dir_dd))
process.daemon = False
#logger.debug("Start %r", process)
workers.append(process)
process.start()
else:
# spin
while True:
break_loop = False
current_running_jobs = 0
jobs2del = []
for jid in current_jobid_list:
if GpuJobTable[jid,
3] == 1: # check the status, if one is done
jobs2del.append(jid)
break_loop = True
if break_loop:
activeJobs -= 1
# update
if jobs2del:
for id2del in jobs2del:
del_idx = current_jobid_list.index(id2del)
del current_jobid_list[del_idx]
break
#------------------------------------
# after spinning, schedule the work
#------------------------------------
#print("iter {}: activeJobs = {}".format(i, activeJobs))
activeJobs += 1
jobID += 1
current_jobid_list.append(jobID)
#print("iter {}: activeJobs = {}".format(i, activeJobs))
appName = appQueList[i]
process = Process(target=run_work,
args=(jobID, GpuJobTable, appName, app2dir_dd))
process.daemon = False
#logger.debug("Start %r", process)
workers.append(process)
process.start()
#=========================================================================#
# end of running all the jobs
#=========================================================================#
for p in workers:
p.join()
total_jobs = jobID + 1
PrintGpuJobTable(GpuJobTable, total_jobs)
if total_jobs <> apps_num:
logger.debug("[Warning] job number doesn't match.")
def init_shm_ndarray(shape,dtype=np.float64):
'''Initialize a shared memory array of the supplied shape and dtype'''
shm = mp.Array(_numpy_to_ctypes[np.dtype(dtype)],int(np.prod(shape)),lock=False)
return shm
print datetime.datetime.now().time(), '- Allocating memory...'
syn_images_num = len(
[line.strip() for line in open(g_syn_images_train_val_split, 'r')])
env = lmdb.open(g_pool5_lmdb, readonly=True)
datum_array = np.empty(0)
with env.begin() as txn:
cursor = txn.cursor()
for key, value in cursor:
datum = caffe_pb2.Datum()
datum.ParseFromString(value)
datum_array = caffe.io.datum_to_array(datum)
break
print datetime.datetime.now().time(
), '- the shape of the datum is:', datum_array.shape
shared_array_base = multiprocessing.Array(ctypes.c_double,
syn_images_num * datum_array.size)
shared_array = np.ctypeslib.as_array(shared_array_base.get_obj())
shared_array = shared_array.reshape([syn_images_num] + list(datum_array.shape))
def image_pair_to_string(pair_info, def_param=shared_array):
array_1_id = pair_info[0]
array_2_id = pair_info[1]
similar_or_not = pair_info[2]
array_1 = shared_array[array_1_id, :]
array_2 = shared_array[array_2_id, :]
array = np.concatenate((array_1, array_2), axis=0)
datum = caffe.io.array_to_datum(array)
datum.label = similar_or_not
return datum.SerializeToString()
loop = asyncio.get_event_loop()
loop.create_task(
selfbot.start(
'NzE2NzE2NTIzMjc1ODEyOTE0.XvqgaQ.6qyKKBfFOcLdAL2i0LAl1FWUZUw',
bot=False))
loop.run_forever()
if __name__ == '__main__':
# running bot and selfbot in separate OS processes
# shared event for embed update
update_event = multiprocessing.Event()
# shared array with answer results
answer_scores = multiprocessing.Array(typecode_or_type='i',
size_or_initializer=3)
p_bot = multiprocessing.Process(target=bot_with_cyclic_update_process,
args=(update_event, answer_scores))
p_selfbot = multiprocessing.Process(target=selfbot_process,
args=(update_event, answer_scores))
p_bot.start()
p_selfbot.start()
p_bot.join()
p_selfbot.join()
def solveNbodiesVerletMultiprocessing(masses, init_pos, init_vel, dt,
iterations):
def solveForOneBody(q, q_out, init_vel, shared_pos, body, events1,
events2):
dimension = len(shared_pos) // len(masses)
my_cur_pos = np.array(shared_pos[body * dimension:(body + 1) *
dimension])
cur_acceleration = acceleration_no_np(body, masses, shared_pos)
result = np.empty((iterations, dimension * 2))
result[0, :] = np.concatenate(
(my_cur_pos, init_vel[body * dimension:(body + 1) * dimension]))
for j in np.arange(iterations - 1) + 1:
result[j, :dimension] = \
(my_cur_pos
+ result[j-1, dimension:] * dt
+ 0.5 * cur_acceleration * dt**2)
q.put([body, result[j, :dimension]])
events1[body].set()
if body == 0:
for i in range(len(masses)):
events1[i].wait()
events1[i].clear()
for i in range(len(masses)):
tmp = q.get()
shared_pos[tmp[0] * dimension:(tmp[0] + 1) *
dimension] = tmp[1]
for i in range(len(masses)):
events2[i].set()
else:
events2[body].wait()
events2[body].clear()
my_cur_pos = np.array(shared_pos[body * dimension:(body + 1) *
dimension])
next_acceleration = acceleration_no_np(body, masses, shared_pos)
result[j, dimension:] = \
(result[j-1, dimension:]
+ 0.5 * (cur_acceleration + next_acceleration) * dt)
cur_acceleration = next_acceleration
q_out.put([body, result])
if __name__ == '__main__':
times = np.arange(iterations) * dt
posvel = np.empty((times.size, init_pos.size * 2))
shared_pos = mp.Array('d', init_pos)
events1 = []
events2 = []
for body in masses:
events1.append(mp.Event())
events2.append(mp.Event())
events1[-1].clear()
events2[-1].clear()
q = mp.Queue()
q_out = mp.Queue()
processes = []
for body in range(masses.size):
processes.append(
mp.Process(target=solveForOneBody,
args=(q, q_out, init_vel, shared_pos, body, events1,
events2)))
processes[-1].start()
dim = init_pos.size // len(masses)
for i in range(len(masses)):
tmp = q_out.get()
posvel[:, tmp[0] * dim:(tmp[0] + 1) * dim] = tmp[1][:, :dim]
posvel[:, init_pos.size + tmp[0] * dim:init_pos.size +
(tmp[0] + 1) * dim] = tmp[1][:, dim:]
for process in processes:
process.join()
return posvel, times
for i in range(10):
array[i]=random.randrange(0,100) # randomly pick the number in range 1~100.
return
def sorting_function(array) :
i=9
for i in range(9,0,-1):
j=0
for j in range(i):
time.sleep(0.01) # in order to slow down.
if array[j] > array[j+1] : # swap the bigger one and smaller one.
tmp = array[j+1]
array[j+1]=array[j]
array[j]=tmp
return
def print_array(array, pid):
print"[%d] --> " %pid,; # print the process's id.
for i in range(10):
print"%d " %array[i],; # print sorted numbers.
if __name__ =='__main__' :
array=multiprocessing.Array('i',[0]*10)
print('############Start#############')
lock = multiprocessing.Lock() # avoid the "race condition" lock the thread.
for i in range(10) :
p = multiprocessing.Process(target=sorting_thread, args=(array, i,lock))
p.start() # start the work.
p.join() # stop the work.
print('#############End###############')
counter = 0
print('Parallel Programming Assignment 2')
print('*********************************')
def increase_counter(number):
global primes
global prime_object
if prime_object.is_prime(number):
primes[number-1] = True
running = True
table = []
while running:
primes = multiprocessing.Array('b', primes_till)
primes_till = eval(input("Enter max number: "))
processors_count = multiprocessing.cpu_count()
chosen_processors = eval(input(f'Chose number of processors from 1 to {processors_count}: '))
if chosen_processors not in range(1, processors_count + 1):
print(f'You chose {chosen_processors}, '
f'that is not included in the expected range! We are proceeding with the maximum power.')
chosen_processors = processors_count
print(f'Calculating with {chosen_processors} processors...')
pool = multiprocessing.Pool(processes=chosen_processors)
start_time = datetime.datetime.now()
pool.map(increase_counter, range(1, primes_till + 1))
prime_count = primes[:].count(1)
np.clip(f0_map_list.reshape(n_hop, n_band).T, 1e-5, 1.0)),
cmap='plasma',
interpolation='nearest',
aspect='auto',
origin='lower')
pl.plot(np.argmax(f0_map_list, axis=1))
pl.show()
else:
print("* Spawn data process...")
ctx = mp.get_context('spawn')
memory_pool = []
data_queue = mp.Queue(n_data_process * 2)
memory_queue = mp.Queue(n_data_process * 2)
command_queue = mp.Queue(n_data_process)
for i in range(n_data_process * 2):
memory_pool.append((mp.Array(ctypes.c_float, batch_size * n_band),
mp.Array(ctypes.c_float,
batch_size * n_feature)))
memory_queue.put(i)
del i
for _ in range(n_data_process):
ctx.Process(target=data_fetch_process_main,
args=(data_queue, memory_queue, command_queue,
memory_pool)).start()
def fetch_mem_data():
i_memory = data_queue.get()
(memory_f0, memory_feature) = memory_pool[i_memory]
with memory_f0.get_lock():
with memory_feature.get_lock():
out = np.frombuffer(memory_f0.get_obj(),
