zorder=2)
plt.xticks(fontsize=16)
plt.yticks(fontsize=16)
plt.xlabel('Time', fontsize=18)
plt.ylabel('Amplitude', fontsize=18)
plt.xlim(0, len(fit.data))
plt.legend(loc=0)
## multiprocess 3 voxels
data = [data, data, data]
indices = ([1, 2, 3], [4, 6, 5], [7, 8, 9])
bundle = utils.multiprocess_bundle(og.GaussianFit,
model,
data,
grids,
bounds,
indices,
auto_fit=True,
verbose=1,
Ns=3)
## run
print("popeye will analyze %d voxels across %d cores" % (len(bundle), 3))
with sharedmem.Pool(np=3) as pool:
t1 = datetime.datetime.now()
output = pool.map(utils.parallel_fit, bundle)
t2 = datetime.datetime.now()
delta = t2 - t1
print("popeye multiprocessing finished in %s.%s seconds" %
(delta.seconds, delta.microseconds))
len(mass_star3),
len(idCent_star3)]
Dlen = 5000000
center_pivD = numpy.fromfile('/home/twopointdata/randdmpos' + dirno +
'.rand' + str(Dlen) + 'alltracer',
dtype=(numpy.float64, 3))
gnp = 16
Blen = Dlen / gnp
fDMcen_z = fDMcenter[:, 2]
slices1 = [slice(i, i + 1) for i in range(0, gnp)]
if (qzero == 1):
fq = numpy.zeros(len(fq))
with sharedmem.Pool() as pool:
def work(slice):
si1 = slice.start
sj1 = slice.stop
seCent = fDMcenter
seeigvec = feigvec
sq = fq
sDcent = center_pivD[si1 * Blen:sj1 * Blen]
if (si1 == (gnp - 1)):
sDcent = center_pivD[si1 * Blen:Dlen]
hij = wgp_shapcorr_allsg(seCent, seeigvec, sq, sDcent, rpbins, zpbins,
si1, sj1, boxlen)
hij.tofile('/home/wgpdata_alltracer/' + rdstr2 + '_' + str(Dlen) +
'Sgp' + clrstr1 + '.' + dirno + '.' + str(mmin) + '.' +
str(si1) + str(sj1) + typ1 + '.alltracer')
def test_parallel_fit_manual_grids():
# stimulus features
viewing_distance = 38
screen_width = 25
thetas = np.arange(0, 360, 45)
num_blank_steps = 0
num_bar_steps = 30
ecc = 10
tr_length = 1.0
frames_per_tr = 1.0
scale_factor = 0.10
pixels_down = 100
pixels_across = 100
dtype = ctypes.c_int16
voxel_index = (1, 2, 3)
auto_fit = True
verbose = 1
# create the sweeping bar stimulus in memory
bar = simulate_bar_stimulus(pixels_across, pixels_down, viewing_distance,
screen_width, thetas, num_bar_steps,
num_blank_steps, ecc)
# create an instance of the Stimulus class
stimulus = VisualStimulus(bar, viewing_distance, screen_width,
scale_factor, tr_length, dtype)
# initialize the gaussian model
model = og.GaussianModel(stimulus, utils.double_gamma_hrf)
model.hrf_delay = 0
# generate a random pRF estimate
x = -5.24
y = 2.58
sigma = 1.24
beta = 2.5
baseline = -0.25
# create the "data"
data = model.generate_prediction(x, y, sigma, beta, baseline)
# set search grid
x_grid = slice(-5, 4, 5)
y_grid = slice(-5, 7, 5)
s_grid = slice(1 / stimulus.ppd, 5.25, 5)
b_grid = slice(0.1, 4.0, 5)
# set search bounds
x_bound = (-12.0, 12.0)
y_bound = (-12.0, 12.0)
s_bound = (1 / stimulus.ppd, 12.0)
b_bound = (1e-8, 1e2)
m_bound = (None, None)
# loop over each voxel and set up a GaussianFit object
grids = (
x_grid,
y_grid,
s_grid,
)
bounds = (x_bound, y_bound, s_bound, b_bound, m_bound)
# make 3 voxels
all_data = np.array([data, data, data])
num_voxels = data.shape[0]
indices = [(1, 2, 3)] * 3
# bundle the voxels
bundle = utils.multiprocess_bundle(og.GaussianFit, model, all_data, grids,
bounds, indices)
# run analysis
with sharedmem.Pool(np=sharedmem.cpu_count() - 1) as pool:
output = pool.map(utils.parallel_fit, bundle)
# assert equivalence
for fit in output:
npt.assert_almost_equal(fit.x, x, 2)
npt.assert_almost_equal(fit.y, y, 2)
npt.assert_almost_equal(fit.sigma, sigma, 2)
npt.assert_almost_equal(fit.beta, beta, 2)
npt.assert_almost_equal(fit.baseline, baseline, 2)
def test_recast_estimation_results():
# stimulus features
viewing_distance = 38
screen_width = 25
thetas = np.arange(0, 360, 45)
num_blank_steps = 0
num_bar_steps = 30
ecc = 10
tr_length = 1.0
frames_per_tr = 1.0
scale_factor = 0.10
pixels_down = 100
pixels_across = 100
dtype = ctypes.c_int16
voxel_index = (1, 2, 3)
auto_fit = True
verbose = 1
# create the sweeping bar stimulus in memory
bar = simulate_bar_stimulus(pixels_across, pixels_down, viewing_distance,
screen_width, thetas, num_bar_steps,
num_blank_steps, ecc)
# create an instance of the Stimulus class
stimulus = VisualStimulus(bar, viewing_distance, screen_width,
scale_factor, tr_length, dtype)
# initialize the gaussian model
model = og.GaussianModel(stimulus, utils.spm_hrf)
model.hrf_delay = 0
# generate a random pRF estimate
x = -5.24
y = 2.58
sigma = 1.24
beta = 2.5
baseline = -0.25
# create the "data"
data = model.generate_prediction(x, y, sigma, beta, baseline)
# set search grid
x_grid = utils.grid_slice(-5, 4, 5)
y_grid = utils.grid_slice(-5, 7, 5)
s_grid = utils.grid_slice(1 / stimulus.ppd, 5.25, 5)
b_grid = utils.grid_slice(0.1, 4.0, 5)
# set search bounds
x_bound = (-12.0, 12.0)
y_bound = (-12.0, 12.0)
s_bound = (1 / stimulus.ppd, 12.0)
b_bound = (1e-8, 1e2)
m_bound = (None, None)
# loop over each voxel and set up a GaussianFit object
grids = (
x_grid,
y_grid,
s_grid,
)
bounds = (x_bound, y_bound, s_bound, b_bound, m_bound)
# create 3 voxels of data
all_data = np.array([data, data, data])
indices = [(0, 0, 0), (0, 0, 1), (0, 0, 2)]
# bundle the voxels
bundle = utils.multiprocess_bundle(og.GaussianFit, model, all_data, grids,
bounds, indices)
# run analysis
with sharedmem.Pool(np=3) as pool:
output = pool.map(utils.parallel_fit, bundle)
# create grid parent
arr = np.zeros((1, 1, 3))
grid_parent = nibabel.Nifti1Image(arr, np.eye(4, 4))
# recast the estimation results
nif = utils.recast_estimation_results(output, grid_parent)
dat = nif.get_data()
# assert equivalence
npt.assert_almost_equal(np.mean(dat[..., 0]), x)
npt.assert_almost_equal(np.mean(dat[..., 1]), y)
npt.assert_almost_equal(np.mean(dat[..., 2]), sigma)
npt.assert_almost_equal(np.mean(dat[..., 3]), beta)
npt.assert_almost_equal(np.mean(dat[..., 4]), baseline)
# recast the estimation results - OVERLOADED
nif = utils.recast_estimation_results(output, grid_parent, True)
dat = nif.get_data()
# assert equivalence
npt.assert_almost_equal(np.mean(dat[..., 0]), np.arctan2(y, x), 2)
npt.assert_almost_equal(np.mean(dat[..., 1]), np.sqrt(x**2 + y**2), 2)
npt.assert_almost_equal(np.mean(dat[..., 2]), sigma)
npt.assert_almost_equal(np.mean(dat[..., 3]), beta)
npt.assert_almost_equal(np.mean(dat[..., 4]), baseline)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("filename")
parser.add_argument("-f", "--format", dest='format', required=True)
parser.add_argument("-p",
"--ptype",
dest='ptypes',
action='append',
required=False,
type=int)
parser.add_argument("-o", "--output", dest='output', required=True)
parser.add_argument("--maxnpar",
dest='maxnpar',
default=20 * 1024 * 1024,
required=False,
type=int)
parser.add_argument("-F",
"--filter",
dest='filter',
required=False,
type=lambda x: x.split(':'))
group = parser.add_argument_group()
group.add_argument("-m",
"--meshindex",
dest='meshindex',
required=False,
type=MeshIndex.fromfile)
x = group.add_mutually_exclusive_group()
x.add_argument("--origin",
nargs='+',
dest='origin',
type=float,
required=False)
x.add_argument("--center",
nargs='+',
dest='center',
type=float,
required=False)
group.add_argument("--boxsize",
nargs='+',
dest='boxsize',
type=float,
required=False)
args = parser.parse_args()
origin = None
boxsize = None
def open(template, fid, create=False):
if '%d' in template:
snap = Snapshot(template % fid, args.format, create=create)
else:
snap = Snapshot(template, args.format, create=create)
return snap
def filter(snap, ptype, origin, boxsize):
if snap.C['N'][ptype] == 0: return None, 0
if origin is None or boxsize is None:
return None, snap.C['N'][ptype]
tail = origin + boxsize
pos = snap[ptype, 'pos']
iter = numpy.nditer([pos[:, 0], pos[:, 1], pos[:, 2], None],
op_dtypes=[None, None, None, '?'],
op_flags=[['readonly']] * 3 +
[['writeonly', 'allocate']],
flags=['external_loop', 'buffered'])
for x, y, z, m in iter:
m[...] = \
(x >= origin[0]) & (y >= origin[1]) & (z >= origin[2]) \
& (x <= tail [0]) & (y <= tail [1]) & (z <= tail [2])
return iter.operands[3], iter.operands[3].sum()
def select(snap, ptype, block, mask):
if mask is None:
result = snap[ptype, block]
else:
result = snap[ptype, block][mask]
del snap[ptype, block]
return result
snap0 = open(args.filename, 0)
nptypes = snap0.reader.schema.nptypes
if args.ptypes is None or len(args.ptypes) == 0:
args.ptypes = range(nptypes)
print 'doing', args.ptypes
if '%d' in args.filename:
Nfile = snap0.C['Nfiles']
else:
Nfile = 1
fids = range(Nfile)
if args.boxsize is not None:
boxsize = numpy.empty(3, dtype='f8')
boxsize[:] = args.boxsize
if args.origin is None and args.center is None:
parser.print_help()
parser.exit()
else:
origin = numpy.empty(3, dtype='f8')
if args.origin is not None:
origin[:] = args.origin
else:
origin[:] = args.center
origin[:] = origin - boxsize * 0.5
if args.meshindex is not None:
fids = args.meshindex.cut(origin, boxsize)
args.meshindex = None
print origin, boxsize
defaultheader = snap0.header
Ntot = snap0.C['Ntot']
Ntot_out = numpy.zeros(nptypes, dtype='i8')
with sharedmem.Pool(use_threads=True) as pool:
def work(fid):
snap = open(args.filename, fid)
N_out = numpy.zeros(nptypes, dtype='i8')
for ptype in args.ptypes:
mask, count = filter(snap, ptype, origin, boxsize)
N_out[ptype] = count
return N_out
def reduce(N_out):
Ntot_out[...] = N_out + Ntot_out
pool.map(work, fids, callback=reduce)
Nfile_out = (Ntot_out.sum() // args.maxnpar + 1)
if Nfile_out > 1 and '%d' not in args.output:
args.output += '.%d'
outputs = [open(args.output, fid, create=True) for fid in range(Nfile_out)]
written = numpy.zeros((Nfile_out, nptypes), dtype='i8')
free = numpy.empty_like(written)
writing = numpy.zeros((Nfile_out, nptypes), dtype='i8')
cursor = numpy.zeros(nptypes, dtype='i8')
for i, output in enumerate(outputs):
output.header[...] = defaultheader
output.C['Ntot'] = Ntot_out
output.C['N'] = (Ntot_out * (i + 1) // Nfile_out) \
- (Ntot_out * i // Nfile_out)
free[i] = output.C['N']
output.C['Nfiles'] = Nfile_out
output.create_structure()
with sharedmem.Pool(use_threads=True) as pool:
def work(fid):
snap = open(args.filename, fid)
for ptype in args.ptypes:
mask, count = filter(snap, ptype, origin, boxsize)
if count == 0: continue
with pool.lock:
cumfree = free[:, ptype].cumsum()
last_output = cumfree.searchsorted(count, side='left')
first_output = cumfree.searchsorted(0, side='right')
table = numpy.zeros(last_output - first_output + 1,
('i8', 4))
outputid, istart, len, ostart = table.T
outputid[...] = range(first_output, last_output + 1)
ostart[...] = written[first_output:last_output + 1, ptype]
len[:-1] = free[first_output:last_output, ptype]
len[-1] = count - len[:-1].sum()
istart[1:] = len.cumsum()[:-1]
writing[first_output:last_output + 1, ptype] += len
written[first_output:last_output + 1, ptype] += len
free[first_output:last_output + 1, ptype] -= len
for i in range(first_output, last_output + 1):
for block in snap.reader.schema:
outputs[i].alloc(block, ptype)
for block in snap.reader.schema:
if (ptype, block) not in snap: continue
towrite = select(snap, ptype, block, mask)
for id, i, l, o in table:
outputs[id][ptype, block][o:o + l] = towrite[i:i + l]
with pool.lock:
writing[first_output:last_output + 1, ptype] -= len
for i, output in enumerate(outputs):
if (free[i] == 0).all() and (
writing[i]
== 0).all() and outputs[i] is not None:
print output[0, 'mass']
output.save_all()
outputs[i] = None
output = None
print 'saving ', i
return
pool.map(work, fids)
import numpy
import os.path
import sharedmem
idsfile = argv[1]
sfgasdir = argv[2]
first = Snapshot(idsfile % 0,
'cmugadget.GroupIDs',
idtype='u8',
floattype='f4')
sfgasid = numpy.fromfile(sfgasdir + '/sfgasid', dtype='u8')
sfgassfr = numpy.fromfile(sfgasdir + '/sfgassfr', dtype='f4')
matchedfile = file(sfgasdir + '/matchedsfr', 'w')
matchedidfile = file(sfgasdir + '/matchedid', 'w')
with sharedmem.Pool(use_threads=False) as pool:
def work(i):
snap = Snapshot(idsfile % i,
'cmugadget.GroupIDs',
idtype='u8',
floattype='f4')
id = snap[0, 'id']
result = numpy.zeros(len(id), dtype='f4')
lookedid = numpy.zeros(len(id), dtype='u8')
ind = sfgasid.searchsorted(id)
ind.clip(0, len(sfgasid) - 1, out=ind)
found = sfgasid[ind] == id
result[found] = sfgassfr[ind[found]]
# auto_fit = True fits the model on assignment
# verbose = 0 is silent
# verbose = 1 is a single print
# verbose = 2 is very verbose
if fixed_hrf is not False:
fit = og_nohrf.GaussianFit(model, vx, grids, bounds, Ns=Ns,
voxel_index=(ii, 1, 1), auto_fit=True, verbose=2)
else:
fit = og.GaussianFit(model, vx, grids, bounds, Ns=Ns,
voxel_index=(ii, 1, 1), auto_fit=True, verbose=2)
return (ii, vx) + tuple(fit.overloaded_estimate) + (fit.prediction,)
if mps == 1:
voxs = [fit_voxel((ii, vx, js)) for (ii, vx, js) in zip(range(len(bold)), bold, stim_json)]
else:
tups = list(zip(range(len(bold)), bold, stim_json))
with sharedmem.Pool(np=mps) as pool:
voxs = pool.map(fit_voxel, tups)
voxs = list(sorted(voxs, key=lambda tup:tup[0]))
# Update the results to match the x0/y0, sigma style used by prfanalyze
all_fields = ('index','voxel') + fields + ('pred',)
res = {k:np.asarray([u[ii] for u in voxs]) for (ii,k) in enumerate(all_fields)}
rr = {}
rr['centerx0'] = np.cos(res['theta']) * res['rho']
rr['centery0'] = -np.sin(res['theta']) * res['rho']
rr['sigmamajor'] = res['sigma']
rr['sigmaminor'] = res['sigma']
rr['beta'] = res['beta']
rr['baseline'] = res['baseline']
if fixed_hrf is False: rr['hrfdelay'] = res['hrfdelay']
# Export the files
for (k,v) in six.iteritems(rr):