def analyze(self):
if self.p1.image.ndim == 2:
self.viewstack()
self.p1.play(0)
df = pd.DataFrame()
peakCount = []
roiSPosx, roiSPosy = self.p1.roi.pos()
rows = 22
columns = 8
with pg.ProgressDialog("Scanning...", 0, rows*columns) as dlg:
for i in range(columns):
for j in range(rows):
time, data = self.p1.roiCurve1.getData()
df[str((i,j))] = pd.Series(data)
roiPosx, roiPosy = self.p1.roi.pos()
roiPosx += 7.5
self.p1.roi.setPos([roiPosx,roiPosy])
dlg += 1
roiPosx = roiSPosx
roiPosy += 20
self.p1.roi.setPos([roiPosx,roiPosy])
self.win = pg.GraphicsWindow(title="pyqtgraph example: Linked Views")
self.win.resize(800,800)
self.win.scene().sigMouseClicked.connect(self.plotzoom)
plotlist = [[]]
count = 0
with pg.ProgressDialog("Analyzing...", 0, 20*8) as dlg:
for i,x in enumerate(df.keys()):
if i%10 == 0:
self.win.nextRow()
plotTemp = self.win.addPlot()
plotTemp.plot(df[x],pen = pg.intColor(i, 100))
plotTemp.hideAxis('bottom')
plotTemp.hideAxis('left')
plotlist.append(plotTemp)
try:
datastart, dataend, datalimit = pfz.findpeaks(df[x],12)
count += len(datastart)
peakCount.append(len(datastart))
point = pg.PlotDataItem(x = datastart, y = df[x][datastart],
pen = None, symbol = 'o',
symbolBrush = pg.intColor(i, 100),
symbolSize=6)
plotTemp.addItem(point)
except IndexError:
pass
except ValueError:
pass
dlg += 1
self.p1.roi.setPos([roiSPosx,roiSPosy])
print('Events per file per ZMW = ' + str(count/(len(df.keys()))))
filebase = str(os.path.splitext(self.datafilename)[0])
np.savetxt(filebase + 'peakCount.txt', peakCount, delimiter = ',')
def download(url, dest, chunksize=1000000):
"""Download a file from *url* and save it to *dest*, while displaying a
progress bar.
"""
req = urlopen(url)
size = req.info().get('content-length')
size = 0 if size is None else int(size)
tmpdst = dest + '.partial'
fh = open(tmpdst, 'wb')
with pg.ProgressDialog("Downloading\n%s" % url, maximum=size,
nested=True) as dlg:
try:
tot = 0
while True:
chunk = req.read(chunksize)
if chunk == '':
break
fh.write(chunk)
tot += len(chunk)
dlg.setValue(tot)
if dlg.wasCanceled():
raise Exception("User cancelled download.")
os.rename(tmpdst, dest)
finally:
if os.path.isfile(tmpdst):
os.remove(tmpdst)
def __init__(self, message, maximum, update_interval=0.1, mode=None):
self.message = message
self.maximum = maximum
self.value = 0
if mode is None:
mode = interactive_mode()
self.mode = mode
self.update_dt = update_interval
self._last_update = 0
self._need_update = True
if self.mode == 'qt':
import pyqtgraph as pg
self.dlg = pg.ProgressDialog(self.message, maximum=1000)
elif self.mode == 'file':
print(message)
sys.stdout.flush()
# logging to file; don't need frequent updates.
self.update_dt = 30.0
else:
print(message)
sys.stdout.flush()
self.last_line_len = 0
def fit_clicked(self):
from synaptic_release import ReleaseModel
self.fit_plot.clear()
self.fit_plot.show()
n_sets = len(self.event_sets)
self.fit_plot.set_shape(n_sets, 1)
l = self.fit_plot[0, 0].legend
if l is not None:
l.scene.removeItem(l)
self.fit_plot[0, 0].legend = None
self.fit_plot[0, 0].addLegend()
for i in range(n_sets):
self.fit_plot[i, 0].setXLink(self.fit_plot[0, 0])
spike_sets = []
for i, evset in enumerate(self.event_sets):
evset = evset[2][:-1] # select events, skip last row (average)
x = np.array([ev['spike_time'] for ev in evset])
y = np.array([ev['amp'] for ev in evset])
x -= x[0]
x *= 1000
y /= y[0]
spike_sets.append((x, y))
self.fit_plot[i, 0].plot(x / 1000., y, pen=None, symbol='o')
model = ReleaseModel()
dynamics_types = ['Dep', 'Fac', 'UR', 'SMR', 'DSR']
for k in dynamics_types:
model.Dynamics[k] = 0
fit_params = []
with pg.ProgressDialog("Fitting release model..", 0,
len(dynamics_types)) as dlg:
for k in dynamics_types:
model.Dynamics[k] = 1
fit_params.append(model.run_fit(spike_sets))
dlg += 1
if dlg.wasCanceled():
return
max_color = len(fit_params) * 1.5
for i, params in enumerate(fit_params):
for j, spikes in enumerate(spike_sets):
x, y = spikes
t = np.linspace(0, x.max(), 1000)
output = model.eval(x, params.values())
y = output[:, 1]
x = output[:, 0] / 1000.
self.fit_plot[j, 0].plot(x,
y,
pen=(i, max_color),
name=dynamics_types[i])
def runManyStages(i):
"""Iterate over runStage() 3 times while incrementing a progress bar.
"""
with pg.ProgressDialog("Running stage %s.." % i, maximum=3, nested=True, wait=0) as dlg:
for j in range(1,4):
runStage('%d.%d' % (i, j))
dlg += 1
if dlg.wasCanceled():
print("Canceled stage %s" % i)
break
def runStage(i):
"""Waste time for 2 seconds while incrementing a progress bar.
"""
with pg.ProgressDialog("Running stage %s.." % i, maximum=100, nested=True) as dlg:
for j in range(100):
time.sleep(0.02)
dlg += 1
if dlg.wasCanceled():
print("Canceled stage %s" % i)
break
def fit_responses(self, latency=None):
if latency is None:
latency_window = [0.5e-3, 10e-3]
else:
latency_window = [latency - 100e-6, latency + 100e-6]
with pg.ProgressDialog("curve fitting..",
maximum=len(modes) * len(holdings)) as dlg:
self.last_fit = {}
for mode in modes:
for holding in holdings:
self.fit_pass = False
sign = self.signs[mode][holding].get(
self.ctrl_panel.user_params['Synapse call'],
self.signs[mode][holding].get(
self.ctrl_panel.user_params['Polysynaptic call'],
0))
# ofp, x_offset, best_fit = fit_avg_response(self.traces, mode, holding, latency, sign)
prs = self.sorted_responses[mode, holding]['qc_pass']
if len(prs) == 0:
dlg += 1
continue
fit, avg = fit_avg_pulse_response(
prs, latency_window=latency_window, sign=sign)
fit_ts = avg.copy(data=fit.best_fit)
self.last_fit[mode, holding] = fit
self.initial_fit_parameters[mode][str(
holding)]['xoffset'] = latency
self.output_fit_parameters[mode][str(
holding)]['nrmse'] = fit.nrmse()
self.output_fit_parameters[mode][str(holding)].update(
fit.best_values)
self.fit_pass = fit.nrmse() < self.nrmse_thresh
self.ctrl_panel.output_params.child(
'Fit parameters',
str(holding) + ' ' + mode.upper(),
'Fit Pass').setValue(self.fit_pass)
if mode == 'vc':
self.vc_plot.plot_fit(fit_ts, holding, self.fit_pass)
elif mode == 'ic':
self.ic_plot.plot_fit(fit_ts, holding, self.fit_pass)
dlg += 1
if dlg.wasCanceled():
raise Exception("User canceled fit")
self.fit_params['initial'] = self.initial_fit_parameters
self.fit_params['fit'] = self.output_fit_parameters
self.ctrl_panel.update_fit_params(self.fit_params['fit'])
self.generate_warnings()
def scrapePeaks(self):
"""Some peak-finding function with output filtering based on SNR"""
self.prepGuiParameters()
self.pk_extracted_by_condi = {}
for _condi in self.tracedata.keys():
maxVal = len(self.tPeaks)
ROI_df = self.tracedata[_condi]
#print (ROI_df)
peaksList = []
progMsg = "Get {0} peaks, {1} set..".format(maxVal, _condi)
with pg.ProgressDialog(progMsg, 0, maxVal) as dlg:
dlg.setMinimumWidth(300)
for t in self.tPeaks:
dlg += 1
idx = np.searchsorted(ROI_df.index, t)
# avoid falling off start or end of columns
e = max(idx - self.psr, 0)
# zero biased so add one.
l = min(idx + self.psr + 1, len(ROI_df.index))
print(t, e, idx, l, ROI_df.iloc[e:l])
p = ROI_df.iloc[e:l].max().to_frame().transpose()
peaksList.append(p)
#stick rows together (all have same zero index...)
peaksdf = pd.concat(peaksList)
# Overwrite index with the original peak positions
# (somewhat inexact because of the 'range')
peaksdf.index = self.tPeaks
self.pk_extracted_by_condi[_condi] = peaksdf
# yes, output may be modified below
self.peaksScraped = True
self.acceptBtn.setEnabled(True)
self.getRundownBtn.setEnabled(True)
self.noiseRB.setEnabled(True)
self.noiseSB.setEnabled(True)
self.excludedListedByCondi = {}
if self.ignore:
# freshly excludedList peaks from traces with low SNR
self.maskLowSNR()
self.splitAllowedExcluded()
def download_and_cache(self, resolution=None):
"""Download atlas data, convert to intermediate format, and store in cache
folder.
"""
if resolution is None:
dlg = AtlasResolutionDialog(self.available_resolutions,
self.cached_resolutions.keys())
dlg.exec_()
resolution = dlg.selected_resolution()
if resolution is None:
raise Exception("No atlas resolution selected.")
cache_path = self.cache_path(resolution)
if not os.path.exists(cache_path):
os.makedirs(cache_path)
with pg.ProgressDialog("Preparing %dum CCF data" % resolution,
maximum=6,
nested=True) as dlg:
image_url = self.image_url.format(resolution=resolution)
image_file = os.path.join(cache_path, image_url.split('/')[-1])
image_cache = os.path.join(cache_path, "image.ma")
if not os.path.exists(image_file):
download(image_url, image_file)
dlg += 1
label_url = self.label_url.format(resolution=resolution)
label_file = os.path.join(cache_path, label_url.split('/')[-1])
label_cache = os.path.join(cache_path, "label.ma")
if not os.path.exists(label_file):
download(label_url, label_file)
dlg += 1
onto_file = os.path.join(cache_path, 'ontology.json')
if not os.path.exists(onto_file):
download(self.ontology_url, onto_file)
self.load_image_data(image_file)
dlg += 1
write_file(self.image, image_cache)
dlg += 1
self.load_label_data(label_file, onto_file)
dlg += 1
write_file(self.label, label_cache)
dlg += 1
self.cached_resolutions[resolution] = (image_cache, label_cache)
return resolution
def __init__(self,
tasks=None,
workers=None,
block=True,
progressDialog=None,
randomReseed=True,
**kwds):
"""
=============== ===================================================================
Arguments:
tasks list of objects to be processed (Parallelize will determine how to
distribute the tasks). If unspecified, then each worker will receive
a single task with a unique id number.
workers number of worker processes or None to use number of CPUs in the
system
progressDialog optional dict of arguments for ProgressDialog
to update while tasks are processed
randomReseed If True, each forked process will reseed its random number generator
to ensure independent results. Works with the built-in random
and numpy.random.
kwds objects to be shared by proxy with child processes (they will
appear as attributes of the tasker)
=============== ===================================================================
"""
## Generate progress dialog.
## Note that we want to avoid letting forked child processes play with progress dialogs..
self.showProgress = False
if progressDialog is not None:
self.showProgress = True
if isinstance(progressDialog, basestring):
progressDialog = {'labelText': progressDialog}
import pyqtgraph as pg
self.progressDlg = pg.ProgressDialog(**progressDialog)
if workers is None:
workers = self.suggestedWorkerCount()
if not hasattr(os, 'fork'):
workers = 1
self.workers = workers
if tasks is None:
tasks = range(workers)
self.tasks = list(tasks)
self.reseed = randomReseed
self.kwds = kwds.copy()
self.kwds['_taskStarted'] = self._taskStarted
def baselineIterator(data, lam, p, niter=20):
""" iterate baseline subtraction over dictionary of dataframes of ROI traces """
#is there a problem with running it twice? No, the problem was peaks were being chucked out
bdata = {}
for _set, df in data.items():
print(
"Auto baseline for {0} set. lambda: {1:.3f} and p: {2:.3f}".format(
_set, lam, p))
maxVal = len(df.columns)
progMsg = "Auto baseline for {0} traces".format(maxVal)
with pg.ProgressDialog(progMsg, 0, maxVal) as dlg:
dlg.setMinimumWidth(300)
for col in df:
dlg += 1
y = np.asarray(df[col])
# subtract appropriate baseline from each column of df
df[col] -= baseline_als(y, lam, p, niter=20, quiet=True)
bdata[_set] = df
return bdata
def createFile(finalSize=2000000000):
"""Create a large HDF5 data file for testing.
Data consists of 1M random samples tiled through the end of the array.
"""
chunk = np.random.normal(size=1000000).astype(np.float32)
f = h5py.File('test.hdf5', 'w')
f.create_dataset('data', data=chunk, chunks=True, maxshape=(None, ))
data = f['data']
nChunks = finalSize // (chunk.size * chunk.itemsize)
with pg.ProgressDialog("Generating test.hdf5...", 0, nChunks) as dlg:
for i in range(nChunks):
newshape = [data.shape[0] + chunk.shape[0]]
data.resize(newshape)
data[-chunk.shape[0]:] = chunk
dlg += 1
if dlg.wasCanceled():
f.close()
os.remove('test.hdf5')
sys.exit()
dlg += 1
f.close()
"""Iterate over runStage() 3 times while incrementing a progress bar.
"""
with pg.ProgressDialog("Running stage %s.." % i,
maximum=3,
nested=True,
wait=0) as dlg:
for j in range(1, 4):
runStage('%d.%d' % (i, j))
dlg += 1
if dlg.wasCanceled():
print("Canceled stage %s" % i)
break
with pg.ProgressDialog("Doing a multi-stage process..",
maximum=5,
nested=True,
wait=0) as dlg1:
for i in range(1, 6):
if i == 3:
# this stage will have 3 nested progress bars
runManyStages(i)
else:
# this stage will have 2 nested progress bars
runStage(i)
dlg1 += 1
if dlg1.wasCanceled():
print("Canceled process")
break
def on_calcAll_requested(self):
'''
Calculation of gradient for all timesteps has been requested
'''
try:
info = self.ctrlWidget().selectedWellsInfo()
if len(info.keys()) < 3:
raise ValueError(
'Select at least 3 wells to calculate gradient')
datetimeColName = self.ctrlWidget().comboBox_Datetime.currentText()
if not datetimeColName:
return
# now generate long dataframe
df = self.inputValues()[
'data'] #pd.DataFrame in the input terminal `data`
All_df = pd.DataFrame({
datetimeColName:
df[datetimeColName],
'gradient':
np.zeros(len(df.index)),
'direction(degrees North)':
np.zeros(len(df.index))
})
# then get the information
dictForDf = {
'well': [],
'x': [],
'y': []
} # dictionary that will be used for a convinient creation of the DataFrame that is requered for calculations
for wellName, wellInfo in info.iteritems():
dictForDf['well'].append(wellName)
dictForDf['x'].append(wellInfo['x'])
dictForDf['y'].append(wellInfo['y'])
with pg.ProgressDialog(
"Calculating gradient for All timesteps {0}".format(
len(All_df.index)), 0, len(All_df.index)) as dlg:
for row_i in df.index:
row = df.loc[row_i]
z = np.zeros(len(dictForDf['well']))
for i, well_n in enumerate(dictForDf['well']):
z[i] = float(row[info[well_n]['z']])
x = np.array(dictForDf['x'])
y = np.array(dictForDf['y'])
_, gradient, angle = devlin2003(np.matrix([x, y, z]).T)
All_df.loc[row_i, 'gradient'] = gradient
All_df.loc[row_i,
'direction(degrees North)'] = angle2bearing(
angle, origin='N')[0]
dlg += 1
del z
if dlg.wasCanceled():
del All_df
All_df = None
break
dlg += 1
self.setOutput(All=All_df)
self.clearException()
except:
self.setOutput(All=None)
self.ctrlWidget().clear(clearTable=False)
self.setException(sys.exc_info())
self.sigOutputChanged.emit(
self) ## triggers flowchart to propagate new data
def process(self, coord, data):
if data is not None:
colname = [
col for col in data.columns if isNumpyDatetime(data[col].dtype)
]
self._ctrlWidget.param('Datetime').setLimits(colname)
self.data = data
else:
self.data = None
return dict(this=None, All=self.All_out)
if coord is not None:
colname = [
col for col in coord.columns
if isNumpyNumeric(coord[col].dtype)
]
self._ctrlWidget.param('coords_grp', 'x').setLimits(colname)
self._ctrlWidget.param('coords_grp', 'y').setLimits(colname)
self.CW().disconnect_valueChanged2upd(self.CW().param(
'coords_grp', 'x'))
self.CW().disconnect_valueChanged2upd(self.CW().param(
'coords_grp', 'y'))
self.CW().param('coords_grp', 'x').setValue(colname[0])
self.CW().param('coords_grp', 'y').setValue(colname[1])
self.CW().connect_valueChanged2upd(self.CW().param(
'coords_grp', 'x'))
self.CW().connect_valueChanged2upd(self.CW().param(
'coords_grp', 'y'))
else:
return dict(this=None, All=self.All_out)
# now make sure All well specified in `coord` dataframe are found in `data`
well_names = coord.index.values
for well_n in well_names:
if well_n not in data.columns:
raise ValueError(
'Well named `{0}` not found in `data` but is declared in `coords`'
.format(well_n))
kwargs = self.ctrlWidget().prepareInputArguments()
# select row whith user-specified datetime `timestep`
row = data.loc[data[kwargs['datetime']] == kwargs['t']]
if row.empty:
raise IndexError(
'Selected timestep `{0}` not found in `data`s column {1}. Select correct one'
.format(kwargs['t'], kwargs['datetime']))
# now prepare dataframe for devlin calculations
df = coord.copy()
df['z'] = np.zeros(len(df.index))
for well_n in well_names:
df.loc[well_n, 'z'] = float(row[well_n])
gradient, direction = devlin2003pandas(df, kwargs['x'], kwargs['y'],
'z')
self.CW().param('grad').setValue(gradient)
self.CW().param('angle').setValue(direction)
# here we will generate large dataset of all timesteps
if self.CW().CALCULATE_ALL:
# now generate long dataframe
All = pd.DataFrame({
kwargs['datetime']:
data[kwargs['datetime']],
'gradient':
np.zeros(len(data.index)),
'direction(degrees North)':
np.zeros(len(data.index))
})
self.All_out = All # pointer
with pg.ProgressDialog(
"Calculating gradient for All timesteps {0}".format(
len(All.index)), 0, len(All.index)) as dlg:
for row_i in data.index:
row = data.loc[row_i]
z = np.zeros(len(coord.index))
for i, well_n in enumerate(well_names):
z[i] = float(row[well_n])
x = coord[kwargs['x']].values
y = coord[kwargs['y']].values
_, gradient, angle = devlin2003(np.matrix([x, y, z]).T)
All.loc[row_i, 'gradient'] = gradient
All.loc[row_i, 'direction(degrees North)'] = angle2bearing(
angle, origin='N')[0]
dlg += 1
del z
if dlg.wasCanceled():
del All
self.All_out = None
break
#return dict(df=df, All=self.All_out)
dlg += 1
return dict(this=df, All=self.All_out)
def run(self):
progress = pg.ProgressDialog("simuliert...")
evaluated = self.mainNodeScene.evaluateGraph()
progress += 10
fc = fl.Flowchart()
# add nodes and build terminals
for node in self.mainNodeScene.scene().nodes.values():
fc.addNode(node.userData, node.userData.name())
for socket in node.sockets:
node.userData.addInput(socket + "/S/")
for plug in node.plugs:
node.userData.addOutput(plug + "/P/")
progress += 10
# build connections
for connection in evaluated:
fc.connectTerminals(
connection[0][0].userData.terminals[connection[0][2] + "/P/"],
connection[1][0].userData.terminals[connection[1][2] + "/S/"])
progress += 10
# # debug pg fc interface
# dialog = QtWidgets.QDialog()
# layout = QtWidgets.QVBoxLayout()
# layout.addWidget(fc.widget())
# dialog.setLayout(layout)
# dialog.show()
# dialog.exec_()
# generate sim file
shutil.copyfile(os.path.abspath(Config.template),
Config.TEMP_DIR + Config.template)
# TODO Apply Settings
# run fc
try:
fc.process()
progress += 10
except Exception as e:
print("\n>>> Cannot process fc")
print(e)
print("\n")
else:
try:
sim = Interface.Interface(Config.TEMP_DIR + Config.template)
sim.runSim()
except Exception as e:
print("\n>>> Cannot execute sim")
print(e)
print("\n")
else:
progress += 10
data = sim.readRaw()
progress += 10
for node in self.mainNodeScene.scene().nodes.values():
if isinstance(node.userData, NodeItem.PlotNode):
node.userData.setData(data)
self.plotViewer.plt.vb.autoRange()
finally:
# remove node terminals, cannot use for loop, since terminals dict will be mutated in iterations
for node in self.mainNodeScene.scene().nodes.values():
while len(node.userData.terminals) > 1:
node.userData.removeTerminal(node.userData.terminals[next(
iter(node.userData.terminals))])
# cleanup
progress.setValue(100)
fc.clear()
fc.close()
del fc
if bestval is None or diff < bestval:
bestval = diff
best = x
bestn = (fp + fn) / 2.
return best, bestn
algorithms = [
('Poisson Score', PoissonScore.score),
('Poisson Score + Amp', PoissonAmpScore.score),
#('Poisson Multi', PoissonRepeatScore.score),
#('Poisson Multi + Amp', PoissonRepeatAmpScore.score),
]
app = pg.mkQApp()
win = pg.GraphicsWindow(border=0.3)
with pg.ProgressDialog('processing..', maximum=len(tests)) as dlg:
for i in range(len(tests)):
first = (i == 0)
last = (i == len(tests) - 1)
if first:
evLabel = win.addLabel('Event amplitude',
angle=-90,
rowspan=len(tests))
evPlt = win.addPlot()
plots = []
for title, fn in algorithms:
if first:
label = win.addLabel(title, angle=-90, rowspan=len(tests))
plt = win.addPlot()
'xoffset': (50, 0, 100),
'yoffset': 0,
'amp': (0, -50, 50),
'sigma': (5, 1, 20)
})
plt.plot(x, fit.best_fit, pen='r')
# Instead, brute-force search for the best fit over multiple ranges for xoffset and amp:
amp = [{'amp': (-10, -50, 0)}, {'amp': (10, 0, 50)}]
xoffset = [{'xoffset': (x + 5, x, x + 10)} for x in range(0, 100, 10)]
# Total number of fit attempts is len(amp) * len(xoffset) = 20
search = SearchFit(model, [amp, xoffset],
params={
'sigma': (5, 1, 20),
'yoffset': 0
},
x=x,
data=y)
# Optionally, let the user know how the fit is progressing:
with pg.ProgressDialog("Fitting...", maximum=len(search)) as dlg:
for result in search.iter_fit():
print("Init params this iteration:", result['params'])
dlg += 1
if dlg.wasCanceled():
raise Exception("User canceled fit")
plt.plot(x, search.best_result.best_fit, pen='g')
print("Best fit parameters:", search.best_result.best_values)
def export(cls, file_path, project):
"""
ReportExporter subclasses should not override this method.
export() is called by the MarkWrite application to create and
save a report.
Example usage:
MyReportExporter.export(path_to_output_file, markwrite_app.project)
:param file_path: Absolute file path to save the report to.
:param project: The MarkWriteProject instance that will be used
for data and further calculations by the datarows
method.
:return: None
"""
cls.project = project
try:
import pyqtgraph
with codecs.open(file_path, "w", "utf-8") as f:
rp = cls.preamble()
if len(rp) > 0:
# TODO: Should split into lines and prefix each line
# with a 'comment' character(s), like '#' is used in python
f.write(cls.preamble() + cls.nl)
if len(cls.columnnames()) > 0:
f.write(cls.sep.join(cls.columnnames()) + cls.nl)
rowformatstr = cls.rowformat()
ri = 0
if project._mwapp:
with pyqtgraph.ProgressDialog(cls.progress_dialog_title,
0,
cls.datarowcount(),
cancelText=None) as dlg:
for row in cls.datarows():
row.extend([
cls.missingval,
] * (cls.columncount() - len(row)))
f.write(rowformatstr.format(*row))
if ri % cls.progress_update_rate == 0:
dlg.setValue(ri)
if dlg.wasCanceled():
# TODO: Should the incomplete report file be deleted
# if dialog is cancelled?
break
ri += 1
else:
for row in cls.datarows():
row.extend([
cls.missingval,
] * (cls.columncount() - len(row)))
f.write(rowformatstr.format(*row))
ri += 1
return ri
except:
import traceback
traceback.print_exc()
finally:
cls.project = None
return 0
## - once without Parallelize
## - once with Parallelize, but forced to use a single worker
## - once with Parallelize automatically determining how many workers to use
##
tasks = range(10)
results = [None] * len(tasks)
results2 = results[:]
results3 = results[:]
size = 2000000
pg.mkQApp()
### Purely serial processing
start = time.time()
with pg.ProgressDialog('processing serially..', maximum=len(tasks)) as dlg:
for i, x in enumerate(tasks):
tot = 0
for j in range(size):
tot += j * x
results[i] = tot
dlg += 1
if dlg.wasCanceled():
raise Exception('processing canceled')
print("Serial time: %0.2f" % (time.time() - start))
### Use parallelize, but force a single worker
### (this simulates the behavior seen on windows, which lacks os.fork)
start = time.time()
with mp.Parallelize(
enumerate(tasks),
def readNRRDLabels(nrrdFile=None, ontologyFile=None):
"""
Download label files from:
http://help.brain-map.org/display/mouseconnectivity/API#API-DownloadAtlas
Download ontology files from:
http://api.brain-map.org/api/v2/structure_graph_download/1.json
see:
http://help.brain-map.org/display/api/Downloading+an+Ontology%27s+Structure+Graph
http://help.brain-map.org/display/api/Atlas+Drawings+and+Ontologies#AtlasDrawingsandOntologies-StructuresAndOntologies
This method compresses the annotation data down to a 16-bit array by remapping
the larger annotations to smaller, unused values.
"""
global onto, ontology, data, mapping, inds, vxsize, info, ma
import nrrd
if nrrdFile is None:
displayMessage('Select NRRD annotation file')
nrrdFile = QtGui.QFileDialog.getOpenFileName(
None, "Select NRRD annotation file")
if ontologyFile is None:
displayMessage('Select ontology file (json)')
ontoFile = QtGui.QFileDialog.getOpenFileName(
None, "Select ontology file (json)")
with pg.ProgressDialog("Loading annotation file...", 0, 5, wait=0) as dlg:
print "Loading annotation file..."
app.processEvents()
# Read ontology and convert to flat table
onto = json.load(open(ontoFile, 'rb'))
onto = parseOntology(onto['msg'][0])
l1 = max([len(row[2]) for row in onto])
l2 = max([len(row[3]) for row in onto])
ontology = np.array(onto,
dtype=[('id', 'int32'), ('parent', 'int32'),
('name', 'S%d' % l1),
('acronym', 'S%d' % l2), ('color', 'S6')])
if dlg.wasCanceled():
return
dlg += 1
# read annotation data
data, header = nrrd.read(nrrdFile)
if dlg.wasCanceled():
return
dlg += 1
# data must have axes (anterior, dorsal, right)
# rearrange axes to fit -- CCF data comes in (posterior, inferior, right) order.
data = data[::-1, ::-1, :]
if dlg.wasCanceled():
return
dlg += 1
# compress down to uint16
print "Compressing.."
u = np.unique(data)
# decide on a 32-to-64-bit label mapping
mask = u <= 2**16 - 1
next_id = 2**16 - 1
mapping = OrderedDict()
inds = set()
for i in u[mask]:
mapping[i] = i
inds.add(i)
with pg.ProgressDialog("Remapping annotations to 16-bit...",
0, (~mask).sum(),
wait=0) as dlg:
app.processEvents()
for i in u[~mask]:
while next_id in inds:
next_id -= 1
mapping[i] = next_id
inds.add(next_id)
data[data == i] = next_id
ontology['id'][ontology['id'] == i] = next_id
ontology['parent'][ontology['parent'] == i] = next_id
if dlg.wasCanceled():
return
dlg += 1
data = data.astype('uint16')
mapping = np.array(list(mapping.items()))
# voxel size in um
vxsize = 1e-6 * float(header['space directions'][0][0])
info = [{
'name': 'anterior',
'values': np.arange(data.shape[0]) * vxsize,
'units': 'm'
}, {
'name': 'dorsal',
'values': np.arange(data.shape[1]) * vxsize,
'units': 'm'
}, {
'name': 'right',
'values': np.arange(data.shape[2]) * vxsize,
'units': 'm'
}, {
'vxsize': vxsize,
'ai_ontology_map': mapping,
'ontology': ontology
}]
ma = metaarray.MetaArray(data, info=info)
return ma
for method in methods:
dtype.extend(method.dtype)
name = method.name
dtype.extend([
(name+'_true_err', float),
]
for par_name in method.params:
dtype.append(
examples = np.empty(N, dtype=dtype)
with pg.ProgressDialog("making some noise..", maximum=N) as dlg:
for i in range(N):
ex = examples[i]
yoffset = np.random.uniform(-80e-3, -60e-3)
amp = np.random.uniform(-100e-3, 100e-3)
tau = np.random.uniform(5e-3, 500e-3)
x = yoffset, amp, tau
true_y = exp_fn(x, t)
y = true_y + make_noise(t)
ex['x'] = x
ex['y'] = y
ex['t'] = t
ex['true_y'] = true_y
