def remove_background_ipy(obj, **kwargs):
wdict = {}
left = ipywidgets.FloatText(disabled=True, description="Left")
right = ipywidgets.FloatText(disabled=True, description="Right")
link((obj, "ss_left_value"), (left, "value"))
link((obj, "ss_right_value"), (right, "value"))
fast = ipywidgets.Checkbox(description="Fast")
help = ipywidgets.HTML(
"Click on the signal figure and drag to the right to select a"
"range. Press `Apply` to remove the background in the whole dataset. "
"If fast is checked, the background parameters are estimated using a "
"fast (analytical) method that can compromise accuray. When unchecked "
"non linear least squares is employed instead.", )
wdict["help"] = help
help = ipywidgets.Accordion(children=[help])
help.set_title(0, "Help")
close = ipywidgets.Button(
description="Close",
tooltip="Close widget and remove span selector from the signal figure."
)
apply = ipywidgets.Button(
description="Apply",
tooltip="Remove the background in the whole dataset.")
polynomial_order = ipywidgets.IntText(description="Polynomial order")
background_type = enum2dropdown(obj.traits()["background_type"])
background_type.description = "Background type"
def enable_poly_order(change):
if change.new == "Polynomial":
polynomial_order.layout.display = ""
else:
polynomial_order.layout.display = "none"
background_type.observe(enable_poly_order, "value")
link((obj, "background_type"), (background_type, "value"))
# Trigger the function that controls the visibility of poly order as
# setting the default value doesn't trigger it.
class Dummy:
new = background_type.value
enable_poly_order(change=Dummy())
link((obj, "polynomial_order"), (polynomial_order, "value"))
link((obj, "fast"), (fast, "value"))
wdict["left"] = left
wdict["right"] = right
wdict["fast"] = fast
wdict["polynomial_order"] = polynomial_order
wdict["background_type"] = background_type
wdict["apply_button"] = apply
box = ipywidgets.VBox([
left,
right,
background_type,
polynomial_order,
fast,
help,
ipywidgets.HBox((apply, close)),
])
def on_apply_clicked(b):
obj.apply()
box.close()
apply.on_click(on_apply_clicked)
def on_close_clicked(b):
obj.span_selector_switch(False)
box.close()
close.on_click(on_close_clicked)
return {
"widget": box,
"wdict": wdict,
}
def display_cluster(model,
X,
X_test,
y_test,
leaves_test,
clusters_test,
n_clusters,
encode_features=None,
present_features=None,
feature_description=None):
orig_X = X
orig_X_test = X_test
if present_features:
orig_X = present_features(orig_X)
orig_X_test = present_features(orig_X_test)
if encode_features:
X = encode_features(X)
X_test = encode_features(X_test)
vs = model.predict_proba(X)
vs_test = model.predict_proba(X_test)
pred_test = model.predict(X_test)
influential_paths = InfluentialPaths(model, leaves_test, X_test,
clusters_test, n_clusters)
def cluster_desc(c):
orig_X_cluster = orig_X_test[clusters_test == c]
X_cluster = X_test[clusters_test == c]
y_cluster = y_test[clusters_test == c]
vs_cluster = vs_test[clusters_test == c]
pred_cluster = pred_test[clusters_test == c]
winner = mode(pred_cluster)[0][0]
winner_idx = np.where(model.classes_ == winner)[0][0]
mean = np.mean(vs_cluster[:, winner_idx])
distances = [{
'Column': col,
'Distance': distance(orig_X[col], orig_X_cluster[col])
} for col in orig_X.columns]
distances = pd.DataFrame(distances).sort_values('Distance',
ascending=False)
distances = distances.set_index('Column')
return {
'orig_X':
orig_X_cluster,
'X':
X_cluster,
'y':
y_cluster,
'votes':
vs_cluster,
'mean':
mean,
'predictions':
pred_cluster,
'winner':
winner,
'winner_idx':
winner_idx,
'Size':
len(X_cluster),
'Accuracy':
np.mean(
pred_cluster == y_cluster) if len(X_cluster) != 0 else '--',
'distances':
distances
}
cluster_descs = [cluster_desc(c) for c in range(n_clusters)]
with pd.option_context('display.max_rows', None):
display(
pd.DataFrame([
OrderedDict([('Size', d['Size']), ('Accuracy', d['Accuracy']),
('Winner', d['winner']),
('Mean Winner Prob.', d['mean'])])
for d in cluster_descs
]))
cluster_widget = widgets.BoundedIntText(value=0,
min=0,
max=n_clusters,
step=1,
description='Cluster')
column_widget = widgets.Dropdown(options=orig_X.columns,
description='Column')
rank_widget = widgets.IntText(value=0, description='Path rank')
def do_display(cluster, curr_col, rank):
desc = cluster_descs[cluster]
orig_X_cluster = desc['orig_X']
X_cluster = desc['X']
y_cluster = desc['y']
vs_cluster = desc['votes']
winner_cluster_idx = desc['winner_idx']
if feature_description:
print(feature_description[curr_col])
print('Cluster accuracy: %.03f' % model.score(X_cluster, y_cluster))
print('Cluster size: %d/%d' % (len(X_cluster), len(X_test)))
print('Cluster mean winner probability: %.03f' % desc['mean'])
distances = desc['distances']
out1 = widgets.Output()
with out1:
with pd.option_context('display.max_rows', None):
display(distances)
fig, axes = plt.subplots(figsize=(5, 5))
axes.set_title('Distribution of scores')
axes.grid()
labels = ['Cluster %d' % cluster, 'General']
axes.hist(
[vs_cluster[:, winner_cluster_idx], vs[:, winner_cluster_idx]],
density=True,
label=labels)
axes.legend()
out2 = widgets.Output()
with out2:
plt.show(fig)
ws = [out1, out2]
if curr_col:
plt.ioff()
fig, axes = plt.subplots(figsize=(5, 5))
distr_cluster = orig_X_cluster[curr_col]
distr = orig_X[curr_col]
distance = distances.loc[curr_col]
axes.set_title('Distribution of column %s\nDistance %f' %
(curr_col, distance))
axes.grid()
axes.hist([distr_cluster, distr],
density=True,
bins=20,
label=labels)
if np.issubdtype(distr.dtype, np.number):
plt.xticks(rotation='horizontal')
else:
plt.xticks(rotation='vertical')
axes.legend()
plt.ion()
out3 = widgets.Output()
with out3:
plt.show(fig)
ws.append(out3)
display(widgets.HBox(ws, layout={'height': '700px'}))
display(influential_paths.scores[cluster].head(10))
display(influential_paths[cluster, rank])
w = widgets.interactive(do_display,
cluster=cluster_widget,
curr_col=column_widget,
rank=rank_widget)
display(w)
class ctfReview:
#Settings:
thumbNailSize = (256, 256)
rawThumbNailSize = (256, 256)
starFileName = 'selected_micrographs_ctf.star'
defaultValue = 99999999
#style and Layout
styleBasic = {'description_width': '120px'}
styleAdvanced = {'description_width': '160px'}
basicLayout = Layout(width='60%')
ctfLayout = Layout(width='95%')
advLayout = Layout(width='50%')
errorLayout = Layout(width='60%', border='2px solid red')
startButton = widgets.Button(description='Start review',
disabled=False,
button_style='',
tooltip='Start CTF review',
icon='')
resetFilterButton = widgets.Button(description='Reset',
disabled=False,
button_style='',
tooltip='Reset',
icon='')
applyFilterButton = widgets.Button(description='Apply',
disabled=False,
button_style='',
tooltip='Apply',
icon='')
saveButton = widgets.Button(description='Save',
disabled=False,
button_style='',
tooltip='Save filtered mrcs to .star file',
icon='check')
projectDirectory = widgets.Text(description='Project directory: ',
placeholder='Relion project directory',
value='',
disabled=True,
style=styleBasic,
layout=basicLayout)
gctfFolderName = widgets.Text(description='Folder: ',
value='',
disabled=True,
style=styleBasic,
layout=basicLayout)
errorText = widgets.Text(description='',
placeholder='Errors',
disabled=True,
style=styleBasic,
layout=basicLayout)
jobSelection = widgets.RadioButtons(options='',
description='Available jobs:',
disabled=False,
style=styleBasic,
layout=basicLayout)
defocusUFrom = widgets.FloatText(description='Defocus U from: ',
disabled=False,
style=styleAdvanced,
layout=advLayout)
defocusUTo = widgets.FloatText(description='to: ',
value=defaultValue,
disabled=False,
style=styleAdvanced,
layout=advLayout)
defocusVFrom = widgets.FloatText(description='Defocus V from: ',
disabled=False,
style=styleAdvanced,
layout=advLayout)
defocusVTo = widgets.FloatText(description='to: ',
value=defaultValue,
disabled=False,
style=styleAdvanced,
layout=advLayout)
defocusAFrom = widgets.FloatText(description='Defocus angle from: ',
disabled=False,
style=styleAdvanced,
layout=advLayout)
defocusATo = widgets.FloatText(description='to: ',
value=defaultValue,
disabled=False,
style=styleAdvanced,
layout=advLayout)
voltageFrom = widgets.FloatText(description='Voltage from: ',
disabled=False,
style=styleAdvanced,
layout=advLayout)
voltageTo = widgets.FloatText(description='to: ',
value=defaultValue,
disabled=False,
style=styleAdvanced,
layout=advLayout)
sphericalAberrationFrom = widgets.FloatText(
description='Spherical aberration from: ',
disabled=False,
style=styleAdvanced,
layout=advLayout)
sphericalAberrationTo = widgets.FloatText(description='to: ',
value=defaultValue,
disabled=False,
style=styleAdvanced,
layout=advLayout)
ampContrastFrom = widgets.FloatText(
description='Amplitude contrast from: ',
disabled=False,
style=styleAdvanced,
layout=advLayout)
ampContrastTo = widgets.FloatText(description='to: ',
value=defaultValue,
disabled=False,
style=styleAdvanced,
layout=advLayout)
magnificationFrom = widgets.FloatText(description='Magnification from: ',
disabled=False,
style=styleAdvanced,
layout=advLayout)
magnificationTo = widgets.FloatText(description='to: ',
value=defaultValue,
disabled=False,
style=styleAdvanced,
layout=advLayout)
detectPixSizeFrom = widgets.FloatText(
description='Detector pix size from: ',
disabled=False,
style=styleAdvanced,
layout=advLayout)
detectPixSizeTo = widgets.FloatText(description='to: ',
value=defaultValue,
disabled=False,
style=styleAdvanced,
layout=advLayout)
ctfFigMeritFrom = widgets.FloatText(
description='Ctf figure of merit from: ',
disabled=False,
style=styleAdvanced,
layout=advLayout)
ctfFigMeritTo = widgets.FloatText(description='to: ',
value=defaultValue,
disabled=False,
style=styleAdvanced,
layout=advLayout)
ctfMaxResFrom = widgets.FloatText(description='Ctf max resolution from: ',
disabled=False,
style=styleAdvanced,
layout=advLayout)
ctfMaxResTo = widgets.FloatText(description='to: ',
value=defaultValue,
disabled=False,
style=styleAdvanced,
layout=advLayout)
orderBy = widgets.RadioButtons(
options=['Defocus U', 'Figure of merit', 'Ctf max resolution'],
value='Defocus U',
description='Order by:',
disabled=False,
style=styleBasic,
layout=basicLayout)
totalMicrographs = widgets.Textarea(description='Total micrographs',
value='',
disabled=True,
style=styleBasic,
layout=basicLayout)
totalFilteredMicrographs = widgets.IntText(description='Total filtered: ',
value=0,
disabled=True,
style=styleBasic,
layout=basicLayout)
jobsBox = widgets.Output(layout={'border': '2px solid black'})
reviewBox = widgets.Output(layout={'border': '2px solid black'})
##Debug assistance
debug = widgets.Output(style=styleBasic, layout=Layout(width='90%'))
debugText = widgets.Textarea(description='Debugging:',
description_tooltip='Standard output',
disabled=False,
rows=10,
style=styleBasic,
layout=Layout(width='90%'))
# __init__() - initialise the class jobMaintenance
# Arguments:
# workFlow - an instance of the workFlow class
# contrastTransfer - an instance of the contrastTransfer class e.g. gctf_106
# showDebug - display debug fields
#
def __init__(self, workFlow, contrastTransfer, showDebug):
self.workFlow = workFlow
self.contrastTransfer = contrastTransfer
self.showDebug = showDebug
self.ctfReviewList = []
self.filteredCtfReviewList = []
# convertToPNG() - convert the Image object to a PNG image
# Arguments - im - image object
#
@debug.capture(clear_output=True)
def convertToPNG(self, im):
with BytesIO() as f:
im.save(f, format='PNG')
return f.getvalue()
# dynamicOnClick() - handles button actions for displayed CTF micrographs.
# Arguments - ctfData - a list containing the data created by gctf for a single micrograph
# - button - the button that was clicked
#
@debug.capture(clear_output=True)
def dynamicOnClick(self, ctfData, button):
if hasattr(button, 'showData'):
button.showData.clear_output()
with button.showData:
print('Defocus U: ' + str(ctfData['defocusU']) + '\n' + \
'Defocus V: ' + str(ctfData['defocusV']) + '\n' + \
'Defocus Angle: ' + str(ctfData['defocusAngle']) + '\n' + \
'Voltage: ' + str(ctfData['voltage']) + '\n' + \
'Spherical Aberration: ' + str(ctfData['sphericalAberration']) + '\n' + \
'Amplitude Contrast: ' + str(ctfData['ampContrast']) + '\n' + \
'Magnification: ' + str(ctfData['magnification']) + '\n' + \
'Detector Pixel Size: ' + str(ctfData['detectorPixelSize']) + '\n' + \
'Figure of Merit: ' + str(ctfData['figOfMerit']) + '\n' + \
'Ctf Max Resolution: ' + str(ctfData['maxResolution']))
if hasattr(button, 'showMrc'):
button.showMrc.clear_output()
with button.showMrc:
fileName = ctfData['micrographNameNoDW']
#Problem, MotionCorr2 created mrcs have an issue with the MAP ID string in the header, it should be 'MAP ' but MotionCorr ones have 'MAP'.
#Opening in permissive mode and catching the warning, we can get around this.
try:
with warnings.catch_warnings(record=True) as w:
mrc = mrcfile.mmap(self.projectDirectory.value +
fileName,
permissive=True)
except FileNotFoundError as err:
self.errorText.value = "Unable to load Micrograph: {0}".format(
err)
self.errorText.layout = self.errorLayout
else:
mrcArray = np.array(mrc.data)
#extract data, convert values to 0 - 255 int8 format
formatted = (mrcArray * 255 /
np.max(mrcArray)).astype('uint8')
img = Image.fromarray(formatted)
img.thumbnail(self.rawThumbNailSize)
#increase contast to aid visibility
enhancer = ImageEnhance.Contrast(img)
img = enhancer.enhance(6)
imagePng = self.convertToPNG(img)
mrc.close()
img = widgets.Image(value=imagePng,
format='PNG',
width=self.rawThumbNailSize[0],
height=self.rawThumbNailSize[1])
display(img)
# showCtfMicrographs() - builds a list of CTF micrographs with accompanying buttons to display data and raw the micrograph
# Arguments - projectDirectory, relion project folder - full path
# - ctfs - a list of dicts containing CTF values generated by gctf.
# Returns a list of VBox objects containing the micrograph image, with accompanying buttons to display data and the raw micrograph.
#
@debug.capture(clear_output=True)
def showCtfMicrographs(self, projectDirectory, ctfs):
allMicrographs = []
for i in range(len(ctfs)):
fileName = ctfs[i]['ctfImage'].replace(':mrc', '')
try:
mrc = mrcfile.mmap(projectDirectory + fileName)
except (FileNotFoundError, ValueError) as err:
self.errorText.value = "Unable to load Micrograph: {0}".format(
err)
self.errorText.layout = self.errorLayout
else:
#extract data, convert values to 0 - 255 int8 format
formatted = (mrc.data[0] * 255 /
np.max(mrc.data)).astype('uint8')
img = Image.fromarray(formatted)
img.thumbnail(self.thumbNailSize)
imagePng = self.convertToPNG(img)
mrc.close()
#Dynamically build the Ctf mrcs and buttons.
img = widgets.Image(value=imagePng,
format='PNG',
width=self.thumbNailSize[0],
height=self.thumbNailSize[1])
dataButton = widgets.Button(description='Data',
disabled=False,
button_style='',
tooltip='Show data',
icon='check')
mrcButton = widgets.Button(description='Raw mrc',
disabled=False,
button_style='',
tooltip='Show micrograph',
icon='check')
buttonOutput = widgets.Output(layout={'border': '2px solid black'})
dataButton.add_traits(showData=traitlets.Any(buttonOutput))
mrcButton.add_traits(showMrc=traitlets.Any(buttonOutput))
#adding on_click actions to buttons dynamically, lambda is an anonymous function
#lambda arguments: function
dataButton.on_click(lambda dataButton, num=i: self.dynamicOnClick(
ctfs[num], dataButton))
mrcButton.on_click(lambda mrcButton, num=i: self.dynamicOnClick(
ctfs[num], mrcButton))
micrograph = VBox(
[img, HBox([dataButton, mrcButton]), buttonOutput])
allMicrographs.append(micrograph)
return allMicrographs
# buildReviewJobs() - obtains a list of folders (jobs), then uses these to build a list
# of radio buttons
# @debug.capture causes any exceptions to be displayed in the debug field
#
@debug.capture(clear_output=True)
def buildReviewJobs(self):
folders = glob.glob(self.projectDirectory.value +
self.gctfFolderName.value + '*')
folders.sort()
jobs = []
for i in range(len(folders)):
split = folders[i].rsplit('/', 1)
jobs.append(split[1])
self.totalMicrographs.value = ''
self.errorText.layout = self.basicLayout
#Determine total count of Ctf micrographs for review for each job.
for j in range(len(jobs)):
mrcCount = len(
glob.glob(self.projectDirectory.value +
self.gctfFolderName.value + jobs[j] +
'/Micrographs/*.ctf'))
self.totalMicrographs.value += jobs[j] + ': ' + str(
mrcCount) + '\n'
#initialise filter count at start of review
if j == 0:
self.totalFilteredMicrographs.value = mrcCount
if not jobs:
self.errorText.value = "No jobs found: Check project directory"
self.errorText.layout = self.errorLayout
return HBox([])
else:
self.jobSelection.options = jobs
return HBox([self.jobSelection, self.totalMicrographs])
# buildFilters() - contruct the filters tab
# returns a VBox containing the filter fields.
@debug.capture(clear_output=True)
def buildFilters(self):
filters = VBox([
HBox([self.defocusUFrom, self.defocusUTo]),
HBox([self.defocusVFrom, self.defocusVTo]),
HBox([self.defocusAFrom, self.defocusATo]),
HBox([self.voltageFrom, self.voltageTo]),
HBox([self.sphericalAberrationFrom, self.sphericalAberrationTo]),
HBox([self.ampContrastFrom, self.ampContrastTo]),
HBox([self.magnificationFrom, self.magnificationTo]),
HBox([self.detectPixSizeFrom, self.detectPixSizeTo]),
HBox([self.ctfFigMeritFrom, self.ctfFigMeritTo]),
HBox([self.ctfMaxResFrom, self.ctfMaxResTo]),
HBox([self.orderBy, self.totalFilteredMicrographs]),
HBox([self.resetFilterButton, self.applyFilterButton])
])
tab = widgets.Tab(children=[filters])
tab.set_title(0, 'Filters')
return VBox([tab])
# startReview() - when clicked builds a list of jobs for review.
# @debug.capture causes any exceptions to be displayed in the debug field
#
@debug.capture(clear_output=True)
def startReview(self, target):
self.errorText.layout = self.basicLayout
if self.workFlow.projectDirectory.value == '':
self.errorText.value = "Workflow Project Directory required"
self.errorText.layout = self.errorLayout
else:
if self.workFlow.projectDirectory.value.endswith('/') == False:
self.workFlow.projectDirectory.value += '/'
self.projectDirectory.value = self.workFlow.projectDirectory.value
self.gctfFolderName.value = self.workFlow.gctfFolderName
self.jobsBox.clear_output()
with self.jobsBox:
display(HBox([self.projectDirectory, self.gctfFolderName]),
self.buildReviewJobs(), self.buildFilters(),
HBox([self.saveButton]))
self.reviewCtfs('')
# applyFilter() - apply filters to knockout micrographs for review
# Returns a list of filtered micrographs
#
@debug.capture(clear_output=True)
def applyFilter(self, target):
df = pd.DataFrame(self.ctfReviewList)
#apply selected order by, ascending
if self.orderBy.value == 'Defocus U':
df.sort_values(by='defocusU', inplace=True)
if self.orderBy.value == 'Figure of merit':
df.sort_values(by='figOfMerit', inplace=True)
if self.orderBy.value == 'Ctf max resolution':
df.sort_values(by='maxResolution', inplace=True)
filtered = df.loc[
(df['defocusU'] >= self.defocusUFrom.value)
& (df['defocusU'] <= self.defocusUTo.value) &
(df['defocusV'] >= self.defocusVFrom.value) &
(df['defocusV'] <= self.defocusVTo.value) &
(df['defocusAngle'] >= self.defocusAFrom.value) &
(df['defocusAngle'] <= self.defocusATo.value) &
(df['voltage'] >= self.voltageFrom.value) &
(df['voltage'] <= self.voltageTo.value) &
(df['sphericalAberration'] >= self.sphericalAberrationFrom.value) &
(df['sphericalAberration'] <= self.sphericalAberrationTo.value) &
(df['ampContrast'] >= self.ampContrastFrom.value) &
(df['ampContrast'] <= self.ampContrastTo.value) &
(df['magnification'] >= self.magnificationFrom.value) &
(df['magnification'] <= self.magnificationTo.value) &
(df['detectorPixelSize'] >= self.detectPixSizeFrom.value) &
(df['detectorPixelSize'] <= self.detectPixSizeTo.value) &
(df['figOfMerit'] >= self.ctfFigMeritFrom.value) &
(df['figOfMerit'] <= self.ctfFigMeritTo.value) &
(df['maxResolution'] >= self.ctfMaxResFrom.value) &
(df['maxResolution'] <= self.ctfMaxResTo.value)].to_dict('records')
self.filteredCtfReviewList = filtered
self.totalFilteredMicrographs.value = len(self.filteredCtfReviewList)
self.reviewCtfs('')
# reviewCtfs() - when clicked builds a list of jobs for review.
# @debug.capture causes any exceptions to be displayed in the debug field
#
@debug.capture(clear_output=True)
def reviewCtfs(self, target):
allMicrographs = self.showCtfMicrographs(
self.workFlow.projectDirectory.value, self.filteredCtfReviewList)
hBoxes = []
rowItemCnt = 0
rowItemMax = 5
row = []
#Construct a grid layout to display the Ctf Micrographs - max 5 per row.
for i in range(len(allMicrographs)):
if rowItemCnt < rowItemMax:
row.append(allMicrographs[i])
rowItemCnt += 1
if (len(row) == rowItemMax) or (i == len(allMicrographs) - 1):
hBoxes.append(HBox(row))
row = []
rowItemCnt = 0
self.reviewBox.clear_output()
with self.reviewBox:
display(VBox(hBoxes))
# resetFilter() - reset filter to original values
#
@debug.capture(clear_output=True)
def resetFilter(self, target):
self.defocusUFrom.value = 0
self.defocusUTo.value = self.defaultValue
self.defocusVFrom.value = 0
self.defocusVTo.value = self.defaultValue
self.defocusAFrom.value = 0
self.defocusATo.value = self.defaultValue
self.voltageFrom.value = 0
self.voltageTo.value = self.defaultValue
self.sphericalAberrationFrom.value = 0
self.sphericalAberrationTo.value = self.defaultValue
self.ampContrastFrom.value = 0
self.ampContrastTo.value = self.defaultValue
self.magnificationFrom.value = 0
self.magnificationTo.value = self.defaultValue
self.detectPixSizeFrom.value = 0
self.detectPixSizeTo.value = self.defaultValue
self.ctfFigMeritFrom.value = 0
self.ctfFigMeritTo.value = self.defaultValue
self.ctfMaxResFrom.value = 0
self.ctfMaxResTo.value = self.defaultValue
self.orderBy.value = 'Defocus U'
self.applyFilter('')
# saveSelected() - create a star file containing the selected CTFs
#
@debug.capture(clear_output=True)
def saveSelected(self, target):
self.contrastTransfer.createStar(self.projectDirectory.value,
self.gctfFolderName.value,
self.jobSelection.value,
self.filteredCtfReviewList,
self.starFileName)
# on_click() - handle click actions for the jobSelection radio button
@debug.capture(clear_output=True)
def on_click(self, change):
#reset images and filter.
self.reviewBox.clear_output()
#build the list of Micrograph images for selected job
self.ctfReviewList = self.contrastTransfer.buildStarFileData(
self.workFlow.projectDirectory.value, self.workFlow.gctfFolderName,
self.jobSelection.value + '/')
self.resetFilter('')
#update total of filtered micrographs
self.totalFilteredMicrographs.value = len(self.ctfReviewList)
# buildInputWidgets() - write all the workflow fields to the screen.
#
@debug.capture(clear_output=True)
def buildWidgets(self):
#linking button on_click to function
self.startButton.on_click(self.startReview)
self.resetFilterButton.on_click(self.resetFilter)
self.applyFilterButton.on_click(self.applyFilter)
self.saveButton.on_click(self.saveSelected)
self.jobSelection.observe(self.on_click, 'value')
advBoxLayout = Layout(display='flex',
flex_flow='row',
align_items='stretch',
border='none',
width='100%')
if self.showDebug:
return VBox([
self.debug, self.debugText,
HBox([self.startButton, self.errorText]), self.jobsBox,
self.reviewBox
])
else:
return VBox([
self.debug,
HBox([self.startButton, self.errorText]), self.jobsBox,
self.reviewBox
])
# # Jupter Widgets: Erstelle interaktive Notebooks
#
# In dieser Lektion lernst du, wie du:
#
# - Texteingabefelder, Buttons und Slider erstellst
# - Und mit denen interagieren kannst.
# In[2]:
import ipywidgets as widgets
from IPython.display import display
# In[9]:
age = widgets.IntText(description="Alter:", value=25)
display(age)
# In[12]:
print(age.value)
# In[18]:
button = widgets.Button(description="OK")
display(button)
def on_button_click(p):
print("on_button_click()")
def bag_player(bagfile=''):
"""
Create a form widget for playing ROS bags.
This function takes the bag file path, extracts the bag summary
and play the bag with the given arguments.
@param bagfile The ROS bag file path
@return jupyter widget for display
"""
raise FutureWarning("Bag files have not been implemented fully in ros2!")
widget_list = []
bag_player.sp = None
###### Fields #########################################################
bgpath_txt = widgets.Text()
bgpath_box = widgets.HBox([widgets.Label("Bag file path:"), bgpath_txt])
bgpath_txt.value = bagfile
play_btn = widgets.Button(description="Play", icon='play')
pause_btn = widgets.Button(description="Pause",
icon='pause',
disabled=True)
step_btn = widgets.Button(description="Step",
icon='step-forward',
disabled=True)
ibox = widgets.Checkbox(description="Immediate")
lbox = widgets.Checkbox(description="Loop")
clockbox = widgets.Checkbox(description="Clock")
dzbox = widgets.Checkbox(description="Duration")
kabox = widgets.Checkbox(description="Keep alive")
start_float = widgets.FloatText(value=0)
start_box = widgets.HBox([widgets.Label("Start time:"), start_float])
que_int = widgets.IntText(value=100)
que_box = widgets.HBox([widgets.Label("Queue size:"), que_int])
factor_float = widgets.FloatText(value=1)
factor_box = widgets.HBox(
[widgets.Label("Multiply the publish rate by:"), factor_float])
delay_float = widgets.FloatText(value=0)
delay_box = widgets.HBox(
[widgets.Label("Delay after every advertise call:"), delay_float])
duration_float = widgets.FloatText(value=0)
duration_box = widgets.HBox(
[dzbox, widgets.Label("Duration in secs:"), duration_float])
out_box = widgets.Output(layout={'border': '1px solid black'})
def ply_clk(arg):
""" Play Button """
if play_btn.description == "Play":
info_dict = yaml.load(
subprocess.Popen(
['rosbag', 'info', '--yaml', bgpath_txt.value],
stdout=subprocess.PIPE).communicate()[0])
if info_dict is None:
raise FileNotFoundError("Bag file not found!")
else:
cmd = ['rosbag', 'play', bgpath_txt.value]
if ibox.value:
cmd.append('-i')
if lbox.value:
cmd.append('-l')
if kabox.value:
cmd.append('-k')
if clockbox.value:
cmd.append('--clock')
if dzbox.value:
cmd.append("--duration={}".format(
max(0, duration_float.value)))
cmd.append("--rate={}".format(max(0, factor_float.value)))
cmd.append("--start={}".format(max(0, start_float.value)))
cmd.append("--queue={}".format(max(0, que_int.value)))
cmd.append("--delay={}".format(max(0, delay_float.value)))
play_btn.description = "Stop"
play_btn.icon = 'stop'
pause_btn.disabled = False
bag_player.sp = subprocess.Popen(cmd, stdin=subprocess.PIPE)
with out_box:
print("Bag summary:")
for key, val in info_dict.items():
print(key, ":", val)
else:
try:
os.killpg(os.getpgid(bag_player.sp.pid),
subprocess.signal.SIGINT)
except KeyboardInterrupt:
pass
play_btn.description = "Play"
play_btn.icon = 'play'
pause_btn.disabled = True
pause_btn.description = 'Pause'
pause_btn.icon = 'pause'
step_btn.disabled = True
play_btn.on_click(ply_clk)
def pause_clk(arg):
""" Pause Button """
bag_player.sp.stdin.write(b' \n')
bag_player.sp.stdin.flush()
if pause_btn.description == 'Pause':
pause_btn.description = 'Continue'
pause_btn.icon = 'play'
step_btn.disabled = False
else:
pause_btn.description = 'Pause'
pause_btn.icon = 'pause'
step_btn.disabled = True
pause_btn.on_click(pause_clk)
def step_clk(arg):
""" Step Button """
bag_player.sp.stdin.write(b's\n')
bag_player.sp.stdin.flush()
step_btn.on_click(step_clk)
options_hbox = widgets.HBox([ibox, lbox, clockbox, kabox])
buttons_hbox = widgets.HBox([play_btn, pause_btn, step_btn])
btm_box = widgets.VBox([
bgpath_box, options_hbox, duration_box, start_box, que_box, factor_box,
delay_box, buttons_hbox, out_box
])
widget_list.append(btm_box)
vbox = widgets.VBox(children=widget_list)
return vbox
min=3.0,
max=20.0,
step=0.1,
description='ENL:',
disabled=False)
w_orbitpass = widgets.RadioButtons(options=['ASCENDING', 'DESCENDING'],
value='ASCENDING',
description='Orbit pass:'******'Bitemporal', 'First', 'Last', 'Frequency'],
value='First',
description='Map:',
disabled=False)
w_bmap = widgets.IntText(layout=widgets.Layout(width='50px'),
value='1',
description='',
disabled=False)
w_platform = widgets.RadioButtons(layout=widgets.Layout(width='20%'),
options=['Both', 'A', 'B'],
value='Both',
description='Platform:',
disabled=False)
w_relativeorbitnumber = widgets.IntText(value='0',
description='Rel orbit:',
disabled=False)
w_exportassetsname = widgets.Text(value='users/<username>/<path>',
placeholder=' ',
disabled=False)
w_exportdrivename = widgets.Text(value='EarthEngineImages/<path>',
placeholder=' ',
disabled=False)
def detectKeypointsInterface(video_dropdown, json_dropdown, data_dropdown,
frame_n):
model_nn = wg.Dropdown(options=["None", "Openpose Model"],
value="None",
description='Inference:',
disabled=False)
summary = wg.Textarea(value='',
placeholder='description',
description='Summary:',
disabled=False)
output_name = wg.Text(value='',
placeholder='File output name',
description='Output:',
disabled=False)
process_vid = wg.Button(description='Process Video')
persons = wg.RadioButtons(options=['All', 'Custom'],
value='Custom',
rows=2,
description='Choose:',
disabled=False)
custom = wg.BoundedIntText(value=0,
min=0,
max=10,
step=1,
description='Person:',
disabled=False,
layout=wg.Layout(display='flex',
flex_flow='line',
align_items='flex-start',
justify_content='flex-start',
width='90%'))
frame_slider = wg.IntSlider()
detect_vid = wg.Button(description='Save Keypoints')
paf_score_th = wg.FloatText(value=0.1,
step=0.1,
description='PAF Thres:',
layout=wg.Layout(display='flex',
flex_flow='line',
align_items='flex-start',
justify_content='flex-start',
width='90%'))
conf_th = wg.FloatText(value=0.7,
step=0.1,
description='Conf. Thres:',
layout=wg.Layout(display='flex',
flex_flow='line',
align_items='flex-start',
justify_content='flex-start',
width='90%'))
n_interp_samples = wg.IntText(value=10,
description='Samples Interpolated:',
layout=wg.Layout(
display='flex',
flex_flow='line',
align_items='flex-start',
justify_content='flex-start',
width='90%'))
joint_n = wg.Dropdown(options=keypoints_mapping,
value='Nose',
description='Joint:',
disabled=False)
threshold = wg.FloatText(value=0.1,
step=0.1,
description='Threshold:',
layout=wg.Layout(display='flex',
flex_flow='line',
align_items='flex-start',
justify_content='flex-start',
width='80%'))
alpha = wg.FloatText(value=0.6,
step=0.1,
description='Transparency:',
layout=wg.Layout(display='flex',
flex_flow='line',
align_items='flex-start',
justify_content='flex-start',
width='80%'))
show_point = wg.Checkbox(value=False,
description='Show Point',
disabled=False,
layout=wg.Layout(display='flex',
flex_flow='line',
align_items='flex-start',
justify_content='flex-start',
width='80%'))
binary = wg.Checkbox(value=False,
description='Binary',
disabled=False,
layout=wg.Layout(display='flex',
flex_flow='line',
align_items='flex-start',
justify_content='flex-start',
width='80%'))
wg.jslink((frame_n, 'value'), (frame_slider, 'value'))
show_heatmap = wg.ToggleButtons(
options=['Keypoints', 'Heatmap'],
value='Keypoints',
disabled=False,
button_style='', # 'success', 'info', 'warning', 'danger' or ''
tooltips=['Show keypoints', 'Show heatmap'],
# icons=['check'] * 3
)
summary = wg.Textarea(value='',
placeholder='description',
description='Summary:',
disabled=False)
output_name = wg.Text(value='',
placeholder='File output name',
description='Output:',
disabled=False)
def DetectView(show_heatmap, video_name, file_name, persons, custom,
joint_n, alpha, binary, threshold, n_interp_samples,
paf_score_th, conf_th, frame_n, show_point):
joint_pairs = [-1]
if (persons == 'Custom'):
persons = 'Unsorted'
if (show_heatmap == 'Keypoints'):
keypointsFromJSON(video_name, file_name, persons, custom,
joint_pairs, frame_n, threshold,
n_interp_samples, paf_score_th, conf_th)
else:
heatmapFromJSON(video_name, file_name,
keypoints_mapping.index(joint_n), threshold, alpha,
binary, n_interp_samples, paf_score_th, conf_th,
frame_n, show_point)
video_display = wg.interactive_output(
DetectView, {
"show_heatmap": show_heatmap,
"video_name": video_dropdown,
"file_name": json_dropdown,
"persons": persons,
"custom": custom,
"joint_n": joint_n,
"alpha": alpha,
"binary": binary,
"threshold": threshold,
"n_interp_samples": n_interp_samples,
"paf_score_th": paf_score_th,
"conf_th": conf_th,
"frame_n": frame_n,
"show_point": show_point
})
def onProcessClicked(b):
if (model_nn.value == 'None'):
print("Select inference method")
elif (model_nn.value == 'Openpose Model'):
videoInference(video_dropdown.value, summary.value,
output_name.value, threshold.value,
n_interp_samples.value, paf_score_th.value,
conf_th.value)
else:
print("No such inference")
video_name = (video_dropdown.value).split(sep='.')[0]
file_dir = data_dir + video_name + '/'
files_list = os.listdir(file_dir)
json_list = ["None"]
for names in files_list:
if names.endswith(".json"):
json_list.append(names)
json_dropdown.options = json_list
process_vid.on_click(onProcessClicked)
hbox_input = wg.HBox([video_dropdown, json_dropdown])
hbox_play = wg.HBox([frame_n, frame_slider])
ht_vbox_1 = wg.VBox([joint_n, show_point, alpha, binary, threshold],
layout=wg.Layout(display='flex',
flex_flow='line',
align_items='flex-start',
justify_content='flex-start'))
vbox_params = wg.VBox([paf_score_th, conf_th, n_interp_samples])
vbox_per = wg.VBox([persons, custom],
layout=wg.Layout(display='flex',
flex_flow='column',
align_items='flex-start',
justify_content='flex-start'))
tabs = ['Person', 'Heatmap', 'PAF']
children = []
children.append(vbox_per)
children.append(ht_vbox_1)
children.append(vbox_params)
tab = wg.Tab()
tab.children = children
for i in range(len(children)):
tab.set_title(i, tabs[i])
vbox1 = wg.VBox([output_name, model_nn, summary, process_vid])
#hbox_out = wg.HBox([vbox1, summary, process_vid])
vbox_video = wg.VBox([video_display, hbox_play])
vbox_config = wg.VBox([show_heatmap, tab, vbox1])
hbox_res = wg.HBox([vbox_video, vbox_config])
vbox_res = wg.VBox([hbox_input, hbox_res])
return vbox_res
def interact_data(self):
update = widgets.ToggleButton(
value=False,
description="update",
disabled=False,
button_style="", # 'success', 'info', 'warning', 'danger' or ''
tooltip="Description",
icon="check",
)
percentage = widgets.BoundedFloatText(
value=0,
min=0,
max=100.0,
step=1,
disabled=False,
description="percent ($\%$):",
)
floor = widgets.FloatText(
value=1e-2,
min=0.0,
max=10.0,
step=1,
description="floor (s):",
disabled=False,
)
seed = widgets.IntText(value=1,
min=1,
max=10,
step=1,
description="random seed",
disabled=False)
add_noise = widgets.Checkbox(value=True,
description="add noise?",
disabled=False)
plot_type = widgets.ToggleButtons(
options=["tx_rx_plane", "histogram"],
disabled=False,
button_style="", # 'success', 'info', 'warning', 'danger' or ''
tooltips=["tx-rx plane", "histogram"],
)
out = widgets.interactive_output(
self.plot_survey_data,
{
"percentage": percentage,
"floor": floor,
"seed": seed,
"add_noise": add_noise,
"plot_type": plot_type,
"update": update,
},
)
form_item_layout = widgets.Layout(display="flex",
flex_flow="row",
justify_content="space-between")
form_items = [
widgets.Box([percentage], layout=form_item_layout),
widgets.Box([floor], layout=form_item_layout),
widgets.Box([seed], layout=form_item_layout),
widgets.Box([add_noise], layout=form_item_layout),
widgets.Box([plot_type], layout=form_item_layout),
widgets.Box([update], layout=form_item_layout),
]
form = widgets.Box(
form_items,
layout=widgets.Layout(
display="flex",
flex_flow="column",
border="solid 2px",
align_items="stretch",
width="40%",
height="30%",
margin="5%",
),
)
return widgets.HBox([out, form])
def __init__(self, **kwargs):
# Username.
self.username = ipw.Text(description="SSH username:"******"SSH port:",
value=22,
layout=LAYOUT,
style=STYLE,
)
# Hostname.
self.hostname = ipw.Text(
description="Computer hostname:",
layout=LAYOUT,
style=STYLE,
)
# ProxyJump.
self.proxy_jump = ipw.Text(
description="ProxyJump:",
layout=LAYOUT,
style=STYLE,
)
# ProxyJump.
self.proxy_command = ipw.Text(
description="ProxyCommand:",
layout=LAYOUT,
style=STYLE,
)
self._inp_private_key = ipw.FileUpload(
accept="",
layout=LAYOUT,
style=STYLE,
description="Private key",
multiple=False,
)
self._verification_mode = ipw.Dropdown(
options=[
("Password", "password"),
("Use custom private key", "private_key"),
("Download public key", "public_key"),
],
layout=LAYOUT,
style=STYLE,
value="password",
description="Verification mode:",
disabled=False,
)
self._verification_mode.observe(self._on_verification_mode_change,
names="value")
self._verification_mode_output = ipw.Output()
self._continue_button = ipw.ToggleButton(description="Continue",
layout={"width": "100px"},
value=False)
# Setup ssh button and output.
btn_setup_ssh = ipw.Button(description="Setup ssh")
btn_setup_ssh.on_click(self.on_setup_ssh)
self.setup_ssh_out = ipw.Output()
children = [
self.hostname,
self.port,
self.username,
self.proxy_jump,
self.proxy_command,
self._verification_mode,
self._verification_mode_output,
btn_setup_ssh,
self.setup_ssh_out,
]
super().__init__(children, **kwargs)
# * Incorrect: myFile
#
# The second input text widget (Sheet), specifies the index of the sheet to be read only if the file is an .xlsx file. If the file is not an .xlsx then this input is ignored.
# In[7]:
#read file
fileInput = widgets.Text(placeholder='Enter file name with extension',
description='File:',
disabled=False)
fileInput.set_title = 'fileInput'
#specify sheet index
sheet_number = widgets.IntText(value=0, description='Sheet', disabled=False)
sheet_number.set_title = 'sheet_number'
#display widgets
display(fileInput, sheet_number)
# # Import file into a python format
# Convert the input file into a Pandas dataframe.
#
# Two dataframes are created:
#
# * df: this is the working dataframe which is used for visualisation and analysis. This is also the file that is filtered (subset). After filtering, it can be reverted to the original input dataset.
#
# * original_data: this is the imported dataframe and it is not changed throughout the program (as opposed to df). This is the file that df is reverted to whenever requested by the user.
def __init__(self):
empty = ipywidgets.HTML()
self._radius_sel = ipywidgets.FloatText(
value=1., step=0.1, description="radius")
self._thresh_sel = ipywidgets.FloatText(
value=100., step=10, description="threshold")
self._model_sel = ipywidgets.Dropdown(
options=["2d_fixed", "2d", "3d"], description="model", value="2d")
self._min_dist_check = ipywidgets.Checkbox(
indent=False, layout=ipywidgets.Layout(width="min-content"))
self._min_dist_sel = ipywidgets.FloatText(
description="Min. distance", value=1., step=0.1)
size_label = ipywidgets.Label("Size range")
self._size_check = ipywidgets.Checkbox(
indent=False, layout=ipywidgets.Layout(width="min-content"))
self._min_size_sel = ipywidgets.FloatText(
value=0.5, step=0.1, description="min.",
layout=ipywidgets.Layout(display="none"))
self._max_size_sel = ipywidgets.FloatText(
value=2., step=0.1, description="max.",
layout=ipywidgets.Layout(display="none"))
size_box = ipywidgets.VBox([size_label, self._min_size_sel,
self._max_size_sel])
self._filter_sel = ipywidgets.Dropdown(
options=["Identity", "Crocker-Grier", "Gaussian"],
description="find filter")
self._filter_cg_size_sel = ipywidgets.IntText(
value=3, description="feat. size",
layout=ipywidgets.Layout(display="none"))
self._filter_gauss_sigma_sel = ipywidgets.FloatText(
value=1., step=0.1, description="sigma",
layout=ipywidgets.Layout(display="none"))
self._filter_box = ipywidgets.VBox([
self._filter_sel, self._filter_cg_size_sel,
self._filter_gauss_sigma_sel])
children = [
empty, self._radius_sel,
empty, self._thresh_sel,
empty, self._model_sel,
empty, self._filter_box,
self._min_dist_check, self._min_dist_sel,
self._size_check, size_box]
super().__init__(
children, layout=ipywidgets.Layout(
grid_template_columns="min-content min-content"))
self._filter_sel.observe(self._set_find_filter, names="value")
self._size_check.observe(self._enable_size_range, names="value")
for w in (self._radius_sel, self._thresh_sel,
self._model_sel, self._min_dist_check,
self._min_dist_sel, self._size_check, self._min_size_sel,
self._max_size_sel, self._filter_cg_size_sel,
self._filter_gauss_sigma_sel):
w.observe(self._options_from_ui, names="value")
self._options_from_ui()
self.observe(self._options_to_ui, "options")
description='ENL:',
disabled=False)
w_orbitpass = widgets.RadioButtons(options=['ASCENDING', 'DESCENDING'],
value='ASCENDING',
description='Orbit pass:'******'First', 'Last', 'Frequency'],
value='First',
description='Change map:',
disabled=False)
w_platform = widgets.RadioButtons(options=['Both', 'A', 'B'],
value='Both',
description='Platform:',
disabled=False)
w_relativeorbitnumber = widgets.IntText(value=0,
description='Rel orbit:',
disabled=False)
w_exportname = widgets.Text(value='users/<username>/<path>',
placeholder=' ',
disabled=False)
w_exportscale = widgets.Text(value='10',
placeholder=' ',
description='at scale ',
disabled=False)
w_startdate = widgets.Text(value='2018-04-01',
placeholder=' ',
description='Start date:',
disabled=False)
w_enddate = widgets.Text(value='2018-10-01',
placeholder=' ',
description='End date:',
def widget_emso_qc(wf,
depth_range: List[float] = [-10, 10000],
range_test: List[float] = [-1000, 1000],
spike_window: int = 100,
spike_threshold: float = 3.5,
spike_influence: float = 0.5,
user: str = '',
password: str = '',
token: str = ''):
"""
It makes a Jupyter Notebook Widget used to download EMSO data and make data
quality control tests.
Parameters
----------
wf: WaterFrame
depth_range: List[float]
Range of depth
user: str
User for the EMSO ERIC API
password: str
Password for the EMSO ERIC API
token: str
Token for the EMSO ERIC API
parameter: str
Parameter
range_test: List[float]
Limits for the range test [min value, max value]
spike_window: int
Window for the spike test
spike_threshold: float
Float for the spike test
spike_influence: float
Influence of the bad data on the spike test
Returns
-------
main_box: ipwidgets.VBox
Jupyter notebook widget
"""
pyo.init_notebook_mode()
# Layout config
label_width = '7rem'
sublabel_width = '5rem'
checkbox_width = '8rem'
def show_result(wf, parameter_in, chart_title=''):
# Change name of flags
wf2 = wf.copy()
qc_labels = {0: 'No QC', 1: 'Good data', 4: 'Bad data'}
wf2.data[f'{parameter_in}_QC'].replace(qc_labels, inplace=True)
fig = wf2.iplot_line(
parameter_in,
# color=f'{parameter_in}_QC',
marginal_y=None,
line_shape='linear',
rangeslider_visible=False,
line_dash_sequence=['dot', 'dot'],
title=chart_title,
)
# line_group='DEPTH')
fig.show()
# Sing in
authentification_title = widgets.HTML('<b>Authentification</b>')
user_label = widgets.Label('User:'******'Password:'******'or', layout=widgets.Layout(width=label_width))
token_label = widgets.Label('Token:',
layout=widgets.Layout(width=label_width))
input_token = widgets.Text(value=token)
# map
emso = mooda.EMSO(user=input_user.value,
password=input_password.value,
token=input_token.value)
map_dict = emso.get_fig_map()
pio.show(map_dict)
# Platform code
data_title = widgets.HTML('<b>Downloading data</b>')
platform_label = widgets.Label('Platform code:',
layout=widgets.Layout(width=label_width))
platform_codes = emso.get_info_platform_code()
input_platform = widgets.Dropdown(options=platform_codes,
value=platform_codes[0],
disabled=False)
# Get metadata
metadata_list = emso.get_metadata(platform_codes=[platform_codes[0]])
try:
metadata = metadata_list[0]
parameter_list = metadata.get('parameters')
if parameter_list is None:
parameter_list = emso.get_info_parameter()
depth_min = metadata.get('geospatial_vertical_min', depth_range[0])
depth_max = metadata.get('geospatial_vertical_max', depth_range[1])
start_date = metadata.get('time_coverage_start')
if start_date:
start_date = start_date.split('T')[0]
end_date = metadata.get('time_coverage_end')
if end_date:
end_date = end_date.split('T')[0]
except:
parameter_list = emso.get_info_parameter()
depth_min = depth_range[0]
depth_max = depth_range[1]
start_date = None
end_date = None
# Parameters
parameters_label = widgets.Label('Parameters:',
layout=widgets.Layout(width=label_width))
input_parameters = widgets.SelectMultiple(options=parameter_list,
rows=10,
disabled=False,
value=[parameter_list[0]])
# Size
size_label = widgets.Label('Size:',
layout=widgets.Layout(width=label_width))
input_size = widgets.BoundedIntText(value=10, min=1, max=100000, step=1)
# Depth
depth_label = widgets.Label('Depth:',
layout=widgets.Layout(width=label_width))
input_depth = widgets.FloatRangeSlider(value=[depth_min, depth_max],
min=depth_min,
max=depth_max,
step=0.1,
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.1f')
# Time
start_date_label = widgets.Label('Start date:',
layout=widgets.Layout(width=label_width))
input_start_date = widgets.DatePicker(
disabled=False,
min_date=date(int(start_date.split('-')[0]),
int(start_date.split('-')[1]),
int(start_date.split('-')[2])),
max_date=date(int(end_date.split('-')[0]), int(end_date.split('-')[1]),
int(end_date.split('-')[2])),
value=date(int(start_date.split('-')[0]),
int(start_date.split('-')[1]),
int(start_date.split('-')[2])))
end_date_label = widgets.Label('End date:',
layout=widgets.Layout(width=label_width))
input_end_date = widgets.DatePicker(
disabled=False,
min_date=date(int(start_date.split('-')[0]),
int(start_date.split('-')[1]),
int(start_date.split('-')[2])),
max_date=date(int(end_date.split('-')[0]), int(end_date.split('-')[1]),
int(end_date.split('-')[2])),
value=date(int(end_date.split('-')[0]), int(end_date.split('-')[1]),
int(end_date.split('-')[2])))
qc_label = widgets.HTML('<b>Data QC Tests - Select the test to run</b>')
# Layout config
checkbox_width = '8rem'
# Reset flags
# reset_label = widgets.Label('Reset flags:')
reset_checkbox = widgets.Checkbox(
description='Reset flags',
style={'description_width': '0'},
layout=widgets.Layout(width=checkbox_width))
# Flat test
flat_checkbox = widgets.Checkbox(
description='Flat test',
style={'description_width': '0'},
layout=widgets.Layout(width=checkbox_width))
window_label = widgets.Label('Window:',
layout=widgets.Layout(width=sublabel_width))
window_flat = widgets.IntText(value=2, disabled=False, color='black')
# Range test
# range_label = widgets.Label('Range test:')
range_checkbox = widgets.Checkbox(
description='Range test',
style={'description_width': '0'},
layout=widgets.Layout(width=checkbox_width))
limit_label = widgets.Label('Limits:',
layout=widgets.Layout(width=sublabel_width))
limits = widgets.FloatRangeSlider(value=[range_test[0], range_test[1]],
min=range_test[0],
max=range_test[1],
step=0.1,
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.1f')
# Spike test
# spike_label = widgets.Label('Spike test:')
spike_checkbox = widgets.Checkbox(
description='Spike test',
style={'description_width': '0'},
layout=widgets.Layout(width=checkbox_width))
window_spike = widgets.IntText(value=spike_window,
disabled=False,
color='black')
threshold_label = widgets.Label(
'Threshold:', layout=widgets.Layout(width=sublabel_width))
threshold = widgets.FloatText(value=spike_threshold,
disabled=False,
color='black')
influence_label = widgets.Label(
'Influence:', layout=widgets.Layout(width=sublabel_width))
influence = widgets.FloatSlider(value=spike_influence,
min=0,
max=1,
step=0.1,
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.1f',
slider_color='white')
# Replace
# replace_label = widgets.Label('Replace QC:')
replace_checkbox = widgets.Checkbox(description='Replace flags',
style={'description_width': '0'})
def update_components(_):
# Update all components
emso = mooda.EMSO(user=input_user.value,
password=input_password.value,
token=input_token.value)
# Get metadata
metadata_list = emso.get_metadata(
platform_codes=[input_platform.value])
try:
metadata = metadata_list[0]
parameter_list = metadata.get('parameters')
if parameter_list is None:
parameter_list = emso.get_info_parameter()
depth_min = metadata.get('geospatial_vertical_min', depth_range[0])
depth_max = metadata.get('geospatial_vertical_max', depth_range[1])
start_date = metadata.get('time_coverage_start')
if start_date:
start_date = start_date.split('T')[0]
end_date = metadata.get('time_coverage_end')
if end_date:
end_date = end_date.split('T')[0]
except:
parameter_list = emso.get_info_parameter()
depth_min = depth_range[0]
depth_max = depth_range[1]
start_date = None
end_date = None
# Parameters
input_parameters.options = parameter_list
# Depth
input_depth.min = depth_min
input_depth.max = depth_max
input_depth.value = [depth_min, depth_max]
# Time
input_start_date.min_date = date(int(start_date.split('-')[0]),
int(start_date.split('-')[1]),
int(start_date.split('-')[2]))
input_start_date.max_date = date(int(end_date.split('-')[0]),
int(end_date.split('-')[1]),
int(end_date.split('-')[2]))
input_start_date.value = date(int(start_date.split('-')[0]),
int(start_date.split('-')[1]),
int(start_date.split('-')[2]))
input_end_date.min_date = date(int(start_date.split('-')[0]),
int(start_date.split('-')[1]),
int(start_date.split('-')[2]))
input_end_date.max_date = date(int(end_date.split('-')[0]),
int(end_date.split('-')[1]),
int(end_date.split('-')[2]))
input_end_date.value = date(int(end_date.split('-')[0]),
int(end_date.split('-')[1]),
int(end_date.split('-')[2]))
input_platform.observe(update_components, names='value')
# Button
button = widgets.Button(description='Get data')
out = widgets.Output()
out2 = widgets.Output()
def on_button_clicked(_):
# "linking function with output"
with out:
# what happens when we press the button
clear_output()
parameters = []
for input_parameter in input_parameters.value:
parameters.append(input_parameter.split('-')[0].strip())
if input_start_date.value is None:
start_time = ''
else:
start_time = str(input_start_date.value) + ' 00:00:00'
if input_end_date.value is None:
end_time = ''
else:
end_time = str(input_end_date.value) + ' 00:00:00'
if input_token.value is None:
token = ''
else:
token = input_token.value
print('Downloading data, please wait')
wf2 = mooda.from_emso(platform_code=input_platform.value,
parameters=parameters,
start_time=start_time,
end_time=end_time,
depth_min=input_depth.value[0],
depth_max=input_depth.value[1],
user=input_user.value,
password=input_password.value,
size=input_size.value,
token=token)
wf.data = wf2.data.copy()
wf.metadata = wf2.metadata.copy()
wf.vocabulary = wf2.vocabulary.copy()
clear_output()
display(wf)
with out2:
# what happens when we press the button
clear_output()
print('Making tests')
for parameter_one in wf.parameters:
if reset_checkbox.value:
wf.qc_replace(parameters=[parameter_one],
to_replace=1,
value=0)
wf.qc_replace(parameters=[parameter_one],
to_replace=4,
value=0)
if flat_checkbox.value:
wf.qc_flat_test(window=window_flat.value,
parameters=[parameter_one])
if range_checkbox.value:
wf.qc_range_test(limits=limits.value,
parameters=[parameter_one])
if spike_checkbox.value:
wf.qc_spike_test(window=window_spike.value,
influence=influence.value,
threshold=threshold.value,
parameters=[parameter_one])
if replace_checkbox.value:
start = 0
if spike_checkbox.value:
start = window_spike.value
wf.qc_replace(parameters=[parameter_one], start=start)
show_result(wf, parameter_one, f'Final result')
# linking button and function together using a button's method
button.on_click(on_button_clicked)
user_box = widgets.HBox([user_label, input_user])
password_box = widgets.HBox([password_label, input_password])
token_box = widgets.HBox([token_label, input_token])
platform_box = widgets.HBox([platform_label, input_platform])
parameters_box = widgets.HBox([parameters_label, input_parameters])
size_box = widgets.HBox([size_label, input_size])
depth_box = widgets.HBox([depth_label, input_depth])
start_date_box = widgets.HBox([start_date_label, input_start_date])
end_date_box = widgets.HBox([end_date_label, input_end_date])
reset_box = widgets.HBox([reset_checkbox])
flat_box = widgets.HBox([flat_checkbox, window_label, window_flat])
range_box = widgets.HBox([range_checkbox, limit_label, limits])
spike_window_column = widgets.HBox([window_label, window_spike])
spike_threshold_column = widgets.HBox([threshold_label, threshold])
spike_influence_column = widgets.HBox([influence_label, influence])
spike_column = widgets.VBox(
[spike_window_column, spike_threshold_column, spike_influence_column])
spike_box = widgets.HBox([spike_checkbox, spike_column])
replace_box = widgets.HBox([replace_checkbox])
main_box = widgets.VBox([
authentification_title, user_box, password_box, or_label, token_box,
data_title, platform_box, parameters_box, size_box, depth_box,
start_date_box, end_date_box, qc_label, reset_box, flat_box, range_box,
spike_box, replace_box, button, out, out2
])
return main_box
def spikes_removal_ipy(obj, **kwargs):
wdict = {}
threshold = ipywidgets.FloatText()
add_noise = ipywidgets.Checkbox()
default_spike_width = ipywidgets.IntText()
interpolator_kind = enum2dropdown(obj.traits()["interpolator_kind"])
spline_order = ipywidgets.IntSlider(min=1, max=10)
progress_bar = ipywidgets.IntProgress(max=len(obj.coordinates) - 1)
help = ipywidgets.HTML(
value=SPIKES_REMOVAL_INSTRUCTIONS.replace('\n', '<br/>'))
help = ipywidgets.Accordion(children=[help])
help.set_title(0, "Help")
show_diff = ipywidgets.Button(
description="Show derivative histogram",
tooltip="This figure is useful to estimate the threshold.",
layout=ipywidgets.Layout(width="auto"))
close = ipywidgets.Button(
description="Close",
tooltip="Close widget and remove span selector from the signal figure."
)
next = ipywidgets.Button(description="Find next",
tooltip="Find next spike")
previous = ipywidgets.Button(description="Find previous",
tooltip="Find previous spike")
remove = ipywidgets.Button(description="Remove spike",
tooltip="Remove spike and find next one.")
wdict["threshold"] = threshold
wdict["add_noise"] = add_noise
wdict["default_spike_width"] = default_spike_width
wdict["interpolator_kind"] = interpolator_kind
wdict["spline_order"] = spline_order
wdict["progress_bar"] = progress_bar
wdict["show_diff_button"] = show_diff
wdict["close_button"] = close
wdict["next_button"] = next
wdict["previous_button"] = previous
wdict["remove_button"] = remove
def on_show_diff_clicked(b):
obj._show_derivative_histogram_fired()
show_diff.on_click(on_show_diff_clicked)
def on_next_clicked(b):
obj.find()
next.on_click(on_next_clicked)
def on_previous_clicked(b):
obj.find(back=True)
previous.on_click(on_previous_clicked)
def on_remove_clicked(b):
obj.apply()
remove.on_click(on_remove_clicked)
labeled_spline_order = labelme("Spline order", spline_order)
def enable_interpolator_kind(change):
if change.new == "Spline":
for child in labeled_spline_order.children:
child.layout.display = ""
else:
for child in labeled_spline_order.children:
child.layout.display = "none"
interpolator_kind.observe(enable_interpolator_kind, "value")
link((obj, "interpolator_kind"), (interpolator_kind, "value"))
link((obj, "threshold"), (threshold, "value"))
link((obj, "add_noise"), (add_noise, "value"))
link((obj, "default_spike_width"), (default_spike_width, "value"))
link((obj, "spline_order"), (spline_order, "value"))
link((obj, "index"), (progress_bar, "value"))
# Trigger the function that controls the visibility as
# setting the default value doesn't trigger it.
class Dummy:
new = interpolator_kind.value
enable_interpolator_kind(change=Dummy())
advanced = ipywidgets.Accordion((ipywidgets.VBox([
labelme("Add noise", add_noise),
labelme("Interpolator kind", interpolator_kind),
labelme("Default spike width", default_spike_width),
labelme("Spline order", spline_order),
]), ))
advanced.set_title(0, "Advanced settings")
box = ipywidgets.VBox([
ipywidgets.VBox([
show_diff,
labelme("Threshold", threshold),
labelme("Progress", progress_bar),
]), advanced, help,
ipywidgets.HBox([previous, next, remove, close])
])
def on_close_clicked(b):
obj.span_selector_switch(False)
box.close()
close.on_click(on_close_clicked)
return {
"widget": box,
"wdict": wdict,
}
import ipywidgets as w from IPython.display import display a = w.IntSlider() b = w.IntText() w.jslink((a, 'value'), (b, 'value')) display(a, b)
def __init__(self, **kwargs):
# List of widgets to be displayed.
self.label = ipw.Text(
value="",
placeholder="Will only be used within AiiDA",
description="Computer name:",
layout=LAYOUT,
style=STYLE,
)
# Hostname.
self.hostname = ipw.Text(description="Hostname:",
layout=LAYOUT,
style=STYLE)
# Computer description.
self.description = ipw.Text(
value="",
placeholder="No description (yet)",
description="Computer description:",
layout=LAYOUT,
style=STYLE,
)
# Directory where to run the simulations.
self.work_dir = ipw.Text(
value="/scratch/{username}/aiida_run",
description="Workdir:",
layout=LAYOUT,
style=STYLE,
)
# Mpirun command.
self.mpirun_command = ipw.Text(
value="mpirun -n {tot_num_mpiprocs}",
description="Mpirun command:",
layout=LAYOUT,
style=STYLE,
)
# Number of CPUs per node.
self.mpiprocs_per_machine = ipw.IntText(
value=1,
step=1,
description="#CPU(s) per node:",
layout=LAYOUT,
style=STYLE,
)
# Transport type.
self.transport = ipw.Dropdown(
value="local",
options=transports.Transport.get_valid_transports(),
description="Transport type:",
style=STYLE,
)
# Safe interval.
self.safe_interval = ipw.FloatText(
value=30.0,
description="Min. connection interval (sec):",
layout=LAYOUT,
style=STYLE,
)
# Scheduler.
self.scheduler = ipw.Dropdown(
value="slurm",
options=schedulers.Scheduler.get_valid_schedulers(),
description="Scheduler:",
style=STYLE,
)
self.shebang = ipw.Text(
value="#!/usr/bin/env bash",
description="Shebang:",
layout=LAYOUT,
style=STYLE,
)
# Use login shell.
self.use_login_shell = ipw.Checkbox(value=True,
description="Use login shell")
# Prepend text.
self.prepend_text = ipw.Textarea(
placeholder="Text to prepend to each command execution",
description="Prepend text:",
layout=LAYOUT,
)
# Append text.
self.append_text = ipw.Textarea(
placeholder="Text to append to each command execution",
description="Append text:",
layout=LAYOUT,
)
# Buttons and outputs.
self.setup_button = ipw.Button(description="Setup computer")
self.setup_button.on_click(self.on_setup_computer)
test_button = ipw.Button(description="Test computer")
test_button.on_click(self.test)
self.setup_compupter_out = ipw.Output(layout=LAYOUT)
self._test_out = ipw.Output(layout=LAYOUT)
# Organize the widgets
children = [
self.label,
self.hostname,
self.description,
self.work_dir,
self.mpirun_command,
self.mpiprocs_per_machine,
self.transport,
self.safe_interval,
self.scheduler,
self.shebang,
self.use_login_shell,
self.prepend_text,
self.append_text,
self.setup_button,
self.setup_compupter_out,
test_button,
self._test_out,
]
super().__init__(children, **kwargs)
def makeWidgetsForInput():
style = {'description_width': '350px'}
layout = Layout(width='55%')
inputDeckTemplateNameW = widgets.Text(value='qpinput_template.json',
description='Template Input File:',
style=style,
layout=layout)
indxW = widgets.IntText(value=8,
description='indx (indy):',
style=style,
layout=layout)
indzW = widgets.IntText(value=8,
description='indz:',
style=style,
layout=layout)
n0W = widgets.FloatText(value=4,
description='$n_0\;(10^{16}/cm^3)$:',
style=style,
layout=layout)
nbeamsW = widgets.IntSlider(value=2,
min=1,
max=2,
step=1,
description='number of beams:',
style=style,
layout=layout)
boxXlengthW = widgets.FloatText(value=10,
description='box size $x/y$:',
style=style,
layout=layout)
boxZlengthW = widgets.FloatText(value=8,
description='box size $z$:',
style=style,
layout=layout)
# Driving beam
z_driverW = widgets.FloatText(value=2,
description='driver $z$ position:',
style=style,
layout=layout)
sigma_r_driverW = widgets.FloatText(value=0.1,
description='$\sigma_r$ (driver):',
style=style,
layout=layout)
sigma_z_driverW = widgets.FloatText(value=0.5,
description='$\sigma_z$ (driver):',
style=style,
layout=layout)
gammaE_driverW = widgets.FloatText(value=20000,
description='$\gamma$ (driver):',
style=style,
layout=layout)
peak_density_driverW = widgets.FloatText(
value=25,
description='$n_{peak}$ (driver):',
style=style,
layout=layout)
# Witness beam
z_startW = widgets.FloatText(value=5,
description='$z$ start (witness):',
style=style,
layout=layout)
z_endW = widgets.FloatText(value=6,
description='$z$ end (witness):',
style=style,
layout=layout)
nb_startW = widgets.FloatText(value=1,
description='$n_b$ start (witness):',
style=style,
layout=layout)
nb_endW = widgets.FloatText(value=0,
description='$n_b$ end (witness):',
style=style,
layout=layout)
sigma_r_witnessW = widgets.FloatText(value=0.1,
description='$\sigma_r$ (witness):',
style=style,
layout=layout)
gammaE_witnessW = widgets.FloatText(value=20000,
description='$\gamma$ (witness):',
style=style,
layout=layout)
peak_density_witnessW = widgets.FloatText(
value=25,
description='$n_{peak}$ (witness):',
style=style,
layout=layout)
interact_calc(makeInput,
inputDeckTemplateName=inputDeckTemplateNameW,
indx=indxW,
indz=indzW,
n0=n0W,
nbeams=nbeamsW,
boxXlength=boxXlengthW,
boxZlength=boxZlengthW,
z_driver=z_driverW,
sigma_r_driver=sigma_r_driverW,
sigma_z_driver=sigma_z_driverW,
gammaE_driver=gammaE_driverW,
peak_density_driver=peak_density_driverW,
sigma_r_witness=sigma_r_witnessW,
z_start=z_startW,
z_end=z_endW,
nb_start=nb_startW,
nb_end=nb_endW,
gammaE_witness=gammaE_witnessW,
peak_density_witness=peak_density_witnessW)
# Function widgets.jslink returns a Link widget. The link can be broken by calling the unlink method. l.unlink() dl.unlink() # The difference between linking in the kernel and linking in the client ''' Linking in the kernel means linking via python. If two sliders are linked in the kernel, when one slider is changed the browser sends a message to the kernel (python in this case) updating the changed slider, the link widget in the kernel then propagates the change to the other slider object in the kernel, and then the other slider’s kernel object sends a message to the browser to update the other slider’s views in the browser. If the kernel is not running (as in a static web page), then the controls will not be linked. Linking using jslink (i.e., on the browser side) means contructing the link in Javascript. When one slider is changed, Javascript running in the browser changes the value of the other slider in the browser, without needing to communicate with the kernel at all. If the sliders are attached to kernel objects, each slider will update their kernel-side objects independently. To see the difference between the two, go to the ipywidgets documentation and try out the sliders near the bottom. The ones linked in the kernel with link and dlink are no longer linked, but the ones linked in the browser with jslink and jsdlink are still linked. ''' # Continuous updates ''' Some widgets offer a choice with their continuous_update attribute between continually updating values or only updating values when a user submits the value (for example, by pressing Enter or navigating away from the control). In the next example, we see the “Delayed” controls only transmit their value after the user finishes dragging the slider or submitting the textbox. The “Continuous” controls continually transmit their values as they are changed. Try typing a two-digit number into each of the text boxes, or dragging each of the sliders, to see the difference. ''' import traitlets a = widgets.IntSlider(description="Delayed", continuous_update=False) b = widgets.IntText(description="Delayed", continuous_update=False) c = widgets.IntSlider(description="Continuous", continuous_update=True) d = widgets.IntText(description="Continuous", continuous_update=True) traitlets.link((a, 'value'), (b, 'value')) traitlets.link((a, 'value'), (c, 'value')) traitlets.link((a, 'value'), (d, 'value')) widgets.VBox([a, b, c, d]) ''' Sliders, Text, and Textarea controls default to continuous_update=True. IntText and other text boxes for entering integer or float numbers default to continuous_update=False (since often you’ll want to type an entire number before submitting the value by pressing enter or navigating out of the box). '''
def __init__(self):
Bact.__init__(self)
# initializing attributes
self.model = None
# Recover and handle default settings
self.notebook_path = os.getcwd()
self.default_path_text = ipw.Text(
layout={"width": "600px"}, style={"description_width": "initial"})
self.default_path_button = ipw.Button(
description="Update default path",
layout={"width": "200px"},
style={"description_width": "initial"},
)
self.default_path_button.on_click(self.change_default)
self.default_path_from_browser_button = ipw.Button(
description="Update using browser",
layout={"width": "200px"},
style={"description_width": "initial"},
)
self.default_path_from_browser_button.on_click(
self.change_default_from_browser)
# create browser
self.folders = Folders()
self.folders.file_list.observe(self.get_filenames, names="options")
# option to name saving/loading folder
self.saveto_widget = ipw.Text(value="Segmented")
self.saveto_widget.observe(self.update_saveto, names="value")
# output widgets
self.outcheck = ipw.Output()
self.out = ipw.Output()
# widget for naming of acquired channels
self.channel_field = ipw.Text(
description="Channels",
layout={"width": "700px"},
value="DAPI, GFP, Phal, Unknown",
)
self.channel_field.observe(self.update_values, names="value")
# widget to set minimal nucleus size
self.minsize_field = ipw.IntText(
description="Minimal nucleus size",
layout={"width": "200px"},
style={"description_width": "initial"},
value=0,
)
self.minsize_field.observe(self.update_minsize, names="value")
# widget to turn hole filling in nuclei ON/OFF
self.fillholes_checks = ipw.Checkbox(description="Fill holes",
value=False)
self.fillholes_checks.observe(self.update_fillholes, names="value")
# widget to select nucleus masking on/off
self.what_to_keep = ipw.RadioButtons(
options=[
'Only in nucleus', 'Only in cells', 'In both nucleus and cells'
],
value='Only in cells',
description='Select bacteria present in:',
layout={"width": "500px"},
style={"description_width": "initial"})
self.what_to_keep.observe(self.update_what_to_keep, names="value")
# widget to set the diameter used by cellpose for nuclei
self.cellpose_diam_field = ipw.IntText(
description="Average nucleus diameter",
layout={"width": "400px"},
style={"description_width": "initial"},
value=60,
)
self.cellpose_diam_field.observe(self.update_cellpose_diam,
names="value")
# widget for cellpose selection of cyto or nucleus model
self.cellpose_seg_type = ipw.Select(
description="Model type",
options=["cyto", "nuclei"],
layout={"width": "400px"},
style={"description_width": "initial"})
self.cellpose_seg_type.observe(self.update_cellpose_seg_type,
names="value")
# widget to set specific zoom to use if multiple are available
self.zoom_field = ipw.IntText(
description="zoom to analyze",
layout={"width": "200px"},
style={"description_width": "initial"},
value=None,
)
self.zoom_field.observe(self.update_zoom, names="value")
self.zoom_field.observe(self.get_filenames, names="value")
# widget to turn zoom selection ON/OFF
self.zoom_on_off = ipw.Checkbox(description="Select zoom", value=False)
self.out_zoom = ipw.Output()
self.zoom_on_off.observe(self.zoom_params, names="value")
# Selection of nuclei segmentation mode
self.seg_type = ipw.Select(options=["Custom", "Manual ML", "Cellpose"])
self.out_seg = ipw.Output()
self.seg_type.observe(self.seg_type_params, names="value")
# widget for selection of channels to use for nuclei, bacteria and cell
# segmentation
self.nucl_channel_seg = ipw.Select(
options=self.channel_field.value.replace(" ", "").split(","),
layout={"width": "200px"},
style={"description_width": "initial"},
value=None,
)
self.bact_channel_seg = ipw.Select(
options=self.channel_field.value.replace(" ", "").split(","),
layout={"width": "200px"},
style={"description_width": "initial"},
value=None,
)
self.cell_channel_seg = ipw.Select(
options=self.channel_field.value.replace(" ", "").split(","),
layout={"width": "200px"},
style={"description_width": "initial"},
value=None,
)
self.actin_channel_seg = ipw.Select(
options=self.channel_field.value.replace(" ", "").split(","),
layout={"width": "200px"},
style={"description_width": "initial"},
value=None,
)
self.nucl_channel_seg.observe(self.update_nucl_channel, names="value")
self.bact_channel_seg.observe(self.update_bact_channel, names="value")
self.cell_channel_seg.observe(self.update_cell_channel, names="value")
self.actin_channel_seg.observe(self.update_actin_channel,
names="value")
# context for display of channels
self.out_channels = ipw.Output()
# widget to turn actin detection on/off
self.actin_on_off = ipw.Checkbox(description="Analyze actin",
value=False)
self.actin_on_off.observe(self.update_displayed_channels,
names="value")
self.update_displayed_channels(change=None)
# widget for loading of segmentation
self.load_button = ipw.Button(
description="Load ML and/or segmentation",
layout={"width": "200px"},
style={"description_width": "initial"},
)
self.load_button.on_click(self.load_existing)
# widget for loading ML model not present in current folder
self.load_otherML_button = ipw.Button(
description="Load alternative ML")
self.load_otherML_button.on_click(self.load_otherML)
self.MLfolder = Folders()
# widget for running the complete segmentation
self.analyze_button = ipw.Button(
description="Run segmentation",
layout={"width": "200px"},
style={"description_width": "initial"},
)
self.analyze_button.on_click(self.run_interactive_analysis)
# widget for saving the segmentation
self.save_button = ipw.Button(
description="Save segmentation",
layout={"width": "200px"},
style={"description_width": "initial"},
)
self.save_button.on_click(self.save_interactive_segmentation)
# widget for running interactive napari visualization
self.show_button = ipw.Button(
description="Show segmentation",
layout={"width": "200px"},
style={"description_width": "initial"},
)
self.show_button.on_click(self.show_interactive_segmentation)
# recover default path
if os.path.isfile("settings.txt"):
with open("settings.txt", "r") as f:
default_path = f.readline()
os.chdir(default_path)
self.folders.cur_dir = Path(".").resolve()
self.folders.refresh()
self.default_path_text.value = default_path
else:
with open("settings.txt", "w") as f:
f.write(os.getcwd())
# initialize files and variables
self.get_filenames()
self.initialize_output()
self.update_values(None)
def init_gui(self):
stretches = ['linear', 'sqrt', 'log']
slider_layout = {'min_width': '50%'}
minmax_layout = {'width': '100px'}
stretch_layout = {'width': '80px'}
text_range = ipywidgets.Label(value='', layout={'min_width': '50%'})
text_min = ipywidgets.Label(value='min.', layout=minmax_layout)
text_max = ipywidgets.Label(value='max.', layout=minmax_layout)
text_stretch = ipywidgets.Label(value='stretch', layout=stretch_layout)
self.box_text = ipywidgets.HBox(
[text_range, text_min, text_max, text_stretch])
self.slider_r = ipywidgets.FloatRangeSlider(
description='<b style="color:#cc3737">Red:</b>',
min=-9e99,
max=9e99,
step=1e98,
value=(-9e99, 9e99),
layout=slider_layout)
self.min_r = ipywidgets.FloatText(value=-9e99, layout=minmax_layout)
self.max_r = ipywidgets.FloatText(value=9e99, layout=minmax_layout)
self.stretch_r = ipywidgets.Dropdown(options=stretches,
value='linear',
layout=stretch_layout)
self.box_r = ipywidgets.HBox(
[self.slider_r, self.min_r, self.max_r, self.stretch_r])
self.slider_r.observe(self.scale_red, names='value')
self.stretch_r.observe(self.scale_red, names='value')
self.min_r.observe(self.refresh_slider, names='value')
self.max_r.observe(self.refresh_slider, names='value')
self.slider_b = ipywidgets.FloatRangeSlider(
description='<b style="color:#496bd8">Blue:</b>',
min=-9e99,
max=9e99,
step=1e98,
value=(-9e99, 9e99),
layout=slider_layout)
self.min_b = ipywidgets.FloatText(value=-9e99, layout=minmax_layout)
self.max_b = ipywidgets.FloatText(value=9e99, layout=minmax_layout)
self.stretch_b = ipywidgets.Dropdown(options=stretches,
value='linear',
layout=stretch_layout)
self.box_b = ipywidgets.HBox(
[self.slider_b, self.min_b, self.max_b, self.stretch_b])
self.slider_b.observe(self.scale_blue, names='value')
self.stretch_b.observe(self.scale_blue, names='value')
self.min_b.observe(self.refresh_slider, names='value')
self.max_b.observe(self.refresh_slider, names='value')
cube_shape = self.ctrl.get_shape()
ch_len = cube_shape[0]
self.visible_r = ipywidgets.Checkbox(description='view',
value=True,
layout={'width': '140px'})
self.visible_r.observe(self.refresh_image)
self.slider_chr = ipywidgets.IntSlider(
description='<b style="color:#cc3737">ch-R:</b>',
min=0,
max=ch_len - 1,
step=1,
value=0,
layout=slider_layout)
self.slider_chr.observe(self.refresh_image, names='value')
self.nbin_r = ipywidgets.IntText(description='nbin',
value=1,
layout={'width': '140px'})
self.nbin_r.observe(self.nbin_red)
self.chbox_r = ipywidgets.HBox(
[self.visible_r, self.slider_chr, self.nbin_r])
self.visible_b = ipywidgets.Checkbox(description='view',
value=True,
layout={'width': '140px'})
self.visible_b.observe(self.refresh_image)
self.slider_chb = ipywidgets.IntSlider(
description='<b style="color:#496bd8">ch-B:</b>',
min=0,
max=ch_len - 1,
step=1,
value=0,
layout=slider_layout)
self.slider_chb.observe(self.refresh_image, names='value')
self.nbin_b = ipywidgets.IntText(description='nbin',
value=1,
layout={'width': '140px'})
self.nbin_b.observe(self.nbin_blue)
self.chbox_b = ipywidgets.HBox(
[self.visible_b, self.slider_chb, self.nbin_b])
shape = self.ctrl.get_shape()
self.img = ipywidgets.Image(height=shape[1],
width=shape[2],
format='bmp')
return
output_text = widgets.Text()
input_text = widgets.Text()
def get_city_name_from_user(text):
output_text.value = input_text.value
input_text.on_submit(get_city_name_from_user)
print("Please enter city name, hit enter, then run cells below:")
print("Examples include: \nSan Francisco, CA\nRome\nParis\nMoscow, Russia\nBarcelona, Spain\nDubai")
display(input_text)
num_agents = widgets.IntText(
value=7,
description='',
disabled=False
)
num_food_sources = widgets.IntText(
value=7,
description='',
disabled=False
)
map_size = widgets.IntText(
value=300,
description='',
disabled=False
)
print("Please enter the number of agents: ")
display(num_agents)
print("Please enter the number of food sources: ")
def tajima_moving_widget():
style = {'description_width': '350px'}
layout = Layout(width='55%')
a = widgets.Text(value='casea-moving-24.txt',
description='Template Input File:',
style=style,
layout=layout)
b = widgets.Text(value='case1-moving.txt',
description='New Output File:',
style=style,
layout=layout)
c = widgets.BoundedFloatText(value=0.0001,
min=0.0,
max=2.0,
description='v_e/c:',
style=style,
layout=layout)
d = widgets.FloatText(value=1.0,
description='a0:',
style=style,
layout=layout)
e = widgets.BoundedFloatText(value=10.0,
min=0,
max=30,
description='omega0:',
style=style,
layout=layout)
f = widgets.BoundedFloatText(value=3.14,
min=0,
max=100,
description='t_flat:',
style=style,
layout=layout)
g = widgets.BoundedFloatText(value=0,
min=0,
max=100,
description='t_rise:',
style=style,
layout=layout)
h = widgets.BoundedFloatText(value=0,
min=0,
max=100,
description='t_fall:',
style=style,
layout=layout)
xmaxw = widgets.FloatText(value=24,
description='xmax:',
style=style,
layout=layout)
ndumpw = widgets.IntText(value=2,
description='ndump:',
style=style,
layout=layout)
ppc = widgets.IntText(value=10,
description='Particles per cell:',
style=style,
layout=layout)
tmaxw = widgets.FloatText(value=200,
description='tmax:',
style=style,
layout=layout)
im_moving = interact_calc(newifile2,
iname=a,
oname=b,
uth=c,
a0=d,
omega0=e,
t_flat=f,
t_rise=g,
t_fall=h,
xmax=xmaxw,
ndump=ndumpw,
ppc=ppc,
tmax=tmaxw)
im_moving.widget.manual_button.layout.width = '250px'
def publish(node, topic, msg_type):
"""
Create a form widget for message type msg_type.
This function analyzes the fields of msg_type and creates
an appropriate widget.
A publisher is automatically created which publishes to the
topic given as topic parameter. This allows pressing the
"Send Message" button to send the message to ROS.
@param node An rclpy node class
@param msg_type The message type
@param topic The topic name to publish to
@return jupyter widget for display
"""
# Check if a ros2 node is provided.
if (not isinstance(node, rclpy.node.Node)
or not issubclass(type(node), rclpy.node.Node)):
raise TypeError(
"Input argument 'node' is not of type rclpy.node.Node!")
# Check if topic already created.
for operating_publisher in node.publishers:
if topic[0] != "/":
if "/" + topic is operating_publisher.topic:
print(f"Publisher for topic, /{topic}, already created!")
return
if topic is operating_publisher.topic:
print(f"Publisher for topic, {topic}, already created!")
return
publisher = node.create_publisher(msg_type, topic, 10)
widget_list = []
widget_dict = {}
latch_check = widgets.Checkbox(description="Latch Message")
rate_field = widgets.IntText(description="Rate", value=5)
stop_btn = widgets.Button(description="Start")
def latch_value_change(arg):
publisher.impl.is_latch = arg['new']
latch_check.observe(latch_value_change, 'value')
add_widgets(msg_type(), widget_dict, widget_list)
send_btn = widgets.Button(description="Send Message")
def send_msg(arg):
msg_to_send = msg_type()
widget_dict_to_msg(msg_to_send, widget_dict)
publisher.publish(msg_to_send)
print("Message Sent!")
send_btn.on_click(send_msg)
THREAD_MAP[topic] = False
def thread_target():
d = 1.0 / float(rate_field.value)
while THREAD_MAP[topic]:
send_msg(None)
time.sleep(d)
def start_thread(click_args):
THREAD_MAP[topic] = not THREAD_MAP[topic]
if THREAD_MAP[topic]:
local_thread = threading.Thread(target=thread_target)
local_thread.start()
stop_btn.description = "Stop"
else:
stop_btn.description = "Start"
stop_btn.on_click(start_thread)
btm_box = widgets.HBox((send_btn, latch_check, rate_field, stop_btn))
widget_list.append(btm_box)
vbox = widgets.VBox(children=widget_list)
return vbox
def tajima_widget2():
style = {'description_width': '350px'}
layout = Layout(width='55%')
a = widgets.Text(value='linear-a03-w10.txt',
description='Template Input File:',
style=style,
layout=layout)
b = widgets.Text(value='linear-a03-w7_2.txt',
description='New Output File:',
style=style,
layout=layout)
c = widgets.BoundedFloatText(value=1e-4,
min=0.0,
max=2.0,
description='v_e/c:',
style=style,
layout=layout)
d = widgets.FloatText(value=0.3,
description='a0:',
style=style,
layout=layout)
e = widgets.BoundedFloatText(value=10,
min=0,
max=15,
description='omega0:',
style=style,
layout=layout)
f = widgets.BoundedFloatText(value=3.14159,
min=0,
max=100,
description='Lt:',
style=style,
layout=layout)
g = widgets.BoundedFloatText(value=0,
min=0,
max=100,
description='t_rise:',
style=style,
layout=layout)
h = widgets.BoundedFloatText(value=0,
min=0,
max=100,
description='t_fall:',
style=style,
layout=layout)
nx_pw = widgets.IntText(value=1600,
description='nx_p:',
style=style,
layout=layout)
xmaxw = widgets.FloatText(value=24.0,
description='xmax:',
style=style,
layout=layout)
ndumpw = widgets.IntText(value=2,
description='ndump:',
style=style,
layout=layout)
ppc = widgets.IntText(value=10,
description='Particles per cell:',
style=style,
layout=layout)
print('d=' + repr(d))
im = interact_calc(newifile,
iname=a,
oname=b,
uth=c,
a0=d,
omega0=e,
t_flat=f,
t_rise=g,
t_fall=h,
nx_p=nx_pw,
xmax=xmaxw,
ndump=ndumpw,
ppc=ppc)
im.widget.manual_button.layout.width = '250px'
def param(self):
""" Interface de paramétrage.
Cette fonction affiche dans une cellule de dialogue les paramètres
courants et offre la possibilité de les modifier graphiquement.
"""
def update_parameters(retry, keep, sample, features, factor, width,
localv, utime):
self.learn_params['retry_number'] = retry
self.learn_params['keep_best_number'] = keep
self.learn_params['samples_percent'] = sample
self.learn_params['max_features'] = features
self.tube_params['tube_factor'] = factor
self.tube_params['filter_width'] = width
self.feature_params['local_value'] = localv
self.feature_params['use_time'] = utime
# =================================================================
# Widgets pour les indicateurs.
wtrn = widgets.IntText(description="Retry number",
value=self.learn_params['retry_number'],
layout=widgets.Layout(width='250px'))
wtkb = widgets.IntText(description="Final population",
value=self.learn_params['keep_best_number'],
layout=widgets.Layout(width='250px'))
wtsp = widgets.FloatText(description="Sample percentage",
value=self.learn_params['samples_percent'],
layout=widgets.Layout(width='250px'))
wtmf = widgets.IntText(description="Maximum features",
value=self.learn_params['max_features'],
layout=widgets.Layout(width='250px'))
# Widgets pour les caractéristiques spéciales.
wsmc = widgets.Select(options=['Absolute', 'Median', 'Causal'],
value=self.feature_params['local_value'],
description='Local value',
rows=3,
layout=widgets.Layout(width='250px'))
wsmt = widgets.Select(options=['Yes', 'No'],
value=self.feature_params['use_time'],
description='Use time',
rows=2,
layout=widgets.Layout(width='250px'))
# Widgets pour les tubes.
wttt = widgets.FloatText(description="Tube size",
value=self.tube_params['tube_factor'],
layout=widgets.Layout(width='250px'))
wttw = widgets.IntText(description="Filter width",
value=self.tube_params['filter_width'],
layout=widgets.Layout(width='250px'))
fbox = widgets.VBox([
widgets.Label("Learning parameters (.learn_params):"),
widgets.HBox([wtrn, wtkb]),
widgets.HBox([wtsp, wtmf]),
widgets.Label("Feature parameters (.feature_params):"),
widgets.HBox([wsmc, wsmt]),
widgets.Label("Tube parameters (.tube_params:)"),
widgets.HBox([wttt, wttw])
])
out = widgets.interactive(update_parameters,
retry=wtrn,
keep=wtkb,
sample=wtsp,
features=wtmf,
factor=wttt,
width=wttw,
localv=wsmc,
utime=wsmt)
return fbox
def get_sliders(update):
# Start_box value indicates the desired start point of queried window.
start_box = widgets.FloatText(
description="Start Time:",
disabled=True,
)
# End_box value indicates the desired end point of queried window.
end_box = widgets.FloatText(
description="End Time:",
disabled=True,
)
# Percentage slider. Indicates either % of total time or total tasks
# depending on what breakdown_opt is set to.
range_slider = widgets.IntRangeSlider(
value=[0, 100],
min=0,
max=100,
step=1,
description="%:",
continuous_update=False,
orientation="horizontal",
readout=True,
)
# Indicates the number of tasks that the user wants to be returned. Is
# disabled when the breakdown_opt value is set to total_time_value.
num_tasks_box = widgets.IntText(description="Num Tasks:", disabled=False)
# Dropdown bar that lets the user choose between modifying % of total
# time or total number of tasks.
breakdown_opt = widgets.Dropdown(
options=[total_time_value, total_tasks_value],
value=total_tasks_value,
description="Selection Options:")
# Display box for layout.
total_time_box = widgets.VBox([start_box, end_box])
# This sets the CSS style display to hide the box.
total_time_box.layout.display = 'none'
# Initially passed in to the update_wrapper function.
INIT_EVENT = "INIT"
# Create instance of context manager to determine whether callback is
# currently executing
out_recursion = _EventRecursionContextManager()
def update_wrapper(event):
# Feature received a callback, but it shouldn't be executed
# because the callback was the result of a different feature
# executing its callback based on user input.
if not out_recursion.should_recurse:
return
# Feature received a callback and it should be executed because
# the callback was the result of user input.
with out_recursion:
smallest, largest, num_tasks = ray.global_state._job_length()
diff = largest - smallest
if num_tasks is not 0:
# Describes the initial values that the slider/text box
# values should be set to.
if event == INIT_EVENT:
if breakdown_opt.value == total_tasks_value:
num_tasks_box.value = -min(10000, num_tasks)
range_slider.value = (
int(100 -
(100. * -num_tasks_box.value) / num_tasks),
100)
else:
low, high = map(lambda x: x / 100., range_slider.value)
start_box.value = round(diff * low, 2)
end_box.value = round(diff * high, 2)
# Event was triggered by a change in the start_box value.
elif event["owner"] == start_box:
if start_box.value > end_box.value:
start_box.value = end_box.value
elif start_box.value < 0:
start_box.value = 0
low, high = range_slider.value
range_slider.value = (int(
(start_box.value * 100.) / diff), high)
# Event was triggered by a change in the end_box value.
elif event["owner"] == end_box:
if start_box.value > end_box.value:
end_box.value = start_box.value
elif end_box.value > diff:
end_box.value = diff
low, high = range_slider.value
range_slider.value = (low,
int((end_box.value * 100.) / diff))
# Event was triggered by a change in the breakdown options
# toggle.
elif event["owner"] == breakdown_opt:
if breakdown_opt.value == total_tasks_value:
start_box.disabled = True
end_box.disabled = True
num_tasks_box.disabled = False
total_time_box.layout.display = 'none'
# Make CSS display go back to the default settings.
num_tasks_box.layout.display = None
num_tasks_box.value = min(10000, num_tasks)
range_slider.value = (
int(100 -
(100. * num_tasks_box.value) / num_tasks), 100)
else:
start_box.disabled = False
end_box.disabled = False
num_tasks_box.disabled = True
# Make CSS display go back to the default settings.
total_time_box.layout.display = None
num_tasks_box.layout.display = 'none'
range_slider.value = (int(
(start_box.value * 100.) /
diff), int((end_box.value * 100.) / diff))
# Event was triggered by a change in the range_slider
# value.
elif event["owner"] == range_slider:
low, high = map(lambda x: x / 100., range_slider.value)
if breakdown_opt.value == total_tasks_value:
old_low, old_high = event["old"]
new_low, new_high = event["new"]
if old_low != new_low:
range_slider.value = (new_low, 100)
num_tasks_box.value = (-(100. - new_low) / 100. *
num_tasks)
else:
range_slider.value = (0, new_high)
num_tasks_box.value = new_high / 100. * num_tasks
else:
start_box.value = round(diff * low, 2)
end_box.value = round(diff * high, 2)
# Event was triggered by a change in the num_tasks_box
# value.
elif event["owner"] == num_tasks_box:
if num_tasks_box.value > 0:
range_slider.value = (0,
int(100 *
float(num_tasks_box.value) /
num_tasks))
elif num_tasks_box.value < 0:
range_slider.value = (100 + int(
100 * float(num_tasks_box.value) / num_tasks), 100)
if not update:
return
diff = largest - smallest
# Low and high are used to scale the times that are
# queried to be relative to the absolute time.
low, high = map(lambda x: x / 100., range_slider.value)
# Queries to task_profiles based on the slider and text
# box values.
# (Querying based on the % total amount of time.)
if breakdown_opt.value == total_time_value:
tasks = _truncated_task_profiles(
start=(smallest + diff * low),
end=(smallest + diff * high))
# (Querying based on % of total number of tasks that were
# run.)
elif breakdown_opt.value == total_tasks_value:
if range_slider.value[0] == 0:
tasks = _truncated_task_profiles(num_tasks=(int(
num_tasks * high)),
fwd=True)
else:
tasks = _truncated_task_profiles(num_tasks=(int(
num_tasks * (high - low))),
fwd=False)
update(smallest, largest, num_tasks, tasks)
# Get updated values from a slider or text box, and update the rest of
# them accordingly.
range_slider.observe(update_wrapper, names="value")
breakdown_opt.observe(update_wrapper, names="value")
start_box.observe(update_wrapper, names="value")
end_box.observe(update_wrapper, names="value")
num_tasks_box.observe(update_wrapper, names="value")
# Initializes the sliders
update_wrapper(INIT_EVENT)
# Display sliders and search boxes
display(breakdown_opt,
widgets.HBox([range_slider, total_time_box, num_tasks_box]))
# Return the sliders and text boxes
return start_box, end_box, range_slider, breakdown_opt
def __init__(self,
the_workchain=None,
job_details=None,
presub_calls=[],
**kwargs):
""" Submit Button
:param
"""
self.the_workchain = the_workchain
self.submit_details = {}
self.job_details = job_details
# list of methods to call after submit button is pressed
self.presub_calls = presub_calls
self.btn_submit = ipw.Button(description="Submit", disabled=False)
self.walltime = ipw.IntText(value=86000,
description='walltime',
style={'description_width': '120px'},
layout={'width': '30%'})
self.submit_out = ipw.Output()
self.job_details['walltime'] = self.walltime.value
def set_walltime(c):
self.job_details['walltime'] = self.walltime.value
def on_btn_submit_press(b):
for presub_call in self.presub_calls:
presub_call()
#self.parse_job_details()
keys_defined = self.job_details.keys()
with self.submit_out:
clear_output()
### CHECK VALIDITY OF INPUTS
if 'structure' not in keys_defined:
print("Please select a structure.")
return
if 'cp2k_code' not in keys_defined:
print("Please select a computer.")
return
if len(self.job_details['calc_name']) < 5:
print("Please enter a longer calculation description.")
return
### ODD CHARGE AND RKS
odd_charge = self.job_details['slab_analyzed']['total_charge']
if 'charge' in self.job_details.keys():
odd_charge += self.job_details['charge']
rks = True
if 'uks_switch' in self.job_details.keys():
if self.job_details['uks_switch'] == 'UKS':
rks = False
if odd_charge % 2 > 0 and rks:
print("ODD CHARGE AND RKS")
return
if self.job_details['workchain'] == 'NEBWorkchain':
if len(self.job_details['replica_pks'].split()) < 2:
print('Please select at least two replica_pks')
return
if self.job_details['workchain'] == 'PhononsWorkchain':
num_calcs = self.job_details['ncalcs']
num_rep = self.job_details['nreplicas']
if not (num_calcs % num_rep) == 0:
print('#Calculations', num_calcs)
print('#Replicas', num_rep)
print(
'Choose #Replicas so that #Calculations % #Replicas is 0.'
)
return
self.structure = self.job_details['structure']
self.code = self.job_details['cp2k_code']
print("SUBMITTING workchain ", self.the_workchain)
print("")
self.btn_submit.disabled = True
self.parse_job_details()
for field in self.submit_details.keys():
#if field != 'slab_analyzed':
print(field, self.submit_details.get_dict()[field])
if self.job_details['calc_name'] != 'DO NOT SUBMIT':
arg_dict = {
'code': self.code,
'structure': self.structure,
'parameters': self.submit_details
}
if self.struc_folder is not None:
arg_dict['struc_folder'] = self.struc_folder
outputs = submit(self.the_workchain, **arg_dict)
print(outputs)
else:
print("TEST NO SUBMISSION")
theworkchain = self.the_workchain()
outputs = theworkchain.build_calc_inputs(
structure=self.structure,
input_dict=self.submit_details.get_dict())
print(outputs['parameters'].get_dict())
print("")
print("DONE")
if self.job_details['calc_name'] != 'DO NOT SUBMIT':
the_workcalc = load_node(outputs.pid)
the_workcalc.description = self.job_details['calc_name']
self.btn_submit.on_click(on_btn_submit_press)
self.walltime.observe(set_walltime, 'value')
### ---------------------------------------------------------
### Define the ipw structure and create parent VBOX
children = [
ipw.HBox([self.btn_submit, self.walltime]), self.submit_out
]
super(SubmitButton, self).__init__(children=children, **kwargs)
class workFlow:
#Settings:
outputFilename = 'workflowJobs.data'
motionCorrFolderName = 'NBMotionCorr/'
gctfFolderName = 'NBCtfFind/'
gautomatchFolderName = 'NBAutoPick/'
#style and Layout
styleBasic = {'description_width': '120px'}
styleAdvanced = {'description_width': '130px'}
basicLayout = Layout(width='60%')
errorLayout = Layout(width='60%', border='2px solid red')
#Input fields for Workflow
projectDirectory = widgets.Text(description='Project Directory: ',
placeholder='Relion project directory',
value='',
disabled=False,
style=styleBasic,
layout=basicLayout)
mode = widgets.ToggleButtons(
options=['Single', 'Workflow'],
value='Workflow',
description='Mode:',
disabled=False,
button_style='',
tooltips=['Run a separate processes', 'Run in workflow mode'])
loadButton = widgets.Button(description='Load jobs',
disabled=False,
button_style='',
tooltip='Load all jobs',
icon='check')
saveButton = widgets.Button(description='Save jobs',
disabled=False,
button_style='',
tooltip='Save all jobs',
icon='check')
errorText = widgets.Text(description='',
placeholder='Errors',
disabled=True,
style=styleBasic,
layout=basicLayout)
runAllButton = widgets.Button(description='Run all jobs',
disabled=False,
button_style='',
tooltip='Run all workflow jobs',
icon='check')
runProgress = widgets.IntProgress(value=0,
min=0,
max=10,
step=1,
description='Progress:',
bar_style='',
orientation='horizontal')
displayGraphs = widgets.RadioButtons(
options=(['Disable', 'Enable']),
description='Display graphs:',
value='Enable',
tooltip='Display graphs after job run',
disabled=False,
style=styleBasic)
graphBox = widgets.VBox([])
## AutoRun fields - thread control
## Start
enableAutoRun = widgets.RadioButtons(options=(['Disable', 'Enable']),
description='Auto run:',
disabled=False,
style=styleBasic)
startAutoRun = widgets.Button(description='Start auto run',
description_tooltip='Start auto execution',
disabled=False,
style=styleBasic)
runDelay = widgets.IntText(
value=5,
description='Pause (mins): ',
description_tooltip='Auto run after a pause in minutes',
disabled=False,
style=styleBasic)
runCounter = widgets.IntText(value=0,
description='Running (secs): ',
disabled=True,
style=styleBasic)
stopAutoRun = widgets.Button(description='Stop auto run',
description_tooltip='Stop auto execution',
disabled=False,
style=styleBasic)
#thread performing autoRun
threadAutoRun = None
threadRunCounter = None
#thread event - controls running of the thread e.g. stop/start.
threadAutoRun_stop = threading.Event()
threadRunCounter_stop = threading.Event()
## End
##Debug assistance
debug = widgets.Output(style=styleBasic, layout=Layout(width='90%'))
debugText = widgets.Textarea(description='Debugging:',
description_tooltip='Standard output',
disabled=False,
rows=10,
style=styleBasic,
layout=Layout(width='90%'))
motionCorrection = None
gctf = None
gautomatch = None
# __init__() - initialise the class jobMaintenance
# Arguments:
# motionCorrection - an instance of the motionCorrection class
# ctf - an instance of the gctf class
# gautomatch - an instance of the gautomatch class
# showDebug - display debug fields
#
def __init__(self, motionCorrection, gctf, gautomatch, showDebug):
self.motionCorrection = motionCorrection
self.gctf = gctf
self.gautomatch = gautomatch
self.showDebug = showDebug
#default to 'workflow' mode
self.workflowProcess()
#Load all jobs from file
@debug.capture(clear_output=True)
def loadJobs(self, target):
if self.projectDirectory.value.endswith('/') == False:
self.projectDirectory.value += '/'
self.errorText.layout = self.basicLayout
#read the list, unpickle it.
try:
with open(self.projectDirectory.value + self.outputFilename,
'rb') as filehandle:
# read the data as binary data stream
allJobs = pickle.load(filehandle)
except OSError as err:
self.errorText.value = "OS error: {0}".format(err)
self.errorText.layout = self.errorLayout
else:
filehandle.close()
#extract each jobs list from allJobs.
self.motionCorrection.jobMaint.jobsList.options = list(allJobs[0])
self.gctf.jobMaint.jobsList.options = list(allJobs[1])
self.gautomatch.jobMaint.jobsList.options = list(allJobs[2])
##updating the JobNo as the notebook may have been restarted. ##
#obtain the latest jobNumber
lenMotionCorrJobs = len(allJobs[0])
maxMotionCorrJobNo = allJobs[0][lenMotionCorrJobs - 1]['jobNumber']
#left strip the jobNumber by removing the jobPrefix, convert to int, add 1, assign to jobCounter.
self.motionCorrection.jobMaint.jobCounter = int(
maxMotionCorrJobNo.lstrip(self.motionCorrection.jobPrefix)) + 1
#obtain the latest jobNumber
lenGctfJobs = len(allJobs[1])
maxGctfJobNo = allJobs[1][lenGctfJobs - 1]['jobNumber']
#left strip the jobNumber by removing the jobPrefix, convert to int, add 1, assign to jobCounter.
self.gctf.jobMaint.jobCounter = int(
maxGctfJobNo.lstrip(self.gctf.jobPrefix)) + 1
#obtain the latest jobNumber
lenGautomatchJobs = len(allJobs[2])
maxGautomatchJobNo = allJobs[2][lenGautomatchJobs - 1]['jobNumber']
#left strip the jobNumber by removing the jobPrefix, convert to int, add 1, assign to jobCounter.
self.gautomatch.jobMaint.jobCounter = int(
maxGautomatchJobNo.lstrip(self.gautomatch.jobPrefix)) + 1
#Save all jobs to file
@debug.capture(clear_output=True)
def saveJobs(self, target):
mcListedJobs = list(self.motionCorrection.jobMaint.jobsList.options)
gctfListedJobs = list(self.gctf.jobMaint.jobsList.options)
gautoListedJobs = list(self.gautomatch.jobMaint.jobsList.options)
#add the three lists to a new list
allJobs = []
allJobs.append(mcListedJobs)
allJobs.append(gctfListedJobs)
allJobs.append(gautoListedJobs)
if self.projectDirectory.value.endswith('/') == False:
self.projectDirectory.value += '/'
self.errorText.layout = self.basicLayout
#saving the list - pickle it.
try:
with open(self.projectDirectory.value + self.outputFilename,
'wb') as filehandle:
pickle.dump(allJobs, filehandle)
except OSError as err:
self.errorText.value = "OS error: {0}".format(err)
self.errorText.layout = self.errorLayout
else:
filehandle.close()
# singleProcess() - manage fields for single processing
#
@debug.capture(clear_output=True)
def singleProcess(self):
#Motion Correction defaults
self.motionCorrection.outMrc.disabled = False
self.motionCorrection.outMrc.value = ''
self.motionCorrection.runButton.disabled = False
self.motionCorrection.jobMaint.runAllButton.disabled = False
#CTF defaults
self.gctf.inMrc.disabled = False
self.gctf.inMrc.value = ''
self.gctf.outMrc.disabled = False
self.gctf.outMrc.value = ''
self.gctf.runButton.disabled = False
self.gctf.jobMaint.runAllButton.disabled = False
#Auto Piicking defaults
self.gautomatch.inMrc.disabled = False
self.gautomatch.inMrc.value = ''
self.gautomatch.outMrc.disabled = False
self.gautomatch.outMrc.value = ''
self.gautomatch.runButton.disabled = False
self.gautomatch.jobMaint.runAllButton.disabled = False
self.displayGraphs.disabled = True
self.enableAutoRun.disabled = True
# workflowProcess() - manage fields for workflow processing
#
@debug.capture(clear_output=True)
def workflowProcess(self):
self.errorText.layout = self.basicLayout
#Motion Correction Defaults
self.motionCorrection.outMrc.disabled = True
self.motionCorrection.outMrc.value = self.motionCorrFolderName
if self.motionCorrection.frameDose.value == 0:
self.errorText.value = "Motion Correction frame dose must be > 0"
self.errorText.layout = self.errorLayout
self.motionCorrection.runButton.disabled = True
self.motionCorrection.jobMaint.runAllButton.disabled = True
#CTF defaults
self.gctf.inMrc.disabled = True
self.gctf.inMrc.value = '*.mrc'
self.gctf.outMrc.disabled = True
self.gctf.outMrc.value = self.gctfFolderName
if self.gctf.doUnfinished.value == 0:
self.errorText.value = "Contract Transfer Processing: Continue Processing must be 'Yes'"
self.errorText.layout = self.errorLayout
self.gctf.runButton.disabled = True
self.gctf.jobMaint.runAllButton.disabled = True
#Gautomatch defaults
self.gautomatch.inMrc.disabled = True
self.gautomatch.inMrc.value = '*.mrc'
self.gautomatch.outMrc.disabled = True
self.gautomatch.outMrc.value = self.gautomatchFolderName
if self.gautomatch.doUnfinished.value == 0:
self.errorText.value = "Auto Match: Autopick unfinished mrcs must be 'Yes'"
self.errorText.layout = self.errorLayout
self.gautomatch.runButton.disabled = True
self.gautomatch.jobMaint.runAllButton.disabled = True
self.displayGraphs.disabled = False
self.enableAutoRun.disabled = False
# runWorkflowJobs() - Execute MotionCorr, Gctf and Gautomatch as a workflow.
#
@debug.capture(clear_output=True)
def runWorkflowJobs(self, target):
dirExists = False
self.errorText.value = ''
self.errorText.layout = self.basicLayout
#Is the Notebook in Workflow mode?
if self.mode.value == 'Workflow':
#validate Project Directory. Does it exist?
try:
if os.path.isdir(self.projectDirectory.value) == True:
dirExists = True
else:
self.errorText.value = "Project directory is not valid"
self.errorText.layout = self.errorLayout
except OSError as err:
self.errorText.value = "OS error: {0}".format(err)
self.errorText.layout = self.errorLayout
if dirExists == True:
#reset runProgress
self.motionCorrection.jobMaint.runProgress.value = 0
self.gctf.jobMaint.runProgress.value = 0
self.gautomatch.jobMaint.runProgress.value = 0
self.runProgress.max = 4
self.runProgress.value = 1
#runAllJobs for motionCorrection
self.motionCorrection.runAllWorkflowJobs(
self.projectDirectory.value)
self.runProgress.value = 2
#runAllJobs for Gctf
self.gctf.runAllWorkflowJobs(self.projectDirectory.value,
self.motionCorrFolderName)
self.runProgress.value = 3
#runAllJobs for Gautomatch
#gautomatch uses motionCorr output for processing
self.gautomatch.runAllWorkflowJobs(self.projectDirectory.value,
self.motionCorrFolderName)
self.runProgress.value = 4
if self.displayGraphs.value == 'Enable':
self.buildGraphs()
else:
self.errorText.value = "Must be in 'workflow' mode, ensure all jobs created in this mode"
self.errorText.layout = self.errorLayout
# buildGraphs() - display a histogram for each ctf job.
#
@debug.capture(clear_output=True)
def buildGraphs(self):
self.graphBox.children = []
ctfStarFilePath = glob.glob(self.projectDirectory.value +
self.gctfFolderName +
'*/micrographs_ctf.star')
ctfStarFilePath.sort()
if len(ctfStarFilePath) > 0:
rowItemMax = 2
fig = None
axes = None
maxRows = math.ceil(len(ctfStarFilePath) / rowItemMax)
plt.close('all')
#interactive off
plt.ioff()
#clear figure
plt.clf()
fig, axes = plt.subplots(nrows=maxRows, ncols=rowItemMax)
ax = axes.flatten()
for i in range(len(ctfStarFilePath)):
split = ctfStarFilePath[i].rsplit('/', 2)
#create a numpy array from .star file using col 12 MaxCtfResolution
ctfData = np.loadtxt(ctfStarFilePath[i],
skiprows=17,
usecols=12)
ax[i].set_title("Job: " + split[1])
ax[i].set_ylabel('Probablity')
ax[i].set_xlabel("Max CTF resolution (A)")
ax[i].hist(ctfData,
bins=50,
density=True,
histtype='bar',
facecolor='xkcd:nasty green',
alpha=0.75)
fig.tight_layout(pad=1, w_pad=1, h_pad=1)
self.graphBox.children = [fig.canvas]
## AutoRun thread functions
##
# buttonStartAutoRun() - start/restart auto run processing
# Argument - change - not used, exists to cater for ipywidget button actions.
#
@debug.capture(clear_output=True)
def buttonStartAutoRun(self, change):
#check for restart, if thread event flag is set, clear it so thread can be restarted
if self.threadAutoRun_stop.isSet():
self.threadAutoRun_stop.clear()
self.threadRunCounter_stop.clear()
#allocate new thread and start it.
# When AutoRun is stopped, the thread completes processing and terminates, so a new one is required
self.threadAutoRun = threading.Thread(target=self.executeAutoRun,
args=(1,
self.threadAutoRun_stop))
self.threadAutoRun.start()
self.threadRunCounter = threading.Thread(
target=self.executeCounter, args=(1, self.threadRunCounter_stop))
self.threadRunCounter.start()
# buttonStopAutoRun() - stop auto run processing
# Argument - change - not used, exists to cater for ipywidget button actions.
#
@debug.capture(clear_output=True)
def buttonStopAutoRun(self, change):
#set Event flag to stop the looping in executeAutoRun()
self.threadAutoRun_stop.set()
self.threadRunCounter_stop.set()
# executeAutoRun() - execute runWorkflowJobs and then pause for runDelay
# Argument - change - not used, exists to cater for ipywidget button actions.
#
@debug.capture(clear_output=True)
def executeAutoRun(self, arg, threadEvent):
while not threadEvent.is_set():
self.runWorkflowJobs('')
threadEvent.wait(self.runDelay.value * 60)
# executeCounter() - updates runCounter after pausing for 1 second.
# Argument - change - not used, exists to cater for ipywidget button actions.
#
@debug.capture(clear_output=True)
def executeCounter(self, arg, threadEvent):
self.runCounter.value = 0
while not threadEvent.is_set():
threadEvent.wait(1)
self.runCounter.value += 1
# on_clickMode() - handle actions for the Mode button
#
@debug.capture(clear_output=True)
def on_clickMode(self, change):
if self.mode.value == 'Single':
#protect and fill fields as needed.
self.singleProcess()
if self.mode.value == 'Workflow':
self.workflowProcess()
# on_clickAutoRun() - handle actions for enable/disable AutoRun
#
@debug.capture(clear_output=True)
def on_clickAutoRun(self, change):
if self.enableAutoRun.value == 'Enable':
self.startAutoRun.disabled = False
self.stopAutoRun.disabled = False
self.runDelay.disabled = False
if self.enableAutoRun.value == 'Disable':
self.startAutoRun.disabled = True
self.stopAutoRun.disabled = True
self.runDelay.disabled = True
#Disabling, so stop autoRun if running.
self.buttonStopAutoRun('')
# on_clickGraph() - handle actions for the displayGraphs radio button
#
@debug.capture(clear_output=True)
def on_clickGraph(self, change):
if self.displayGraphs.value == 'Disable':
self.graphBox.children = []
# buildInputWidgets() - write all the workflow fields to the screen.
#
@debug.capture(clear_output=True)
def buildWidgets(self):
#linking button on_click to function
self.loadButton.on_click(self.loadJobs)
self.saveButton.on_click(self.saveJobs)
self.runAllButton.on_click(self.runWorkflowJobs)
self.startAutoRun.on_click(self.buttonStartAutoRun)
self.stopAutoRun.on_click(self.buttonStopAutoRun)
jobButtons = HBox([self.loadButton, self.saveButton, self.errorText])
runBox = HBox([self.runAllButton, self.runProgress])
autoRunBox = HBox([
self.enableAutoRun, self.startAutoRun, self.stopAutoRun,
self.runDelay, self.runCounter
])
self.mode.observe(self.on_clickMode, 'value')
self.enableAutoRun.observe(self.on_clickAutoRun, 'value')
self.displayGraphs.observe(self.on_clickGraph, 'value')
#Disable AutoRun fields at startup
self.on_clickAutoRun('Disable')
if self.showDebug:
return VBox([
self.debug, self.debugText, self.mode, self.projectDirectory,
jobButtons, runBox, autoRunBox, self.displayGraphs,
self.graphBox
])
else:
return VBox([
self.debug, self.mode, self.projectDirectory, jobButtons,
runBox, autoRunBox, self.displayGraphs, self.graphBox
])
description='Start T1:',
disabled=False)
w_enddate1 = widgets.Text(value='2020-07-01',
placeholder=' ',
description='End T1:',
disabled=False)
w_startdate2 = widgets.Text(value='2020-08-01',
placeholder=' ',
description='Start T2:',
disabled=False)
w_enddate2 = widgets.Text(value='2020-10-01',
placeholder=' ',
description='End T2:',
disabled=False)
w_iterations = widgets.IntText(value=30,
placeholder=' ',
description='Max Iter:',
disabled=False)
w_scale = widgets.IntText(value=30,
placeholder=' ',
description='Scale:',
disabled=False)
w_exportname = widgets.Text(value='users/<username>/<path>',
placeholder=' ',
disabled=False)
w_location = widgets.Text(value='Jülich',
placeholder=' ',
description='',
disabled=False)
w_goto = widgets.Button(description="GoTo", disabled=False)
w_collect = widgets.Button(description="Collect", disabled=True)
def __init__(self):
# all the widgets used to configure the test
default_out_dir = os.path.join(
'.', 'output',
'pubsub-test-{}'.format(time.strftime("%Y%m%d-%H%M%S")))
default_failed_dir = os.path.join('.', 'output', 'failed')
w = Bunch(
test_execution=Bunch(
output_dir=widgets.Text(
description="Local directory to collect test outputs",
value=default_out_dir),
failed_dir=widgets.Text(
description="Local dir to store output from failed runs",
value=default_failed_dir)),
testground=Bunch(
daemon_endpoint=widgets.Text(description="Daemon Endpoint",
value='localhost:8080'),
builder=widgets.Dropdown(description="Builder",
options=['docker:go', 'exec:go']),
runner=widgets.Dropdown(
description="Runner",
options=['cluster:k8s', 'local:docker', 'local:exec']),
keep_service=widgets.Checkbox(
description="Keep pods after execution? (k8s only)",
value=False),
log_level=widgets.Dropdown(
description="Log level to set on test instances",
options=["info", "debug", "warn", "error"]),
),
time=Bunch(
setup=widgets.Text(description="Test Setup time", value='1m'),
run=widgets.Text(description="Test Runtime", value='2m'),
warm=widgets.Text(description="Warmup time", value='5s'),
cool=widgets.Text(description="Cooldown time", value='10s'),
),
node_counts=Bunch(
total=widgets.IntText(description="Total number of nodes",
disabled=True),
publisher=widgets.IntText(
description="Number of publisher nodes", value=100),
lurker=widgets.IntText(description="Number of lurker nodes",
value=50),
attacker=widgets.IntText(
description="Number of attacker nodes", value=0),
honest_per_container=widgets.IntText(
description="# of honest peers per container", value=1),
attackers_per_container=widgets.IntText(
description="# of attacker peers per container", value=1),
),
pubsub=Bunch(
branch=widgets.Text(
description="go-libp2p-pubsub branch/tag/commit to target",
value="master"),
use_hardened_api=widgets.Checkbox(
description="target hardening branch API", value=True),
heartbeat=widgets.Text(description='Heartbeat interval',
value='1s'),
hearbeat_delay=widgets.Text(
description='Initial heartbeat delay', value='100ms'),
validate_queue_size=widgets.IntText(
description='Size of validation queue', value=32),
outbound_queue_size=widgets.IntText(
description='Size of outbound RPC queue', value=32),
score_inspect_period=widgets.Text(
description='Interval to dump peer scores', value='5s'),
full_traces=widgets.Checkbox(
description='Capture full event traces)', value=False),
degree=widgets.IntText(description='D: target mesh degree',
value=6),
degree_lo=widgets.IntText(
description='D_lo: mesh degree low bound', value=5),
degree_hi=widgets.IntText(
description='D_hi: mesh degree upper bound', value=12),
degree_score=widgets.IntText(
description='D_score: peers to select by score', value=4),
degree_lazy=widgets.IntText(
description='D_lazy: lazy propagation degree', value=6),
degree_outbound=widgets.IntText(
description='D_out: outbound connection quota', value=2),
gossip_factor=widgets.FloatText(description='Gossip Factor',
value=0.25),
opportunistic_graft_ticks=widgets.IntText(
description='Opportunistic Graft heartbeat ticks',
value=60),
),
network=Bunch(
latency=widgets.Text(description="Min latency", value='5ms'),
max_latency=widgets.Text(
description=
"Max latency. If zero, latency will = min latency.",
value='50ms'),
jitter_pct=widgets.IntSlider(description="Latency jitter %",
value=10,
min=1,
max=100),
bandwidth_mb=widgets.IntText(description="Bandwidth (mb)",
value=10240),
degree=widgets.IntText(
description=
"Degree (# of initial connections) for honest peers",
value=20),
attack_degree=widgets.IntText(
description="Degree for attack peers", value=100),
# TODO: support upload of topology file
# topology_file = widgets.FileUpload(description="Upload fixed topology file", accept='.json', multiple=False),
),
honest_behavior=Bunch(
flood_publishing=widgets.Checkbox(
value=True, description='Flood Publishing', indent=False),
connect_delay=widgets.Text(
description=
'Honest peer connection delay. e.g. "30s" or "50@30s,30@1m"',
value='0s'),
connect_jitter_pct=widgets.BoundedIntText(
description='Jitter % for honest connect delay',
value=5,
min=0,
max=100),
),
attack_behavior=Bunch(
attack_node_type=widgets.Dropdown(description='Attack Type',
options=['sybil']),
single_node=widgets.Checkbox(
description='Target a single node?', value=False),
censor_single_node=widgets.Checkbox(
description='Censor target node?', value=False),
publishers_only=widgets.Checkbox(
description='Attack Publishers only?', value=False),
connect_delay=widgets.Text(
description=
'Attacker connection delay. e.g. "30s" or "50@30s,30@1m"',
value='30s'),
connect_jitter_pct=widgets.BoundedIntText(
description='Jitter % for attacker delay',
value=5,
min=0,
max=100),
sybil_degrade=widgets.BoundedFloatText(
description='Sybil Drop Probability',
value=1.0,
min=0,
max=1),
sybil_attack_delay=widgets.Text(
description=
'Sybil Attack Delay (time between connect & attack)',
value='0s'),
sybil_regraft_delay=widgets.Text(description='Regraft Delay',
value='15s'),
sybil_regraft_backoff=widgets.Text(
description='Regraft Backoff', value='1m'),
sybil_seen_cache_duration=widgets.Text(
description='Seen Cache duration', value='2m'),
),
peer_score=Bunch(
gossip_threshold=widgets.FloatText(
description='Gossip Threshold', value=-4000),
publish_threshold=widgets.FloatText(
description='Publish Threshold', value=-5000),
graylist_threshold=widgets.FloatText(
description='Graylist Threshold', value=-10000),
acceptpx_threshold=widgets.FloatText(
description='Accept PX Threshold', value=0),
opportunistic_graft_threshold=widgets.FloatText(
description='Opportunistic Graft Threshold', value=0),
ip_colocation_weight=widgets.FloatText(
description='IP Colocation Factor Weight', value=0),
ip_colocation_threshold=widgets.IntText(
description='IP Colocation Factor Threshold', value=1),
decay_interval=widgets.Text(description='Score Decay Interval',
value='1s'),
decay_to_zero=widgets.FloatText(
description='Decay Zero Threshold', value=0.01),
retain_score=widgets.Text(description="Time to Retain Score",
value='30s'),
))
# wire up node count sliders to update total node count
sum_values(w.node_counts.total, w.node_counts.publisher,
w.node_counts.lurker, w.node_counts.attacker)
self.topic_config = TopicConfigPanel()
self.save_widgets = Bunch(
save_button=widgets.Button(description='Save Config',
button_style='primary'),
load_button=widgets.Button(description='Load Saved Config',
button_style='warning'),
snapshot_filename=widgets.Text(description='Path:',
value='configs/snapshot.json'))
self.save_widgets.save_button.on_click(self.save_clicked)
self.save_widgets.load_button.on_click(self.load_clicked)
save_panel = widgets.HBox(list(self.save_widgets.values()))
self.panel = widgets.VBox([
to_collapsible_sections(w),
collapsible("Topic Config", [self.topic_config.ui()]),
save_panel,
])
self.widgets = w