def widget():
single_progress = widgets.IntProgress(min=0,
max=10,
description='Calculating EIG: ')
overall_progress = widgets.IntProgress(min=0,
max=10,
description='EIG Progress : ')
return widgets.VBox([single_progress, overall_progress])
def merging(self, output_folder):
"""combine images using algorithm provided"""
# get merging algorithm
merging_algo = self.combine_method.value
algorithm = self.__add
if merging_algo == 'mean':
algorithm = self.__mean
# get output folder
output_folder = os.path.abspath(output_folder)
# create dictionary of how the images will be combined
merging_dict = self.__create_merging_dictionary()
self.merginc_dict_debugging = merging_dict
# create final list of files to merge
final_dict_of_files_to_merge = self.__create_dict_of_files_to_merge(merging_dict)
self.final_dict_of_files_to_merge_debugging = final_dict_of_files_to_merge
final_nbr_folders = len(merging_dict.keys())
folder_level_ui = widgets.HBox([widgets.Label("Folder Progress:",
layout=widgets.Layout(width='20%')),
widgets.IntProgress(max=final_nbr_folders,
layout=widgets.Layout(width='50%'))])
display(folder_level_ui)
w1 = folder_level_ui.children[1]
nbr_files_to_merge = self.nbr_files_in_each_folder
file_level_ui = widgets.HBox([widgets.Label("File Progress:",
layout=widgets.Layout(width='20%')),
widgets.IntProgress(max=nbr_files_to_merge,
layout=widgets.Layout(width='50%'))])
display(file_level_ui)
w2 = file_level_ui.children[1]
for _index_final_folder, _final_folder in enumerate(final_dict_of_files_to_merge.keys()):
file_handler.make_or_reset_folder(os.path.join(output_folder, _final_folder))
list_files_to_merge = final_dict_of_files_to_merge[_final_folder]
for _index_files_to_merge, _files_to_merge in enumerate(list_files_to_merge):
_files_to_merge = [_file for _file in _files_to_merge]
self.files_to_merge_for_testing = _files_to_merge
o_load = Normalization()
o_load.load(file=_files_to_merge)
_data = o_load.data['sample']['data']
combined_data = self.__merging_algorithm(algorithm, _data)
self.combined_data_for_testing = combined_data
_base_name_file = os.path.basename(_files_to_merge[0])
output_file_name = os.path.join(output_folder, _final_folder, _base_name_file)
file_handler.save_data(data=combined_data, filename=output_file_name)
w2.value = _index_files_to_merge + 1
w1.value = _index_final_folder + 1
def load_data(filenames='', folder='', showing_progress=False):
'''
load the various file_name format
'''
if folder:
list_files = glob.glob(folder + '/*.*')
stack = []
if showing_progress:
w = widgets.IntProgress()
w.max = len(list_files)
display(w)
for _index, _file in enumerate(list_files):
_data = load_data(filenames=_file)
stack.append(_data)
if showing_progress:
w.value = _index+1
return stack
elif isinstance(filenames, str):
data_type = get_data_type(filenames)
if data_type == '.fits':
hdulist = fits.open(filenames, ignore_missing_end=True)
hdu = hdulist[0]
_image = np.asarray(hdu.data)
hdulist.close()
return _image
elif (data_type == '.tiff') or (data_type == '.tif'):
_image = Image.open(filenames)
return np.array(_image)
else:
return []
else: # list of filenames
list_files = filenames
stack = []
if showing_progress:
w = widgets.IntProgress()
w.max = len(list_files)
display(w)
for _index, _file in enumerate(list_files):
_data = load_data(filenames=_file)
stack.append(_data)
if showing_progress:
w.value = _index+1
return stack
def export(self):
base_folder = os.path.basename(os.path.dirname(
self.list_file_names[0])) + '_normalized'
output_folder = os.path.abspath(
os.path.join(self.output_folder_ui.selected, base_folder))
utilities.make_dir(dir=output_folder)
w = widgets.IntProgress()
w.max = len(self.files.sample)
display(w)
for _index, _file in enumerate(self.list_file_names):
basename = os.path.basename(_file)
_base, _ext = os.path.splitext(basename)
output_file_name = os.path.join(output_folder, _base + '.tiff')
file_handler.make_tiff(filename=output_file_name,
data=self.normalized_data_array[_index])
w.value = _index + 1
display(
HTML(
'<span style="font-size: 20px; color:blue">The normalized images have been '
+ 'created in ' + output_folder + '</span>'))
def export(self, output_folder):
new_folder = 'rotated_{}deg'.format(self.rotation_angle)
list_files = self.list_files
data = self.data
w = widgets.IntProgress()
w.max = len(list_files)
display(w)
output_folder = os.path.abspath(output_folder)
full_output_folder = os.path.join(output_folder, new_folder)
if not os.path.exists(full_output_folder):
os.makedirs(full_output_folder)
for _index, _file in enumerate(list_files):
_base_file_name = os.path.basename(_file)
[_base, _] = os.path.splitext(_base_file_name)
_full_file_name = os.path.join(full_output_folder, _base + '.tiff')
file_handler.make_tiff(data=data[_index], filename=_full_file_name)
w.value = _index + 1
w.close()
display(HTML(''))
display(
HTML(
'<span style="font-size: 20px; color:blue">Files created in ' +
full_output_folder + '</span>'))
def __init__(self,
instrument='CG1D',
facility='HFIR',
list_files=[],
oncat=None,
projection=[],
with_progressbar=False):
projection.append('ingested')
if with_progressbar:
box1 = widgets.HBox([widgets.Label("Retrieving Metadata ...",
layout=widgets.Layout(width='30%')),
widgets.IntProgress(max=len(list_files),
layout=widgets.Layout(width='70%'))])
display(box1)
slider = box1.children[1]
self.datafiles = {}
for _index, _file in enumerate(list_files):
self.datafiles[_file] = oncat.Datafile.retrieve(_file,
facility=facility,
instrument=instrument,
projection=projection)
if with_progressbar:
slider.value = _index
if with_progressbar:
box1.close()
def retrieve_data_and_metadata(self):
self.list_files = self.list_files_ui.selected
w = widgets.IntProgress()
w.max = len(self.list_files)
display(w)
index = 0
files_data = {}
files_metadata = {}
for index, file in enumerate(self.list_files):
_metadata = self.retrieve_metadata(file)
files_metadata[file] = _metadata
_col_names = _metadata['Label'].split(',')
_new_col_names = self.format_col_names(_col_names)
_data = self.retrieve_data(file,
nbr_comment_line=_metadata['nbr_comment_line'],
col_names=_new_col_names)
files_data[file] = _data
index += 1
w.value = index
self.files_data = files_data
self.files_metadata = files_metadata
w.close()
def run(self):
display(
HTML('<span style="font-size: 20px; color:blue">' + self.message +
'</span>'))
sample_image = gui_widgets.gui_dname(dir=self.working_dir)
list_sample_images = np.sort(
glob.glob(os.path.join(sample_image, '*.fits')))
if len(list_sample_images) > 0:
w = widgets.IntProgress()
w.max = len(list_sample_images)
display(w)
_data = []
for _index, _file in enumerate(list_sample_images):
_image = np.array(file_handler.load_data(_file), dtype=float)
_image = image_processing.single_gamma_filtering(_image)
_image[_image == np.inf] = 0 # removing inf values
_image[np.isnan(_image)] = 0
_data.append(_image)
w.value = _index + 1
[nbr_files, height, width] = np.shape(_data)
_data = np.squeeze(_data)
self.data = _data
self.list_sample_images = list_sample_images
def on_choose_workspace(changed):
with output:
output.clear_output()
workspace_paths = ws_paths[changed['new']]
comment_files = get_ipython().getoutput(f'gsutil ls {workspace_paths.get_comment_file_glob()}')
if not comment_files[0].startswith('gs://'):
display(HTML('''<div class="alert alert-block alert-warning">
No comment files found for HTML snapshots in this workspace.</div>'''))
return
progress = widgets.IntProgress(
value=0,
min=0,
max=len(comment_files),
step=1,
description=f'Retrieving {len(comment_files)} comments:',
bar_style='success',
orientation='horizontal',
layout=widgets.Layout(width='450px'),
style={'description_width': 'initial'}
)
display(progress)
comment_num = 0
comment_file_contents = []
for file in comment_files:
comment = get_ipython().getoutput(f"gsutil cat '{file}'")
version = file.replace(workspace_paths.get_subfolder(), '')
comment_file_contents.append({'version': version, 'comment': comment})
comment_num += 1
progress.value = comment_num
comments = pd.DataFrame(comment_file_contents)
display(comments)
def __init__(self,
value=None,
description='',
options=[],
tooltips=[],
branch_names=['Previous', 'Next', 'Complete'],
branch_steps=[None, None, None],
js='',
end_js='',
name=None,
master=None,
button_style='info',
reviewed=False):
self.master = master
self.prediction = value
self.reviewed = reviewed
self.logger = logging.getLogger(__name__)
self.progress = widgets.IntProgress(
min=1,
max=len(self.master.docs),
value=1,
layout=widgets.Layout(width='90%', height='14px'),
style=dict(description_width='initial'))
super().__init__(value=value,
description=description,
options=options,
tooltips=tooltips,
branch_names=branch_names,
branch_steps=branch_steps,
js=js,
end_js=end_js,
name=name,
button_style=button_style)
pass
def output_profiles(self):
# make name of output folder
output_folder = os.path.join(os.path.abspath(self.output_folder_ui.selected),
os.path.basename(os.path.dirname(self.list_file_name[0]) + '_profiles'))
output_folder = os.path.abspath(output_folder)
if os.path.isdir(output_folder):
shutil.rmtree(output_folder)
os.mkdir(output_folder)
[roi_left, roi_top, roi_width, roi_height] = self.roi
self.list_data_files_short = [os.path.basename(_file) for _file in self.df['#filename']]
time_0 = self.__get_time_stamp(self.list_data_files[0])
w = widgets.IntProgress()
w.max = len(self.profile_1d)
display(w)
for _index, _profile in enumerate(self.profile_1d):
_file_name = self.list_data_files[_index]
_time_stamp = self.__get_time_stamp(_file_name)
metadata = []
metadata.append("#Image: {}".format(_file_name))
metadata.append(
"#ROI selected (y0, x0, height, width): ({}, {}, {}, {})".format(roi_top,
roi_left,
roi_height,
roi_width))
metadata.append("#Rebin in y direction: {}".format(self.rebin))
_time_stamp_str = datetime.datetime.fromtimestamp(_time_stamp).strftime("%Y-%m-%d %H:%M:%S")
metadata.append("#Time Stamp: {}".format(_time_stamp_str))
_delta_time = _time_stamp - time_0
metadata.append("#Delta Time (s): {}".format(_delta_time))
metadata.append("#")
metadata.append("#Label: pixel_index, counts")
data = []
for _index_data, value in enumerate(self.profile_1d[_index]):
data.append("{}, {}".format(self.rebin_range[_index_data], value))
_base_file_name = os.path.basename(_file_name)
[base, _] = os.path.splitext(_base_file_name)
output_file_name = os.path.join(output_folder, base + '.txt')
make_ascii_file(metadata=metadata,
data=data,
output_file_name=output_file_name,
dim='1d')
w.value = _index+1
w.close()
display(HTML('<span style="font-size: 20px; color:blue">Files created in folder ' +
output_folder + '</span>'))
def progress_bar( output_area, label='Searching' ):
"""Creates an indefinitely repeating progress bar
which keeps running until something calls output_area.clear_output()
DOES NOT WORK PROPERLY
"""
stop = 100
i = 0
progress = widgets.IntProgress(
value=0,
min=0,
max=stop,
description=label,
bar_style='info',
orientation='horizontal'
)
with output_area:
display( progress )
while True:
if i > stop:
i = 0
sleep( .1 )
i += 1
progress.value = i
def GetFitter(dataDir, inputName, inputDir, load, genBranch_params=None):
#----------------------------------------------------------------------------
if (load):
w_load_bar = widgets.IntProgress(min=0,
max=3,
description='Loading:',
bar_style='success',
orientation='horizontal')
display(w_load_bar)
ut.defaultParameters(dataDir=dataDir,
inputName=inputName,
inputDir=inputDir)
w_load_bar.value += 1
ut.setParams()
w_load_bar.value += 1
if (genBranch_params != None):
ut.params["genBranches"] = genBranch_params
effFitter = ut.loadOrMake()
w_load_bar.value += 1
time.sleep(1)
print('Done loading')
#w_load_bar.close()
return effFitter
def converting_data(self):
try:
a = np.float(self.box.children[1].value)
b = np.float(self.box.children[3].value)
symbol = self.box.children[2].value
coeff = 1
if symbol == '+':
coeff = -1
progress_bar = widgets.IntProgress()
progress_bar.max = len(self.list_data)
display(progress_bar)
list_concentration = []
for _index, _data in enumerate(self.list_data):
_list_concentration = coeff * (a - _data) / b
list_concentration.append(_list_concentration)
progress_bar.value = _index + 1
self.list_concentration = list_concentration
progress_bar.close()
except:
display(
HTML(
'<span style="font-size: 20px; color:red">Make sure the coefficient are floats!</span>'
))
def __load_files(self):
progress_bar_layout = widgets.Layout(border='1px solid blue')
hbox = widgets.HBox([
widgets.IntProgress(description="FUll Progress",
layout=progress_bar_layout),
widgets.Label(value='', layout=widgets.Layout(width='10%'))
])
w = hbox.children[0]
nbr_groups = len(self.exp_dict.keys())
w.max = nbr_groups
label = hbox.children[1]
label.value = f"0/{nbr_groups}"
display(hbox)
for _index, _key in enumerate(self.exp_dict.keys()):
_item = self.exp_dict[_key]
_path = _item['folder']
list_files = _item['list_of_files']
full_list_files = [
os.path.join(_path, _file) for _file in list_files
]
o_norm = Normalization()
o_norm.load(file=full_list_files, notebook=True)
_data = o_norm.data['sample']['data']
_item['list_of_images'] = _data
self.exp_dict[_key] = _item
w.value = _index + 1
label.value = f"{_index+1}/{nbr_groups}"
hbox.close()
display(widgets.Label(value="Done!"))
def retrieve_time_stamp(list_images):
[_, ext] = os.path.splitext(list_images[0])
if ext.lower() in ['.tiff', '.tif']:
ext = 'tif'
elif ext.lower() == '.fits':
ext = 'fits'
else:
raise ValueError
box = widgets.HBox([widgets.Label("Retrieving Time Stamp",
layout=widgets.Layout(width='20%')),
widgets.IntProgress(min=0,
max=len(list_images),
value=0,
layout=widgets.Layout(width='50%'))
])
progress_bar = box.children[1]
display(box)
list_time_stamp = []
list_time_stamp_user_format = []
for _index, _file in enumerate(list_images):
_time_stamp = MetadataHandler.get_time_stamp(file_name=_file, ext=ext)
_time_stamp = _convert_epics_timestamp_to_rfc3339_timestamp(_time_stamp)
list_time_stamp.append(_time_stamp)
_user_format = convert_to_human_readable_format(_time_stamp)
list_time_stamp_user_format.append(_user_format)
progress_bar.value = _index + 1
box.close()
return {'list_images': list_images,
'list_time_stamp': list_time_stamp,
'list_time_stamp_user_format': list_time_stamp_user_format}
def saveSentences(self, dao, df, table_name, split_func, overwrite=False):
import pandas as pd
sents_df = pd.DataFrame(columns=[
'DATASET_ID', 'BUNCH_ID', 'DOC_NAME', 'TEXT', 'DATE', 'REF_DATE'
])
progressbar = widgets.IntProgress(min=0,
max=len(df),
value=0,
layout=widgets.Layout(width='80%'),
description='Splitting:')
progressbar.value = 0
display(progressbar)
for idx, row in df.iterrows():
bunch_id = row['BUNCH_ID']
doc_name = row['DOC_NAME']
text = row['TEXT']
date = row['DATE']
ref_date = row['REF_DATE']
sentence_id = 0
progressbar.value += 1
for sentence in split_func(text):
sentence_id += 1
sents_df.loc[len(sents_df)] = [
self.dataset_name + "_sents", bunch_id,
doc_name + "_" + str(sentence_id), sentence, date, ref_date
]
saveDFtoDB(dao, sents_df, table_name)
self.saveToWhoosh(sents_df, self.dataset_name + "_sents", overwrite)
pass
def export(self, output_folder):
input_folder = self.get_input_folder()
# output_folder = os.path.abspath(os.path.join(self.output_folder_ui.selected,
# "{}_rebin_by_{}".format(input_folder, self.bin_value)))
output_folder = os.path.abspath(
os.path.join(output_folder,
"{}_rebin_by_{}".format(input_folder,
self.bin_value)))
utilities.make_dir(dir=output_folder, overwrite=False)
w = widgets.IntProgress()
w.max = len(self.list_file_names)
display(w)
for _index, _file in enumerate(self.list_file_names):
basename = os.path.basename(_file)
_base, _ext = os.path.splitext(basename)
output_file_name = os.path.join(output_folder, _base + '.tiff')
_rebin_data = self.rebin_data(self.data[_index])
file_handler.make_tiff(filename=output_file_name, data=_rebin_data)
w.value = _index + 1
display(HTML('<span style="font-size: 20px; color:blue">File created in ' + \
output_folder + '</span>'))
def load_data(self, data=None, data_type='sample', notebook=False):
"""Function to save the data already loaded as arrays
Paramters:
data: np array 2D or 3D
data_type: string - 'sample', 'ob' or 'df' (default 'sample')
notebook: boolean - turn on this option if you run the library from a
notebook to have a progress bar displayed showing you the progress of the loading (default False)
"""
if notebook:
from ipywidgets import widgets
from IPython.core.display import display
if len(np.shape(data)) > 2:
if notebook:
_message = "Loading {}".format(data_type)
box1 = widgets.HBox([widgets.Label(_message,
layout=widgets.Layout(width='20%')),
widgets.IntProgress(max=len(data))])
display(box1)
w1 = box1.children[1]
for _index, _data in enumerate(data):
_data = _data.astype(self.working_data_type)
self.__load_individual_data(data=_data, data_type=data_type)
if notebook:
# update progress bar
w1.value = _index + 1
if notebook:
box1.close()
else:
data = data.astype(self.working_data_type)
self.__load_individual_data(data=data, data_type=data_type)
def export(self):
output_folder = self.output_folder_ui.selected
input_folder_basename = os.path.basename(self.image_folder)
output_folder = os.path.join(
output_folder, input_folder_basename + '_with_timestamp_info')
if os.path.exists(output_folder):
import shutil
shutil.rmtree(output_folder)
os.makedirs(output_folder)
text_x = self.preview.widget.result['text_x']
text_y = self.preview.widget.result['text_y']
pre_text = self.preview.widget.result['pre_text']
post_text = self.preview.widget.result['post_text']
color = self.preview.widget.result['color']
def plot_selected_image(index):
_short_file = os.path.basename(self.list_files[index])
output_file_name = os.path.abspath(
os.path.join(output_folder, _short_file + '.png'))
font = {
'family': 'serif',
'color': color,
'weight': 'normal',
'size': 16
}
fig = plt.figure(figsize=(15, 10))
gs = gridspec.GridSpec(1, 1)
ax = plt.subplot(gs[0, 0])
im = ax.imshow(self.images_array[index], interpolation='nearest')
plt.title("image index {}".format(index))
plt.text(text_x,
text_y,
"{}{:.2f}{}".format(pre_text,
self.list_time_offset[index],
post_text),
fontdict=font)
fig.colorbar(im)
plt.savefig(output_file_name)
plt.close(fig)
box = widgets.HBox([
widgets.Label("Exporting Images:",
layout=widgets.Layout(width='20%')),
widgets.IntProgress(min=0,
max=len(self.list_files) - 1,
layout=widgets.Layout(width='50%'))
])
progress_bar = box.children[1]
display(box)
for _index in np.arange(len(self.list_files)):
plot_selected_image(index=_index)
progress_bar.value = _index + 1
def export(self):
output_folder = self.output_folder_ui.selected
input_folder_basename = os.path.basename(self.folder_ui.selected)
output_folder = os.path.join(
output_folder, input_folder_basename + '_vs_metadata_screenshots')
if os.path.exists(output_folder):
import shutil
shutil.rmtree(output_folder)
os.makedirs(output_folder)
def plot_images_and_profile(file_index=0):
fig = plt.figure(figsize=(15, 10))
gs = gridspec.GridSpec(1, 2)
_short_file = os.path.basename(self.images_list[file_index])
_time = self.metadata_profile[_short_file]['time']
_metadata = self.metadata_profile[_short_file]['metadata']
ax_img = plt.subplot(gs[0, 0])
ax_img.imshow(self.images_array[file_index])
plt.title("{}".format(_short_file))
ax_plot = plt.subplot(gs[0, 1])
ax_plot.plot(self.time_array, self.metadata_array, '*')
ax_plot.axvline(x=_time, color='r')
plt.xlabel('Time (s)')
plt.ylabel(self.metadata_name)
output_file_name = os.path.abspath(
os.path.join(output_folder, _short_file + '.png'))
plt.savefig(output_file_name)
plt.close(fig)
box = widgets.HBox([
widgets.Label("Exporting Images:",
layout=widgets.Layout(width='20%')),
widgets.IntProgress(min=0,
max=self.nbr_images - 1,
layout=widgets.Layout(width='50%'))
])
progress_bar = box.children[1]
display(box)
for _index in np.arange(self.nbr_images):
plot_images_and_profile(file_index=_index)
progress_bar.value = _index + 1
box.close()
display(
HTML('<span style="font-size: 20px; color:blue">Images created in '
+ output_folder + '</span>'))
def retrieve_metadata_from_dsc_list_files(list_files=[]):
w = widgets.IntProgress()
w.max = len(list_files)
display(w)
metadata = {}
for _index, _file in enumerate(list_files):
metadata[os.path.basename(_file)] = retrieve_metadata_from_dsc_file(filename=_file)
w.value = _index+1
return metadata
def run_simulation(self):
train_data_list = self.__settings.get_value(
"general", "train_set_path").split(",")
test_data_path = self.__settings.get_value("general",
"test_set_path").split(",")
abbading_format_loader_test = AbbadingoFormatLoader()
abbading_format_loader_test.load(test_data_path[0])
for train_data in train_data_list:
abbading_format_loader = AbbadingoFormatLoader()
abbading_format_loader.load(train_data)
try:
option = self.__settings.get_value("general", "gcs_mode")
if option == "sGCS":
loader_widget = widgets.IntProgress(
value=0,
min=0,
step=1,
bar_style='info',
orientation='horizontal',
layout=widgets.Layout(width='100%', height='100%'),
style={'description_width': 'initial'})
display(loader_widget)
self.__gcs = sGCS(self.__settings)
self.__gcs.train_data = abbading_format_loader.input_data
self.__gcs.test_data = abbading_format_loader_test.input_data
self.__gcs.grammar.stochastic.train_data = abbading_format_loader.input_data
self.__gcs.reset_grammar()
self.run_standard_test()
visualization_enable = self.__settings.get_value(
"general", "visualization_enable")
sim_results, final_rules = self.__gcs.process(
visualization_enable, loader_widget)
sim_results.learning_set_name = train_data
self.__result_list.append(str(sim_results))
TP = self.__gcs.grammar.truePositive
FP = self.__gcs.grammar.falsePositive
FN = self.__gcs.grammar.falseNegative
TN = self.__gcs.grammar.trueNegative
Sens = TP / (TP + FN)
Spec = TN / (TN + FP)
# print("TP = {}". format(TP))
# print("FP = {}". format(FP))
# print("FN = {}". format(FN))
# print("TN = {}". format(TN))
# print("Sensitivity = {}". format(Sens))
# print("Specificity = {}". format(Spec))
# for rule in final_rules:
# print(str(rule) + " usages = " + str(rule.usages_in_proper_parsing))
else:
self.__logger.error("Algorithm mode unknown")
except:
self.__logger.exception("Parameter not found")
return sim_results, final_rules
def progress_bar_iterate(list_):
list_ = list(list_)
f = widgets.IntProgress(min=0,
max=len(list_),
step=1,
description='Loading:',
orientation='horizontal')
display(f)
for item in list_:
f.value += 1
yield item
f.close()
def copy_files(dict_old_new_names={}, new_output_folder='', overwrite=True):
make_dir(dir=new_output_folder, overwrite=overwrite)
nbr_files = len(dict_old_new_names.keys())
w1 = widgets.IntProgress()
w1.max = nbr_files
display(w1)
# create new images
for _index, _old_name in enumerate(dict_old_new_names.keys()):
_new_name = os.path.join(new_output_folder,
dict_old_new_names[_old_name])
copyfile(_old_name, _new_name)
w1.value = _index + 1
def copy_files(button):
import shutil
progress = widgets.IntProgress(min=0, max=len(fns), value=0, description='Copying...')
display(progress)
for fn in fns:
t_fn = target_path.joinpath(Path(fn).name)
if dry_run:
# print(f'shutil.copyfile({fn}, {t_fn})')
pass
else:
shutil.copyfile(fn, t_fn)
progress.value += 1
progress.description = 'Done!'
def merge(self, output_folder):
"""combine images using algorithm provided"""
output_folder = os.path.abspath(output_folder)
merging_list = self.master_list_images_to_combine
algorithm = self.get_merging_algorithm()
merging_ui = widgets.HBox([widgets.Label("Merging Progress",
layout=widgets.Layout(width='20%')),
widgets.IntProgress(max=len(merging_list.keys()),
layout=widgets.Layout(width='80%'))])
display(merging_ui)
progress_bar_ui = merging_ui.children[1]
output_folder = self.make_output_folder(output_folder)
_run_index = 0
for _run in merging_list.keys():
positions_dict = merging_list[_run]
for _position in positions_dict.keys():
list_of_files = positions_dict[_position]['list_of_files']
o_load = Normalization()
o_load.load(file=list_of_files, notebook=True)
_data = o_load.data['sample']['data']
combined_data = SequentialCombineImagesUsingMetadata._merging_algorithm(algorithm, _data)
_new_name = self._define_merged_file_name(output_folder=output_folder,
run_label=_run,
position_label=_position)
output_file_name = os.path.join(output_folder, _new_name)
file_handler.save_data(data=combined_data, filename=output_file_name)
_run_index += 1
progress_bar_ui.value = _run_index
merging_ui.close()
del merging_ui
display(HTML('<span style="font-size: 20px; color:blue">Files have been created in : ' + \
output_folder + '</span>'))
def load_horiba_map(fname, *args, **kwargs):
fext = os.path.splitext(fname)[1]
if fext == '.txt':
df = pd.read_csv(fname, delimiter='\t')
df = df.rename(columns={'Unnamed: 0': 'x', 'Unnamed: 1': 'y'})
shift = np.array(df.columns[2:]).astype(np.float)
E = 1240 / 532 - 1240 * shift * 1e-7
else:
print('error')
return
# working out indexing of an image
xvals = np.sort(df.x.unique())
yvals = np.sort(df.y.unique())
xs = np.array(df.x.astype(np.float))
ys = np.array(df.y.astype(np.float))
#calculating replacement vectors for position value in pandas array
xind = np.searchsorted(xvals, xs)
yind = np.searchsorted(yvals, ys)
#print('indexing check',xind.size,yind.size)
#print('image size check',xvals.size,yvals.size)
#print('SI depth',shift.size)
#wrapping data into si in wavelngth (wl), energy (e)
data = np.zeros([xvals.size, yvals.size, shift.size], dtype='float')
dataes = data
shiftes = np.linspace(shift.min(), shift.max(), shift.size)
Ees = 1240 / 532 - 1240 * shiftes * 1e-7
icorr = 1
w = widgets.IntProgress(value=0,
min=0,
max=xind.size,
description="Progress...")
display(w)
for i in range(0, xind.size):
data[xind[i], yind[i], :] = df.values[i, 2:]
dataes[xind[i], yind[i], :] = sp.interpolate.griddata(
shift, df.as_matrix()[i, 2:] * icorr, shiftes, method='cubic')
#plt.plot(shift,df.values[i,2:])
w.value = i + 1
#print('Complete')
return (xvals, yvals, shiftes, Ees, dataes)
def combine(self, output_folder):
# remove shortcut buttons
self.o_folder.list_output_folders_ui.shortcut_buttons.close()
# create output folder
base_folder_name = os.path.basename(self.input_folder)
new_folder = base_folder_name + "_combined"
full_new_folder_name = os.path.join(os.path.abspath(output_folder),
new_folder)
make_or_reset_folder(full_new_folder_name)
progress_ui = widgets.IntProgress(value=0,
min=0,
max=len(
self.full_combine_dict.keys()))
display(progress_ui)
_index = 0
for _base_name in self.full_combine_dict.keys():
file_name = "{}_{:05d}.tiff".format(_base_name, _index)
full_file_name = os.path.join(full_new_folder_name, file_name)
list_files_to_combine_fullname = [
os.path.join(self.input_folder, _file)
for _file in self.full_combine_dict[_base_name]
]
o_work = Normalization()
o_work.load(file=list_files_to_combine_fullname,
auto_gamma_filter=False)
list_data = o_work.data['sample']['data']
o_combine = CombineImagesAlgorithm(list_array=list_data)
new_array = o_combine.mean_without_outliers(list_array=list_data)
o_work.data['sample']['data'] = [new_array]
o_work.data['sample']['file_name'] = [full_file_name]
o_work.export(folder=full_new_folder_name, data_type='sample')
_index += 1
progress_ui.value = _index
progress_ui.close()
display(HTML('<span style="font-size: 20px; color:blue">' + str(_index) + \
' files have been created in ' + full_new_folder_name + '</span>'))
def deploy_button(project_configuration,
train_py="train_dl.py",
score_py="score.py"):
button = widgets.Button(
description="Deploy Azure Machine Learning Services",
layout=Layout(width="80%", height="80px", justify_content="center"),
)
output = widgets.Output()
slider = widgets.IntProgress(layout=Layout(width="80%"))
def on_button_clicked(b):
c = BackgroundCountThread(12)
slider.value = 0
slider.bar_style = "info"
thread = threading.Thread(target=c.work, args=(slider, ))
thread.start()
try:
button.disabled = True
button.description = "Running"
with output:
button.button_style = "info"
print("Begin Deployment.")
from azure_utils.machine_learning.contexts.realtime_score_context import (
DeepRealtimeScore, )
deep_ws, aks_service = DeepRealtimeScore.get_or_or_create(
configuration_file=project_configuration,
train_py=train_py,
score_py=score_py,
)
button.button_style = "success"
button.description = "Complete, rerun?"
print("Deploy Complete")
display(deep_ws.workspace_widget)
except:
print("Deploy Error")
button.button_style = "danger"
slider.bar_style = "danger"
button.description = "Error"
raise
finally:
c.terminate()
button.disabled = False
button.on_click(on_button_clicked)
run_button = widgets.VBox([button, slider, output])
return run_button
