def widget(): single_progress = widgets.IntProgress(min=0, max=10, description='Calculating EIG: ') overall_progress = widgets.IntProgress(min=0, max=10, description='EIG Progress : ') return widgets.VBox([single_progress, overall_progress])
def merging(self, output_folder): """combine images using algorithm provided""" # get merging algorithm merging_algo = self.combine_method.value algorithm = self.__add if merging_algo == 'mean': algorithm = self.__mean # get output folder output_folder = os.path.abspath(output_folder) # create dictionary of how the images will be combined merging_dict = self.__create_merging_dictionary() self.merginc_dict_debugging = merging_dict # create final list of files to merge final_dict_of_files_to_merge = self.__create_dict_of_files_to_merge(merging_dict) self.final_dict_of_files_to_merge_debugging = final_dict_of_files_to_merge final_nbr_folders = len(merging_dict.keys()) folder_level_ui = widgets.HBox([widgets.Label("Folder Progress:", layout=widgets.Layout(width='20%')), widgets.IntProgress(max=final_nbr_folders, layout=widgets.Layout(width='50%'))]) display(folder_level_ui) w1 = folder_level_ui.children[1] nbr_files_to_merge = self.nbr_files_in_each_folder file_level_ui = widgets.HBox([widgets.Label("File Progress:", layout=widgets.Layout(width='20%')), widgets.IntProgress(max=nbr_files_to_merge, layout=widgets.Layout(width='50%'))]) display(file_level_ui) w2 = file_level_ui.children[1] for _index_final_folder, _final_folder in enumerate(final_dict_of_files_to_merge.keys()): file_handler.make_or_reset_folder(os.path.join(output_folder, _final_folder)) list_files_to_merge = final_dict_of_files_to_merge[_final_folder] for _index_files_to_merge, _files_to_merge in enumerate(list_files_to_merge): _files_to_merge = [_file for _file in _files_to_merge] self.files_to_merge_for_testing = _files_to_merge o_load = Normalization() o_load.load(file=_files_to_merge) _data = o_load.data['sample']['data'] combined_data = self.__merging_algorithm(algorithm, _data) self.combined_data_for_testing = combined_data _base_name_file = os.path.basename(_files_to_merge[0]) output_file_name = os.path.join(output_folder, _final_folder, _base_name_file) file_handler.save_data(data=combined_data, filename=output_file_name) w2.value = _index_files_to_merge + 1 w1.value = _index_final_folder + 1
def load_data(filenames='', folder='', showing_progress=False): ''' load the various file_name format ''' if folder: list_files = glob.glob(folder + '/*.*') stack = [] if showing_progress: w = widgets.IntProgress() w.max = len(list_files) display(w) for _index, _file in enumerate(list_files): _data = load_data(filenames=_file) stack.append(_data) if showing_progress: w.value = _index+1 return stack elif isinstance(filenames, str): data_type = get_data_type(filenames) if data_type == '.fits': hdulist = fits.open(filenames, ignore_missing_end=True) hdu = hdulist[0] _image = np.asarray(hdu.data) hdulist.close() return _image elif (data_type == '.tiff') or (data_type == '.tif'): _image = Image.open(filenames) return np.array(_image) else: return [] else: # list of filenames list_files = filenames stack = [] if showing_progress: w = widgets.IntProgress() w.max = len(list_files) display(w) for _index, _file in enumerate(list_files): _data = load_data(filenames=_file) stack.append(_data) if showing_progress: w.value = _index+1 return stack
def export(self): base_folder = os.path.basename(os.path.dirname( self.list_file_names[0])) + '_normalized' output_folder = os.path.abspath( os.path.join(self.output_folder_ui.selected, base_folder)) utilities.make_dir(dir=output_folder) w = widgets.IntProgress() w.max = len(self.files.sample) display(w) for _index, _file in enumerate(self.list_file_names): basename = os.path.basename(_file) _base, _ext = os.path.splitext(basename) output_file_name = os.path.join(output_folder, _base + '.tiff') file_handler.make_tiff(filename=output_file_name, data=self.normalized_data_array[_index]) w.value = _index + 1 display( HTML( '<span style="font-size: 20px; color:blue">The normalized images have been ' + 'created in ' + output_folder + '</span>'))
def export(self, output_folder): new_folder = 'rotated_{}deg'.format(self.rotation_angle) list_files = self.list_files data = self.data w = widgets.IntProgress() w.max = len(list_files) display(w) output_folder = os.path.abspath(output_folder) full_output_folder = os.path.join(output_folder, new_folder) if not os.path.exists(full_output_folder): os.makedirs(full_output_folder) for _index, _file in enumerate(list_files): _base_file_name = os.path.basename(_file) [_base, _] = os.path.splitext(_base_file_name) _full_file_name = os.path.join(full_output_folder, _base + '.tiff') file_handler.make_tiff(data=data[_index], filename=_full_file_name) w.value = _index + 1 w.close() display(HTML('')) display( HTML( '<span style="font-size: 20px; color:blue">Files created in ' + full_output_folder + '</span>'))
def __init__(self, instrument='CG1D', facility='HFIR', list_files=[], oncat=None, projection=[], with_progressbar=False): projection.append('ingested') if with_progressbar: box1 = widgets.HBox([widgets.Label("Retrieving Metadata ...", layout=widgets.Layout(width='30%')), widgets.IntProgress(max=len(list_files), layout=widgets.Layout(width='70%'))]) display(box1) slider = box1.children[1] self.datafiles = {} for _index, _file in enumerate(list_files): self.datafiles[_file] = oncat.Datafile.retrieve(_file, facility=facility, instrument=instrument, projection=projection) if with_progressbar: slider.value = _index if with_progressbar: box1.close()
def retrieve_data_and_metadata(self): self.list_files = self.list_files_ui.selected w = widgets.IntProgress() w.max = len(self.list_files) display(w) index = 0 files_data = {} files_metadata = {} for index, file in enumerate(self.list_files): _metadata = self.retrieve_metadata(file) files_metadata[file] = _metadata _col_names = _metadata['Label'].split(',') _new_col_names = self.format_col_names(_col_names) _data = self.retrieve_data(file, nbr_comment_line=_metadata['nbr_comment_line'], col_names=_new_col_names) files_data[file] = _data index += 1 w.value = index self.files_data = files_data self.files_metadata = files_metadata w.close()
def run(self): display( HTML('<span style="font-size: 20px; color:blue">' + self.message + '</span>')) sample_image = gui_widgets.gui_dname(dir=self.working_dir) list_sample_images = np.sort( glob.glob(os.path.join(sample_image, '*.fits'))) if len(list_sample_images) > 0: w = widgets.IntProgress() w.max = len(list_sample_images) display(w) _data = [] for _index, _file in enumerate(list_sample_images): _image = np.array(file_handler.load_data(_file), dtype=float) _image = image_processing.single_gamma_filtering(_image) _image[_image == np.inf] = 0 # removing inf values _image[np.isnan(_image)] = 0 _data.append(_image) w.value = _index + 1 [nbr_files, height, width] = np.shape(_data) _data = np.squeeze(_data) self.data = _data self.list_sample_images = list_sample_images
def on_choose_workspace(changed): with output: output.clear_output() workspace_paths = ws_paths[changed['new']] comment_files = get_ipython().getoutput(f'gsutil ls {workspace_paths.get_comment_file_glob()}') if not comment_files[0].startswith('gs://'): display(HTML('''<div class="alert alert-block alert-warning"> No comment files found for HTML snapshots in this workspace.</div>''')) return progress = widgets.IntProgress( value=0, min=0, max=len(comment_files), step=1, description=f'Retrieving {len(comment_files)} comments:', bar_style='success', orientation='horizontal', layout=widgets.Layout(width='450px'), style={'description_width': 'initial'} ) display(progress) comment_num = 0 comment_file_contents = [] for file in comment_files: comment = get_ipython().getoutput(f"gsutil cat '{file}'") version = file.replace(workspace_paths.get_subfolder(), '') comment_file_contents.append({'version': version, 'comment': comment}) comment_num += 1 progress.value = comment_num comments = pd.DataFrame(comment_file_contents) display(comments)
def __init__(self, value=None, description='', options=[], tooltips=[], branch_names=['Previous', 'Next', 'Complete'], branch_steps=[None, None, None], js='', end_js='', name=None, master=None, button_style='info', reviewed=False): self.master = master self.prediction = value self.reviewed = reviewed self.logger = logging.getLogger(__name__) self.progress = widgets.IntProgress( min=1, max=len(self.master.docs), value=1, layout=widgets.Layout(width='90%', height='14px'), style=dict(description_width='initial')) super().__init__(value=value, description=description, options=options, tooltips=tooltips, branch_names=branch_names, branch_steps=branch_steps, js=js, end_js=end_js, name=name, button_style=button_style) pass
def output_profiles(self): # make name of output folder output_folder = os.path.join(os.path.abspath(self.output_folder_ui.selected), os.path.basename(os.path.dirname(self.list_file_name[0]) + '_profiles')) output_folder = os.path.abspath(output_folder) if os.path.isdir(output_folder): shutil.rmtree(output_folder) os.mkdir(output_folder) [roi_left, roi_top, roi_width, roi_height] = self.roi self.list_data_files_short = [os.path.basename(_file) for _file in self.df['#filename']] time_0 = self.__get_time_stamp(self.list_data_files[0]) w = widgets.IntProgress() w.max = len(self.profile_1d) display(w) for _index, _profile in enumerate(self.profile_1d): _file_name = self.list_data_files[_index] _time_stamp = self.__get_time_stamp(_file_name) metadata = [] metadata.append("#Image: {}".format(_file_name)) metadata.append( "#ROI selected (y0, x0, height, width): ({}, {}, {}, {})".format(roi_top, roi_left, roi_height, roi_width)) metadata.append("#Rebin in y direction: {}".format(self.rebin)) _time_stamp_str = datetime.datetime.fromtimestamp(_time_stamp).strftime("%Y-%m-%d %H:%M:%S") metadata.append("#Time Stamp: {}".format(_time_stamp_str)) _delta_time = _time_stamp - time_0 metadata.append("#Delta Time (s): {}".format(_delta_time)) metadata.append("#") metadata.append("#Label: pixel_index, counts") data = [] for _index_data, value in enumerate(self.profile_1d[_index]): data.append("{}, {}".format(self.rebin_range[_index_data], value)) _base_file_name = os.path.basename(_file_name) [base, _] = os.path.splitext(_base_file_name) output_file_name = os.path.join(output_folder, base + '.txt') make_ascii_file(metadata=metadata, data=data, output_file_name=output_file_name, dim='1d') w.value = _index+1 w.close() display(HTML('<span style="font-size: 20px; color:blue">Files created in folder ' + output_folder + '</span>'))
def progress_bar( output_area, label='Searching' ): """Creates an indefinitely repeating progress bar which keeps running until something calls output_area.clear_output() DOES NOT WORK PROPERLY """ stop = 100 i = 0 progress = widgets.IntProgress( value=0, min=0, max=stop, description=label, bar_style='info', orientation='horizontal' ) with output_area: display( progress ) while True: if i > stop: i = 0 sleep( .1 ) i += 1 progress.value = i
def GetFitter(dataDir, inputName, inputDir, load, genBranch_params=None): #---------------------------------------------------------------------------- if (load): w_load_bar = widgets.IntProgress(min=0, max=3, description='Loading:', bar_style='success', orientation='horizontal') display(w_load_bar) ut.defaultParameters(dataDir=dataDir, inputName=inputName, inputDir=inputDir) w_load_bar.value += 1 ut.setParams() w_load_bar.value += 1 if (genBranch_params != None): ut.params["genBranches"] = genBranch_params effFitter = ut.loadOrMake() w_load_bar.value += 1 time.sleep(1) print('Done loading') #w_load_bar.close() return effFitter
def converting_data(self): try: a = np.float(self.box.children[1].value) b = np.float(self.box.children[3].value) symbol = self.box.children[2].value coeff = 1 if symbol == '+': coeff = -1 progress_bar = widgets.IntProgress() progress_bar.max = len(self.list_data) display(progress_bar) list_concentration = [] for _index, _data in enumerate(self.list_data): _list_concentration = coeff * (a - _data) / b list_concentration.append(_list_concentration) progress_bar.value = _index + 1 self.list_concentration = list_concentration progress_bar.close() except: display( HTML( '<span style="font-size: 20px; color:red">Make sure the coefficient are floats!</span>' ))
def __load_files(self): progress_bar_layout = widgets.Layout(border='1px solid blue') hbox = widgets.HBox([ widgets.IntProgress(description="FUll Progress", layout=progress_bar_layout), widgets.Label(value='', layout=widgets.Layout(width='10%')) ]) w = hbox.children[0] nbr_groups = len(self.exp_dict.keys()) w.max = nbr_groups label = hbox.children[1] label.value = f"0/{nbr_groups}" display(hbox) for _index, _key in enumerate(self.exp_dict.keys()): _item = self.exp_dict[_key] _path = _item['folder'] list_files = _item['list_of_files'] full_list_files = [ os.path.join(_path, _file) for _file in list_files ] o_norm = Normalization() o_norm.load(file=full_list_files, notebook=True) _data = o_norm.data['sample']['data'] _item['list_of_images'] = _data self.exp_dict[_key] = _item w.value = _index + 1 label.value = f"{_index+1}/{nbr_groups}" hbox.close() display(widgets.Label(value="Done!"))
def retrieve_time_stamp(list_images): [_, ext] = os.path.splitext(list_images[0]) if ext.lower() in ['.tiff', '.tif']: ext = 'tif' elif ext.lower() == '.fits': ext = 'fits' else: raise ValueError box = widgets.HBox([widgets.Label("Retrieving Time Stamp", layout=widgets.Layout(width='20%')), widgets.IntProgress(min=0, max=len(list_images), value=0, layout=widgets.Layout(width='50%')) ]) progress_bar = box.children[1] display(box) list_time_stamp = [] list_time_stamp_user_format = [] for _index, _file in enumerate(list_images): _time_stamp = MetadataHandler.get_time_stamp(file_name=_file, ext=ext) _time_stamp = _convert_epics_timestamp_to_rfc3339_timestamp(_time_stamp) list_time_stamp.append(_time_stamp) _user_format = convert_to_human_readable_format(_time_stamp) list_time_stamp_user_format.append(_user_format) progress_bar.value = _index + 1 box.close() return {'list_images': list_images, 'list_time_stamp': list_time_stamp, 'list_time_stamp_user_format': list_time_stamp_user_format}
def saveSentences(self, dao, df, table_name, split_func, overwrite=False): import pandas as pd sents_df = pd.DataFrame(columns=[ 'DATASET_ID', 'BUNCH_ID', 'DOC_NAME', 'TEXT', 'DATE', 'REF_DATE' ]) progressbar = widgets.IntProgress(min=0, max=len(df), value=0, layout=widgets.Layout(width='80%'), description='Splitting:') progressbar.value = 0 display(progressbar) for idx, row in df.iterrows(): bunch_id = row['BUNCH_ID'] doc_name = row['DOC_NAME'] text = row['TEXT'] date = row['DATE'] ref_date = row['REF_DATE'] sentence_id = 0 progressbar.value += 1 for sentence in split_func(text): sentence_id += 1 sents_df.loc[len(sents_df)] = [ self.dataset_name + "_sents", bunch_id, doc_name + "_" + str(sentence_id), sentence, date, ref_date ] saveDFtoDB(dao, sents_df, table_name) self.saveToWhoosh(sents_df, self.dataset_name + "_sents", overwrite) pass
def export(self, output_folder): input_folder = self.get_input_folder() # output_folder = os.path.abspath(os.path.join(self.output_folder_ui.selected, # "{}_rebin_by_{}".format(input_folder, self.bin_value))) output_folder = os.path.abspath( os.path.join(output_folder, "{}_rebin_by_{}".format(input_folder, self.bin_value))) utilities.make_dir(dir=output_folder, overwrite=False) w = widgets.IntProgress() w.max = len(self.list_file_names) display(w) for _index, _file in enumerate(self.list_file_names): basename = os.path.basename(_file) _base, _ext = os.path.splitext(basename) output_file_name = os.path.join(output_folder, _base + '.tiff') _rebin_data = self.rebin_data(self.data[_index]) file_handler.make_tiff(filename=output_file_name, data=_rebin_data) w.value = _index + 1 display(HTML('<span style="font-size: 20px; color:blue">File created in ' + \ output_folder + '</span>'))
def load_data(self, data=None, data_type='sample', notebook=False): """Function to save the data already loaded as arrays Paramters: data: np array 2D or 3D data_type: string - 'sample', 'ob' or 'df' (default 'sample') notebook: boolean - turn on this option if you run the library from a notebook to have a progress bar displayed showing you the progress of the loading (default False) """ if notebook: from ipywidgets import widgets from IPython.core.display import display if len(np.shape(data)) > 2: if notebook: _message = "Loading {}".format(data_type) box1 = widgets.HBox([widgets.Label(_message, layout=widgets.Layout(width='20%')), widgets.IntProgress(max=len(data))]) display(box1) w1 = box1.children[1] for _index, _data in enumerate(data): _data = _data.astype(self.working_data_type) self.__load_individual_data(data=_data, data_type=data_type) if notebook: # update progress bar w1.value = _index + 1 if notebook: box1.close() else: data = data.astype(self.working_data_type) self.__load_individual_data(data=data, data_type=data_type)
def export(self): output_folder = self.output_folder_ui.selected input_folder_basename = os.path.basename(self.image_folder) output_folder = os.path.join( output_folder, input_folder_basename + '_with_timestamp_info') if os.path.exists(output_folder): import shutil shutil.rmtree(output_folder) os.makedirs(output_folder) text_x = self.preview.widget.result['text_x'] text_y = self.preview.widget.result['text_y'] pre_text = self.preview.widget.result['pre_text'] post_text = self.preview.widget.result['post_text'] color = self.preview.widget.result['color'] def plot_selected_image(index): _short_file = os.path.basename(self.list_files[index]) output_file_name = os.path.abspath( os.path.join(output_folder, _short_file + '.png')) font = { 'family': 'serif', 'color': color, 'weight': 'normal', 'size': 16 } fig = plt.figure(figsize=(15, 10)) gs = gridspec.GridSpec(1, 1) ax = plt.subplot(gs[0, 0]) im = ax.imshow(self.images_array[index], interpolation='nearest') plt.title("image index {}".format(index)) plt.text(text_x, text_y, "{}{:.2f}{}".format(pre_text, self.list_time_offset[index], post_text), fontdict=font) fig.colorbar(im) plt.savefig(output_file_name) plt.close(fig) box = widgets.HBox([ widgets.Label("Exporting Images:", layout=widgets.Layout(width='20%')), widgets.IntProgress(min=0, max=len(self.list_files) - 1, layout=widgets.Layout(width='50%')) ]) progress_bar = box.children[1] display(box) for _index in np.arange(len(self.list_files)): plot_selected_image(index=_index) progress_bar.value = _index + 1
def export(self): output_folder = self.output_folder_ui.selected input_folder_basename = os.path.basename(self.folder_ui.selected) output_folder = os.path.join( output_folder, input_folder_basename + '_vs_metadata_screenshots') if os.path.exists(output_folder): import shutil shutil.rmtree(output_folder) os.makedirs(output_folder) def plot_images_and_profile(file_index=0): fig = plt.figure(figsize=(15, 10)) gs = gridspec.GridSpec(1, 2) _short_file = os.path.basename(self.images_list[file_index]) _time = self.metadata_profile[_short_file]['time'] _metadata = self.metadata_profile[_short_file]['metadata'] ax_img = plt.subplot(gs[0, 0]) ax_img.imshow(self.images_array[file_index]) plt.title("{}".format(_short_file)) ax_plot = plt.subplot(gs[0, 1]) ax_plot.plot(self.time_array, self.metadata_array, '*') ax_plot.axvline(x=_time, color='r') plt.xlabel('Time (s)') plt.ylabel(self.metadata_name) output_file_name = os.path.abspath( os.path.join(output_folder, _short_file + '.png')) plt.savefig(output_file_name) plt.close(fig) box = widgets.HBox([ widgets.Label("Exporting Images:", layout=widgets.Layout(width='20%')), widgets.IntProgress(min=0, max=self.nbr_images - 1, layout=widgets.Layout(width='50%')) ]) progress_bar = box.children[1] display(box) for _index in np.arange(self.nbr_images): plot_images_and_profile(file_index=_index) progress_bar.value = _index + 1 box.close() display( HTML('<span style="font-size: 20px; color:blue">Images created in ' + output_folder + '</span>'))
def retrieve_metadata_from_dsc_list_files(list_files=[]): w = widgets.IntProgress() w.max = len(list_files) display(w) metadata = {} for _index, _file in enumerate(list_files): metadata[os.path.basename(_file)] = retrieve_metadata_from_dsc_file(filename=_file) w.value = _index+1 return metadata
def run_simulation(self): train_data_list = self.__settings.get_value( "general", "train_set_path").split(",") test_data_path = self.__settings.get_value("general", "test_set_path").split(",") abbading_format_loader_test = AbbadingoFormatLoader() abbading_format_loader_test.load(test_data_path[0]) for train_data in train_data_list: abbading_format_loader = AbbadingoFormatLoader() abbading_format_loader.load(train_data) try: option = self.__settings.get_value("general", "gcs_mode") if option == "sGCS": loader_widget = widgets.IntProgress( value=0, min=0, step=1, bar_style='info', orientation='horizontal', layout=widgets.Layout(width='100%', height='100%'), style={'description_width': 'initial'}) display(loader_widget) self.__gcs = sGCS(self.__settings) self.__gcs.train_data = abbading_format_loader.input_data self.__gcs.test_data = abbading_format_loader_test.input_data self.__gcs.grammar.stochastic.train_data = abbading_format_loader.input_data self.__gcs.reset_grammar() self.run_standard_test() visualization_enable = self.__settings.get_value( "general", "visualization_enable") sim_results, final_rules = self.__gcs.process( visualization_enable, loader_widget) sim_results.learning_set_name = train_data self.__result_list.append(str(sim_results)) TP = self.__gcs.grammar.truePositive FP = self.__gcs.grammar.falsePositive FN = self.__gcs.grammar.falseNegative TN = self.__gcs.grammar.trueNegative Sens = TP / (TP + FN) Spec = TN / (TN + FP) # print("TP = {}". format(TP)) # print("FP = {}". format(FP)) # print("FN = {}". format(FN)) # print("TN = {}". format(TN)) # print("Sensitivity = {}". format(Sens)) # print("Specificity = {}". format(Spec)) # for rule in final_rules: # print(str(rule) + " usages = " + str(rule.usages_in_proper_parsing)) else: self.__logger.error("Algorithm mode unknown") except: self.__logger.exception("Parameter not found") return sim_results, final_rules
def progress_bar_iterate(list_): list_ = list(list_) f = widgets.IntProgress(min=0, max=len(list_), step=1, description='Loading:', orientation='horizontal') display(f) for item in list_: f.value += 1 yield item f.close()
def copy_files(dict_old_new_names={}, new_output_folder='', overwrite=True): make_dir(dir=new_output_folder, overwrite=overwrite) nbr_files = len(dict_old_new_names.keys()) w1 = widgets.IntProgress() w1.max = nbr_files display(w1) # create new images for _index, _old_name in enumerate(dict_old_new_names.keys()): _new_name = os.path.join(new_output_folder, dict_old_new_names[_old_name]) copyfile(_old_name, _new_name) w1.value = _index + 1
def copy_files(button): import shutil progress = widgets.IntProgress(min=0, max=len(fns), value=0, description='Copying...') display(progress) for fn in fns: t_fn = target_path.joinpath(Path(fn).name) if dry_run: # print(f'shutil.copyfile({fn}, {t_fn})') pass else: shutil.copyfile(fn, t_fn) progress.value += 1 progress.description = 'Done!'
def merge(self, output_folder): """combine images using algorithm provided""" output_folder = os.path.abspath(output_folder) merging_list = self.master_list_images_to_combine algorithm = self.get_merging_algorithm() merging_ui = widgets.HBox([widgets.Label("Merging Progress", layout=widgets.Layout(width='20%')), widgets.IntProgress(max=len(merging_list.keys()), layout=widgets.Layout(width='80%'))]) display(merging_ui) progress_bar_ui = merging_ui.children[1] output_folder = self.make_output_folder(output_folder) _run_index = 0 for _run in merging_list.keys(): positions_dict = merging_list[_run] for _position in positions_dict.keys(): list_of_files = positions_dict[_position]['list_of_files'] o_load = Normalization() o_load.load(file=list_of_files, notebook=True) _data = o_load.data['sample']['data'] combined_data = SequentialCombineImagesUsingMetadata._merging_algorithm(algorithm, _data) _new_name = self._define_merged_file_name(output_folder=output_folder, run_label=_run, position_label=_position) output_file_name = os.path.join(output_folder, _new_name) file_handler.save_data(data=combined_data, filename=output_file_name) _run_index += 1 progress_bar_ui.value = _run_index merging_ui.close() del merging_ui display(HTML('<span style="font-size: 20px; color:blue">Files have been created in : ' + \ output_folder + '</span>'))
def load_horiba_map(fname, *args, **kwargs): fext = os.path.splitext(fname)[1] if fext == '.txt': df = pd.read_csv(fname, delimiter='\t') df = df.rename(columns={'Unnamed: 0': 'x', 'Unnamed: 1': 'y'}) shift = np.array(df.columns[2:]).astype(np.float) E = 1240 / 532 - 1240 * shift * 1e-7 else: print('error') return # working out indexing of an image xvals = np.sort(df.x.unique()) yvals = np.sort(df.y.unique()) xs = np.array(df.x.astype(np.float)) ys = np.array(df.y.astype(np.float)) #calculating replacement vectors for position value in pandas array xind = np.searchsorted(xvals, xs) yind = np.searchsorted(yvals, ys) #print('indexing check',xind.size,yind.size) #print('image size check',xvals.size,yvals.size) #print('SI depth',shift.size) #wrapping data into si in wavelngth (wl), energy (e) data = np.zeros([xvals.size, yvals.size, shift.size], dtype='float') dataes = data shiftes = np.linspace(shift.min(), shift.max(), shift.size) Ees = 1240 / 532 - 1240 * shiftes * 1e-7 icorr = 1 w = widgets.IntProgress(value=0, min=0, max=xind.size, description="Progress...") display(w) for i in range(0, xind.size): data[xind[i], yind[i], :] = df.values[i, 2:] dataes[xind[i], yind[i], :] = sp.interpolate.griddata( shift, df.as_matrix()[i, 2:] * icorr, shiftes, method='cubic') #plt.plot(shift,df.values[i,2:]) w.value = i + 1 #print('Complete') return (xvals, yvals, shiftes, Ees, dataes)
def combine(self, output_folder): # remove shortcut buttons self.o_folder.list_output_folders_ui.shortcut_buttons.close() # create output folder base_folder_name = os.path.basename(self.input_folder) new_folder = base_folder_name + "_combined" full_new_folder_name = os.path.join(os.path.abspath(output_folder), new_folder) make_or_reset_folder(full_new_folder_name) progress_ui = widgets.IntProgress(value=0, min=0, max=len( self.full_combine_dict.keys())) display(progress_ui) _index = 0 for _base_name in self.full_combine_dict.keys(): file_name = "{}_{:05d}.tiff".format(_base_name, _index) full_file_name = os.path.join(full_new_folder_name, file_name) list_files_to_combine_fullname = [ os.path.join(self.input_folder, _file) for _file in self.full_combine_dict[_base_name] ] o_work = Normalization() o_work.load(file=list_files_to_combine_fullname, auto_gamma_filter=False) list_data = o_work.data['sample']['data'] o_combine = CombineImagesAlgorithm(list_array=list_data) new_array = o_combine.mean_without_outliers(list_array=list_data) o_work.data['sample']['data'] = [new_array] o_work.data['sample']['file_name'] = [full_file_name] o_work.export(folder=full_new_folder_name, data_type='sample') _index += 1 progress_ui.value = _index progress_ui.close() display(HTML('<span style="font-size: 20px; color:blue">' + str(_index) + \ ' files have been created in ' + full_new_folder_name + '</span>'))
def deploy_button(project_configuration, train_py="train_dl.py", score_py="score.py"): button = widgets.Button( description="Deploy Azure Machine Learning Services", layout=Layout(width="80%", height="80px", justify_content="center"), ) output = widgets.Output() slider = widgets.IntProgress(layout=Layout(width="80%")) def on_button_clicked(b): c = BackgroundCountThread(12) slider.value = 0 slider.bar_style = "info" thread = threading.Thread(target=c.work, args=(slider, )) thread.start() try: button.disabled = True button.description = "Running" with output: button.button_style = "info" print("Begin Deployment.") from azure_utils.machine_learning.contexts.realtime_score_context import ( DeepRealtimeScore, ) deep_ws, aks_service = DeepRealtimeScore.get_or_or_create( configuration_file=project_configuration, train_py=train_py, score_py=score_py, ) button.button_style = "success" button.description = "Complete, rerun?" print("Deploy Complete") display(deep_ws.workspace_widget) except: print("Deploy Error") button.button_style = "danger" slider.bar_style = "danger" button.description = "Error" raise finally: c.terminate() button.disabled = False button.on_click(on_button_clicked) run_button = widgets.VBox([button, slider, output]) return run_button