def main(args): leds = LED_SYSTEMS.create(name=args.led_system, wavevec=K, **args.led_attrs) recoverer = METHODS.create( name=args.recovery_method, leds=leds, loops=args.recover_loops, wavelen=WAVELEN, sample_size=SAMPSIZE, quality=QUALITY, objective=Objective, NA=NA, ) low_data = load_low_resolution_images(leds, args.sources_dir) \ if not args.source_name else np_load(args.source_name) low_size = low_data[0].shape img_size = tuple(int(i / QUALITY) for i in low_data[0].shape) print("Recovery process started ...") data = recoverer.run(low_data) print(DURATION_FORMAT.format(recoverer.duration)) img = pack_image(data['amplitude'], img_size, 'I', norm=True) img.save(args.destination_real) print("Result image was saved into {} file.".format(args.destination_real)) if args.show_images: print("Showing recovered object and phase") img.show() if 'pupil' in data: pack_image(data['pupil'], low_size, norm=True).show() if args.real_object_filename: real_inten = load_image(args.real_object_filename, 'I') # result_inten = get_intensity(data['amplitude']) result_inten = array(img) print("Object RMSE: {:.3f}".format( FP.count_RMSE(real_inten, result_inten), )) if args.show_images: diff_img = pack_image(abs(real_inten - result_inten), real_inten.shape, norm=True) diff_img.show() if args.real_pupil_filename and 'pupil' in data: real_inten = np_load(args.real_pupil_filename) print("Pupil RMSE: {:.3f}".format( FP.count_RMSE(real_inten, data['pupil']), )) if args.show_images: diff_img = pack_image(abs(real_inten - data['pupil']), real_inten.shape, norm=True) diff_img.show() elif args.real_pupil_filename and 'pupil' not in data: print("Use another method to recover pupil.") if args.show_fourier: pack_image(data['ft'], img_size, norm=True).show()
def whatyouwant(paramin): global dir_pri1, dir_pri2, dir_pri3 normW2 = arange(1.4, 2.601, 0.05).tolist() dens_moy = arange(0.02, 0.981, 0.03) for d1 in range(len(dens_moy)): dir_sub1 = dir_pri1 + '/Norm_%.2f_DensMoy_%.2f' % (paramin, dens_moy[d1]) nbpatterns1 = len(os.listdir(dir_sub1)) for p1 in range(nbpatterns1): d_patty1 = dir_sub1 + '/pattern_%.3i.npy' % p1 patty1 = np_load(d_patty1) for w2 in range(len(normW2)): dir_sub2 = dir_pri2 + '/Norm_%.2f' % (normW2[w2]) nbpatterns2 = len(os.listdir(dir_sub2)) / 2 for p2 in range(nbpatterns2): d_patty2 = dir_sub2 + '/pattern_%.3i.npy' % p2 patty2 = np_load(d_patty2) d_patty0 = dir_sub2 + '/states_%.3i.jpeg' % p2 if (corrcoef(patty1, patty2)[0,1] > 0.99) \ and (abs(norm(patty1) - norm(patty2)) / norm(patty2) < 0.0005): direction = dir_pri3 + '/C_W%.2fD%.2fP%.3i_D_W%.2fP%.3i' % ( paramin, dens_moy[d1], p1, normW2[w2], p2) cmd = commands.getoutput('cp ' + d_patty2 + " " + direction + '.npy') cmd = commands.getoutput('cp ' + d_patty0 + " " + direction + '.jpeg') print direction, abs(norm(patty1) - norm(patty2)) / norm(patty2) return 1
def __init__(self, db_path, ext): self.db_path = db_path self.ext = ext if self.ext == ".npy": self.loader = lambda x: np_load(x) else: self.loader = lambda x: np_load(x)["feat"] if db_path.endswith(".lmdb"): self.db_type = "lmdb" self.env = lmdb_open( db_path, subdir=isdir(db_path), readonly=True, lock=False, readahead=False, meminit=False, ) elif db_path.endswith(".pth"): # Assume a key,value dictionary self.db_type = "pth" self.feat_file = torch_load(db_path) self.loader = lambda x: x print("HybridLoader: ext is ignored") elif db_path.endswith("h5"): self.db_type = "h5" self.loader = lambda x: np_array(x).astype("float32") else: self.db_type = "dir"
def whatyouwant(paramin): global dir_pri1, dir_pri2, dir_pri3 dens_moy = arange(0.02, 0.981, 0.03) for d1 in range(len(dens_moy)): dir_sub1 = dir_pri1 + '/Norm_%.2f_DensMoy_%.2f' % (paramin, dens_moy[d1]) nbpatterns1 = len(os.listdir(dir_sub1)) for p1 in range(nbpatterns1): d_patty1 = dir_sub1 + '/pattern_%.3i.npy' % p1 patty1 = np_load(d_patty1) dir_sub2 = dir_pri2 + '/Norm_0.50_DensMoy_0.02' nbpatterns2 = len(os.listdir(dir_sub2)) / 2 for p2 in range(nbpatterns2): d_patty2 = dir_sub2 + '/pattern_%.3i.npy' % p2 patty2 = np_load(d_patty2) d_patty0 = dir_sub2 + '/states_%.3i.jpeg' % p2 if (corrcoef(patty1, patty2)[0,1] > 0.7) \ and (abs(norm(patty1) - norm(patty2)) / norm(patty2) < 10.05): direction = dir_pri3 + '/C_W%.2fD%.2fP%.3i_Pica_%.3i' % ( paramin, dens_moy[d1], p1, p2) cmd = commands.getoutput('cp ' + d_patty2 + " " + direction + '.npy') toimage( array(zip(*reversed(patty2.reshape(1, len( patty2))))).T).save(direction + '.jpeg') print direction, abs(norm(patty1) - norm(patty2)) / norm(patty2) return 1
def load(self): path = join(self.directory, self.name + ".npy") if exists(path): arr = np_load(file=path) self.num_steps = arr[0] self.action_num = arr[1] path = join(self.directory, self.name + "_eps.npy") if exists(path): self.eps = np_load(file=path)
def getFakeData(dataSet, fakeClasses, instances): ''' 64x64 images in imageArray ''' fakeClasses = sorted(fakeClasses) imageArray = np.empty((1, 4096)) labelArray = np.empty((1)) for i in fakeClasses: tempImageArray = np_load('../../DCGAN/results/' + 'compressed' + '/' + dataSet + '/' + dataSet + '_' + str(i) + '_' + str(instances) + '.npy') tempImageArray = tempImageArray.reshape(tempImageArray.shape[0], -1) tempLabelArray = np.empty(tempImageArray.shape[0]) tempLabelArray.fill(i) imageArray = np.concatenate([imageArray, tempImageArray]) labelArray = np.concatenate([labelArray, tempLabelArray]) imageArray = imageArray[1:] labelArray = labelArray[1:] # random shuffling of images and labels p = np.random.permutation(imageArray.shape[0]) imageArray = imageArray[p] labelArray = labelArray[p] labelArray = labelArray.astype('int') imageArray = imageArray / 255.0 return imageArray, labelArray
def ReadData(data_dir): labelfolders = [f for f in os.scandir(data_dir) if f.is_dir()] counter = -1 dataset = [] img_counter = 0 for labelfolder in labelfolders: counter += 1 label = (counter, labelfolder.name) files = [ f.path for f in os.scandir(labelfolder.path) if not f.is_dir() and f.name.endswith('depth.npy') ] depth_frames = [] for f in files: depth_frame = np_load(f).astype(np.float) depth_frames.append(depth_frame) img_counter += 1 dataset.append((label, depth_frames)) t_min, t_max = DetermineMinMaxDistance(dataset) print('Range in data: min=' + str(t_min) + "mm max=" + str(t_max) + "mm") return (img_counter, dataset)
def whatyouwant(paramin): global dir_pri1, dir_pri2, dir_pri3, patient dens_moy = arange(0.02,0.981,0.03) ofi1 = open(dir_pri2 + '/TC%ia.dat' %patient,'r') ofi2 = open(dir_pri2 + '/TC%ib.dat' %patient,'r') A = array([line.split() for line in ofi1]).astype(float) B = array([line.split() for line in ofi2]).astype(float) C = concatenate([A,B], axis=1) ofi1.close() ofi2.close() nruter_a = [] for d1 in range(len(dens_moy)): dir_sub1 = dir_pri1 + '/Norm_%.2f_DensMoy_%.2f' %(paramin, dens_moy[d1]) nbpatterns = len(os.listdir(dir_sub1)) - 1 nbtimeshap = size(C, axis=1) nruter_b = zeros((nbpatterns, nbtimeshap)) for p1 in range(nbpatterns): d_patty = dir_sub1 + '/pattern_%.3i.npy' %p1 patty = np_load(d_patty) for p2 in range(nbtimeshap): nruter_b[p1,p2] = corrcoef(patty, C[:,p2])[0,1] nruter_a.append(nruter_b) nruter = nruter_a[0] for ll in range(1, len(nruter_a)): nruter = append(nruter, nruter_a[ll], axis=0) return nruter return 1
def __init__(self, data_dir: Path, split: str, input_field_name: str, output_field_name: str, load_into_memory: bool) \ -> None: """Initialization of a Clotho dataset object. :param data_dir: Data directory with Clotho dataset files. :type data_dir: pathlib.Path :param split: The split to use (`development`, `validation`) :type split: str :param input_field_name: Field name for the input values :type input_field_name: str :param output_field_name: Field name for the output (target) values. :type output_field_name: str :param load_into_memory: Load the dataset into memory? :type load_into_memory: bool """ super(ClothoDataset, self).__init__() the_dir: Path = data_dir.joinpath(split) self.examples: List[Path] = sorted(the_dir.iterdir()) self.input_name: str = input_field_name self.output_name: str = output_field_name self.load_into_memory: bool = load_into_memory if load_into_memory: self.examples: List[ndarray] = [ np_load(str(f), allow_pickle=True) for f in self.examples ]
def __init__(self, data_dir: Path, split: AnyStr, input_field_name: AnyStr, output_field_name: AnyStr, load_into_memory: bool): """Initialization of a Clotho dataset object. :param data_dir: Directory with data. :type data_dir: pathlib.Path :param split: Split to use (i.e. 'development', 'evaluation') :type split: str :param input_field_name: Field name of the clotho data\ to be used as input data to the\ method. :type input_field_name: str :param output_field_name: Field name of the clotho data\ to be used as output data to the\ method. :type output_field_name: str :param load_into_memory: Load all data into memory? :type load_into_memory: bool """ super(ClothoDataset, self).__init__() the_dir: Path = data_dir.joinpath(split) self.examples: List[Path] = sorted(the_dir.iterdir()) self.input_name: str = input_field_name self.output_name: str = output_field_name self.load_into_memory: bool = load_into_memory if load_into_memory: self.examples: List[recarray] = [ np_load(str(f), allow_pickle=True) for f in self.examples ]
def read_sparse_csc_matrix(filepath): """ Read the data, indices, indptr, and shape arrays from a ``.npz`` file on disk at ``filepath``, and return an instantiated ``scipy.sparse.csc_matrix``. """ npz_file = np_load(filepath) return csc_matrix((npz_file['data'], npz_file['indices'], npz_file['indptr']), shape=npz_file['shape'])
def predict_image(path_of_image, groupStage): path_of_model = os_path.join("./CUSTOMIZE_4_USER/MODEL_TRAINING", groupStage, groupStage + ".pth") path_of_feature = os_path.join("./CUSTOMIZE_4_USER/MODEL_TRAINING", groupStage, groupStage + ".npz") start_time = time() model = NeuralNet(input_size, hidden_size, num_classes).to(device) model.load_state_dict(load(path_of_model)) data = np_load(path_of_feature) [h_max, s_max, v_max] = data['data_max'] [h_min, s_min, v_min] = data['data_min'] img = imread(path_of_image) img = resize(img, (6000, 4000)) img = img[500:-500, 750:-750, :] img = cvtColor(img, COLOR_BGR2HSV) hchan, schan, vchan = split(img) h_hist = calcHist([img], [0], None, [256], [0, 256]).reshape(256, ) s_hist = calcHist([img], [1], None, [256], [0, 256]).reshape(256, ) v_hist = calcHist([img], [2], None, [256], [0, 256]).reshape(256, ) hMean = np_mean(hchan) / 255 DPV_h_max = np_sum(np_absolute(h_hist - h_max)) / (HEIGHT * WIDTH) DPV_h_min = np_sum(np_absolute(h_hist - h_min)) / (HEIGHT * WIDTH) sMean = np_mean(schan) / 255 DPV_s_max = np_sum(np_absolute(s_hist - s_max)) / (HEIGHT * WIDTH) DPV_s_min = np_sum(np_absolute(s_hist - s_min)) / (HEIGHT * WIDTH) vMean = np_mean(vchan) / 255 DPV_v_max = np_sum(np_absolute(v_hist - v_max)) / (HEIGHT * WIDTH) DPV_v_min = np_sum(np_absolute(v_hist - v_min)) / (HEIGHT * WIDTH) correlation = np_corrcoef(h_hist, s_hist)[0][1] #image_feature = np_array((hMean, DPV_h_max, DPV_h_min, sMean, DPV_s_max, DPV_s_min, vMean, DPV_v_max, DPV_v_min)) image_feature = np_array((hMean, DPV_h_max, DPV_h_min, sMean, DPV_s_max, DPV_s_min, correlation)) image_feature = from_numpy(image_feature).to(device).float().view( 1, input_size) with no_grad(): out_predict = model(image_feature) _, predicted_result = torch_max(out_predict.data, 1) original = Tensor([[1, 33, 66, 99]]) # Round xx.xx % percentage_result = np_round( mm(out_predict.view(1, num_classes), original.view(num_classes, 1)).item(), 2) # Processed time processedTime = np_round(time() - start_time, 2) #print("Time ",processedTime) return percentage_result, processedTime
def __getitem__(self, item: int) \ -> Tuple[ndarray, ndarray]: ex: Union[Path, recarray] = self.examples[item] ex: recarray = np_load(str(ex), allow_pickle=True) features = ex['features'].item() file_name = ex['file_name'].item() tag = self.tags[file_name] return features, tag
def load(self): from numpy import load as np_load from os.path import exists, join path = join(self.__directory) if exists(path): path = join(self.__directory, f"memory_meta.npy") arr = np_load(file=path) self.__pos = arr[0] self.__capacity = arr[1] self.__episodes = [None] * self.__capacity for i in range(len(self.__episodes)): path = join(self.__directory, f"memory_{i:05d}.npy") if exists(path): self.__episodes[i] = np_load(file=path) else: break self.loaded = True
def __getitem__(self,item: int): ex: Union[Path, recarray] = self.examples[item] if not self.load_into_memory: ex: recarray = np_load(str(ex), allow_pickle=True) in_e, ou_e = [ex[i].item() for i in [self.input_name, self.output_name]] out_len = len(ou_e) all_ref = get_all_ref(ex['file_name'].item(), self.data_dir) filename = str(ex['file_name'].item()) return in_e, ou_e, out_len
def showImageMatrix(dataSet, primaryClass, helperClass, primaryInstances, helperInstances, showImage): ''' Inputs : dataSets : List : Datasets for which samples are to be genrated instances : List : Number of instances to be used from original dataset classes : List : Classes for which samples are to be generated Outputs : 5x5 image matrix ''' fileName = resultDir+'results/MMD'+'/'+'compressed'+'/'+dataSet+'/'+ dataSet + '_' \ + str(primaryClass) + '_' + str(helperClass) + '_' + str(primaryInstances) + '_' \ + str(helperInstances) + '.npy' images = np_load(fileName) # get random list of images to be displayed randomList = np.random.randint(0,1000,(25)) imageList = images[randomList] fmt = 'png' # need to generalise this snippet fig, axes = plt.subplots(5,5) fig.tight_layout() fig.subplots_adjust(wspace=-0.7, hspace=-0.1) plt.axis('off') numOutputChannels = getChannels(dataSet) if numOutputChannels==3: imageList = np.transpose(imageList,(0,2,3,1)) for i in range(5): for j in range(5): f = StringIO() image = PIL.Image.fromarray((imageList[i*5+j]).astype('uint8')) image.save(f,fmt) axes[i,j].imshow((imageList[i*5+j]).astype('uint8'),cmap='gray') axes[i,j].axis('off') axes[i,j].set_xticklabels([]) axes[i,j].set_yticklabels([]) axes[i,j].set_aspect("equal") plotFileName = resultDir+'results'+'/'+'MMD/samples'+'/'+dataSet+'/'+dataSet+ '_' + str(primaryClass) + '_' + str(helperClass) + '_' + str(primaryInstances) + '_' \ + str(helperInstances) plt.savefig(plotFileName, bbox_inches='tight') if showImage==1: plt.show()
def getFakeData(dataSet, fakeClasses, instances, mmdFlag = 0, numHelperInstances=1000): ''' Output image pixels between (-1,1) ''' dataFolder = resultDir+'results/nonMMD/compressed/'+dataSet+'/'+dataSet if mmdFlag==1: dataFolder = resultDir+'results/MMDall/compressed/'+dataSet+'/'+dataSet fakeClasses = sorted(fakeClasses) imageArray, labelArray = getInitialArray(dataSet) for i in fakeClasses: if mmdFlag==1: tempImageArray = np_load(dataFolder+'_'+str(i)+'_'+str(getHelperClass(dataSet,i))+'_'+str(instances)+'_'+str(numHelperInstances)+'.npy') elif mmdFlag==0: tempImageArray = np_load(dataFolder+'_'+str(i)+'_'+str(instances)+'.npy') tempImageArray = tempImageArray.reshape(tempImageArray.shape[0],-1) tempLabelArray = np.empty(tempImageArray.shape[0]) tempLabelArray.fill(i) #print tempImageArray.shape, imageArray.shape imageArray = np.concatenate([imageArray, tempImageArray]) labelArray = np.concatenate([labelArray, tempLabelArray]) imageArray = imageArray[1:] labelArray = labelArray[1:] # random shuffling of images and labels p = np.random.permutation(imageArray.shape[0]) imageArray = imageArray[p] labelArray = labelArray[p] return imageArray, labelArray
def __init__(self, data_dir: Path, split: AnyStr, input_field_name: AnyStr, output_field_name: AnyStr, fileid_field_name, load_into_memory: bool, mapping_index_dict, has_gt_text: bool) \ -> None: """Initialization of a Clotho dataset object. :param data_dir: Directory with data. :type data_dir: pathlib.Path :param split: Split to use (i.e. 'development', 'evaluation') :type split: str :param input_field_name: Field name of the clotho data\ to be used as input data to the\ method. :type input_field_name: str :param output_field_name: Field name of the clotho data\ to be used as output data to the\ method. :type output_field_name: str :param load_into_memory: Load all data into memory? :type load_into_memory: bool :param return_file_id: whether to return file id :type return_file_id: bool :param has_gt_text: is it the development subset for which we have GT text? :type has_gt_text: bool """ super(ClothoDataset, self).__init__() the_dir: Path = data_dir.joinpath(split) self.examples: List[Path] = sorted(the_dir.iterdir()) self.input_name: str = input_field_name self.output_name: str = output_field_name self.load_into_memory: bool = load_into_memory self.fileid_field_name = fileid_field_name self.mapping_index_dict = mapping_index_dict self.has_gt_text: bool = has_gt_text if load_into_memory: self.examples: List[recarray] = [ np_load(str(f), allow_pickle=True) for f in self.examples ]
def __init__(self, data_dir: Path, split: AnyStr, input_field_name: AnyStr, output_field_name: AnyStr, load_into_memory: bool) \ -> None: super(ClothoDataset, self).__init__() the_dir: Path = data_dir.joinpath(split) self.examples: List[Path] = sorted(the_dir.iterdir()) self.input_name: str = input_field_name self.output_name: str = output_field_name self.load_into_memory: bool = load_into_memory if load_into_memory: self.examples: List[recarray] = [np_load(str(f), allow_pickle=True) for f in self.examples]
def gather_captions_to_text(caption_dir, out_fpath): fh = open(out_fpath, 'wt') i = 0 for npy_fpath in glob.glob(caption_dir + '/*.npy'): recarray = np_load(str(npy_fpath), allow_pickle=True) word_indices_list = recarray['words_ind'][0] # print(word_indices_list) word_str_list = [index2word[w] for w in word_indices_list] # word_str = ' '.join(word_str_list).replace('<sos> ', '') word_str = ' '.join(word_str_list) # print(npy_fpath, word_str) fh.write(word_str + '\n') i += 1 # if i==2: break print("wrote %d lines to file"%i)
def __getitem__(self, item: int) -> Tuple[ndarray, ndarray]: """Gets an example from the dataset. :param item: Index of the item. :type item: int :return: Input and output values. :rtype: numpy.ndarray. numpy.ndarray """ ex: recarray = self.examples[item] if not self.load_into_memory: ex: recarray = np_load(str(ex), allow_pickle=True) in_e, ou_e = [ ex[i].item() for i in [self.input_name, self.output_name] ] return in_e, ou_e
def mnist_train(b:Model)->None: o = build_outpath(b) c = build_cattrs(b) with np_load(build_path(b, c.dataset), allow_pickle=True) as f: b.x_train, b.y_train = f['x_train'], f['y_train'] b.x_test, b.y_test = f['x_test'], f['y_test'] b.x_train = b.x_train.reshape(b.x_train.shape[0], 28, 28, 1).astype('float32') / 255 b.y_train = to_categorical(b.y_train, 10) b.x_test = b.x_test.reshape(b.x_test.shape[0], 28, 28, 1).astype('float32') / 255 b.y_test = to_categorical(b.y_test, 10) print('x_train shape:', b.x_train.shape) print(b.x_train.shape[0], 'train samples') print(b.x_test.shape[0], 'test samples') model = Sequential() b.model = model model.add(Conv2D(32, kernel_size=(3, 3), activation = 'relu', input_shape = (28,28,1))) model.add(Conv2D(64, (3, 3), activation = 'relu')) model.add(MaxPool2D(pool_size = (2,2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation = 'relu')) model.add(Dropout(0.5)) model.add(Dense(10, activation = 'softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) callbacks = [ ModelCheckpoint( monitor='val_accuracy', filepath=join(o, "checkpoint.ckpt"), save_weights_only=True, save_best_only=True, verbose=True)] model.fit(b.x_train, b.y_train, batch_size=32, epochs=c.num_epoches, verbose=0, callbacks=callbacks, validation_split=0.2)
def showImageMatrix(dataSet, cls, instances): ''' Inputs : dataSets : List : Datasets for which samples are to be genrated instances : List : Number of instances to be used from original dataset classes : List : Classes for which samples are to be generated Outputs : 5x5 image matrix ''' fileName = '../DCGAN/results/compressed' + '/' + dataSet + '/' + dataSet + '_' + str( cls) + '_' + str(instances) + '.npy' images = np_load(fileName) # get random list of images to be displayed randomList = np.random.randint(0, 1000, (25)) imageList = images[randomList] fmt = 'png' # need to generalise this snippet fig, axes = plt.subplots(5, 5) fig.tight_layout() fig.subplots_adjust(wspace=-0.7, hspace=-0.1) plt.axis('off') for i in range(5): for j in range(5): f = StringIO() image = PIL.Image.fromarray(np.uint8(imageList[i * 5 + j])) image.save(f, fmt) axes[i, j].imshow(image, cmap='Greys_r') axes[i, j].axis('off') axes[i, j].set_xticklabels([]) axes[i, j].set_yticklabels([]) axes[i, j].set_aspect("equal") plotFileName = 'results' + '/' + 'samples' + '/' + dataSet + '/' + dataSet + '_' + str( cls) + '_' + str(instances) + '.png' plt.savefig(plotFileName, bbox_inches='tight') if params.showImage == 1: plt.show()
def __getitem__(self, item: int): """Gets an example from the dataset. :param item: Index of the item. :type item: int :return: Input and output values. :rtype: numpy.ndarray. numpy.ndarray """ ex: Union[Path, recarray] = self.examples[item] if not self.load_into_memory: ex: recarray = np_load(str(ex), allow_pickle=True) in_e, ou_e = [ ex[i].item() for i in [self.input_name, self.output_name] ] all_ref = get_all_ref(ex['file_name'].item(), self.data_dir) filename = str(ex['file_name'].item()) out_len = len(ou_e) return in_e, ou_e, all_ref, filename, out_len
def learn_manifold(self, X, manifold_out_file_name=None): self.debug_string_out.clear() self.print_and_remember("Learning manifold(" + self.manifold_learner + ")" + str(datetime.now())) learn_time = time() if manifold_out_file_name is not None and isfile( manifold_out_file_name ): # check the learned manifold existance manifold_feats = np_load(manifold_out_file_name, allow_pickle=True) self.print_and_remember("Manifold loaded(" + manifold_out_file_name + ")") elif self.manifold_learner == 'UMAP': manifold_feats = UMAP(random_state=0, metric=self.dist_metric, n_components=self.manifold_dimension, n_neighbors=self.num_of_neighbours, min_dist=float( self.min_dist)).fit_transform(X) elif self.manifold_learner == 'LLE': manifold_feats = LocallyLinearEmbedding( n_components=self.manifold_dimension, n_neighbors=self.num_of_neighbours).fit_transform(X) elif self.manifold_learner == 'tSNE': manifold_feats = TSNE(n_components=self.manifold_dimension, random_state=0, verbose=0).fit_transform(X) elif self.manifold_learner == 'isomap': manifold_feats = Isomap( n_components=self.manifold_dimension, n_neighbors=self.num_of_neighbours).fit_transform(X) self.print_and_remember( "Time to learn manifold: " + str(funcH.getElapsedTimeFormatted(time() - learn_time))) if manifold_out_file_name is not None: np_save(manifold_out_file_name, manifold_feats, allow_pickle=True) self.print_and_remember("Manifold saved(" + manifold_out_file_name + ")") return manifold_feats, self.debug_string_out
def __getitem__(self, item: int) \ -> Tuple[List[ndarray], List[ndarray], List[Path]]: """Gets an example from the dataset. :param item: Index of the item. :type item: int :return: Input and output values, and the Path of the file. :rtype: list[numpy.ndarray], list[numpy.ndarray], list[Path] """ ex = self.examples[item] if self.multiple_captions_mode \ else [self.examples[item]] if not self.load_into_memory: ex = [np_load(str(i), allow_pickle=True) for i in ex] x, y, file_names = [], [], [] for ex_i in ex: x.append(ex_i[self.input_name]) y.append(ex_i[self.output_name]) file_names.append(Path(ex_i.file_name[0])) return x, y, file_names
def __getitem__(self, item: int) \ -> Tuple[ndarray, ndarray]: """Gets an example from the dataset. :param item: Index of the item. :type item: int :return: Input and output values. :rtype: numpy.ndarray. numpy.ndarray """ ex: Union[Path, recarray] = self.examples[item] if not self.load_into_memory: ex: recarray = np_load(str(ex), allow_pickle=True) if self.fileid_field_name is not None: fid_e = [ex[i].item() for i in [self.fileid_field_name]] if self.has_gt_text: in_e, ou_e = [ ex[i].item() for i in [self.input_name, self.output_name] ] # print("dataset class, text with 5k words:", ou_e, len(ou_e)) if self.mapping_index_dict is not None: ou_e = [ self.mapping_index_dict[ind] for ind in ou_e if ind in self.mapping_index_dict ] ou_e = array(ou_e, dtype=int) # print(" text with 1k words:", ou_e, len(ou_e)) if self.fileid_field_name is not None: return in_e, ou_e, fid_e else: return in_e, ou_e else: in_e = [ex[i].item() for i in [self.input_name]] if self.fileid_field_name is not None: return in_e, fid_e else: return in_e
def whatyouwant(paramin): global dir_pri1, dir_pri2, dir_pri3, patient, NormW ofi1 = open(dir_pri2 + '/TC%ia.dat' %patient,'r') ofi2 = open(dir_pri2 + '/TC%ib.dat' %patient,'r') A = array([line.split() for line in ofi1]).astype(float) B = array([line.split() for line in ofi2]).astype(float) C = concatenate([A,B], axis=1) ofi1.close() ofi2.close() dir_sub1 = dir_pri1 + '/Norm_%.2f_DensMoy_%.2f' %(NormW, paramin) nbpatterns = len(os.listdir(dir_sub1)) - 1 nbtimeshap = size(C, axis=1) nruter = zeros((nbpatterns, nbtimeshap)) for p1 in range(nbpatterns): d_patty = dir_sub1 + '/pattern_%.3i.npy' %p1 patty = np_load(d_patty) for p2 in range(nbtimeshap): nruter[p1,p2] = corrcoef(patty, C[:,p2])[0,1] return nruter
def compute_tsne_positions(activations_per_point: array, perplexity: int, name: str) -> array: """ Computes t-SNE positions from a dataset. `activations_per_point`: n_obvs x n_dims """ logger.info(f"TSNE from data of size {activations_per_point.shape}") t_sne_positions_path = Path(TSNE_SAVE_DIR, f"t-sne positions {name} perp={perplexity}.npy") if t_sne_positions_path.exists(): logger.info("Loading...") t_sne_positions = np_load(t_sne_positions_path) else: logger.info("Computing...") t_sne_positions = TSNE( n_components=2, # 2D perplexity=perplexity, # Recommended args n_iter=1_000, learning_rate=200, method="barnes_hut", ).fit_transform(activations_per_point) np_save(t_sne_positions_path, t_sne_positions) return t_sne_positions
args.add_argument('-m', '--methods', help='Dimensionality reduction methods (comma-sep)', default='svd,nmf,plsa,lda,slda') args.add_argument('-l', '--label', help='Value being predcicted', required='True') args = args.parse_args() return args if __name__=="__main__": args = interface() cv_dir = args.input_dir techniques = get_methods(args.methods) techniques = [t[1].upper() for t in techniques] num_folds = args.num_folds * args.cv_iters dim_steps = [int(d) for d in args.dims.split(',')] metadata_values = np_load(path.join(cv_dir, 'labels.npy')) metadata_category = args.label mdl = RandomForestClassifier() qual = roc_auc_score plot_data = [] cv_scores = zeros(num_folds) output = open(args.output_file + '.csv', 'w') for technique in techniques: print technique tech_data = [] for d in dim_steps: for c in xrange(num_folds): prefix = 'CV_%d_%s_%d_' % (c, technique, d) prefix = path.join(cv_dir, prefix)
system('chmod u+x %s' % (filename)) if __name__=="__main__": args = interface() data_matrix_file = args.input_file labels_file = args.input_labels output_dir = args.output_dir dim_steps = [int(x) for x in args.dims.split(',')] output_dir = args.output_dir scripts_dir = args.scripts_dir tag = args.tag labels = np_load(labels_file) cv_folds = BalancedKFold(labels, args.num_folds, n_iter=args.cv_iters) i = 0 for k, (training, testing) in enumerate(cv_folds): file_prefix = path.join(output_dir, 'CV_%d_' % k) training_file = file_prefix+'training.npy' np_save(training_file, training) testing_file = file_prefix+'testing.npy' np_save(testing_file, testing) for num_dims in dim_steps: if i >= args.max_scripts: ind = i % args.max_scripts script_file = path.join(scripts_dir, '%d.sh' % ind) add_to_bash_script(script_file, data_matrix_file, labels_file, output_dir,
def load(self, directory, name): filename = path.join(directory, name) if not '.npz' in name: filename += '.npz' err = 0 try: data = np_load(filename) except: data = {} if 'A_equil' in data: A_equil = data['A_equil'].astype(lib.pyfloat) elif path.exists(path.join(directory, 'A_equil.npy')): A_equil = np_load(path.join(directory, 'A_equil.npy')).astype(lib.pyfloat) else: err = 1 if not err and 'LLT' in data: LLT = data['LLT'].astype(lib.pyfloat) elif path.exists(path.join(directory, 'LLT.npy')): LLT = np_load(path.join(directory, 'LLT.npy')).astype( lib.pyfloat) LLT_ptr = LLT.ctypes.data_as(lib.ok_float_p) else: if lib.direct: err = 1 else: LLT_ptr = c_void_p() if not err: LLT_ptr = LLT.ctypes.data_as(lib.ok_float_p) else: LLT_ptr = None if not err and 'd' in data: d = data['d'].astype(lib.pyfloat) elif path.exists(path.join(directory, 'd.npy')): d = np_load(path.join(directory, 'd.npy')).astype( lib.pyfloat) else: err = 1 if not err and 'e' in data: e = data['e'].astype(lib.pyfloat) elif path.exists(path.join(directory, 'e.npy')): e = np_load(path.join(directory, 'e.npy')).astype( lib.pyfloat) else: err = 1 if err: snippet = '`LLT`, ' if lib.direct else '' ValueError('Minimal requirements to load solver ' 'not met. Specified file must contain ' 'at least one .npz file with entries `A_equil`,' ' {}`d`, and `e`, or the specified folder must' ' contain .npy files of the same names.'.format( snippet)) if 'z' in data: z = data['z'].astype(lib.pyfloat) else: z = zeros(self.m + self.n, dtype=lib.pyfloat) if 'z12' in data: z12 = data['z12'].astype(lib.pyfloat) else: z12 = zeros(self.m + self.n, dtype=lib.pyfloat) if 'zt' in data: zt = data['zt'].astype(lib.pyfloat) else: zt = zeros(self.m + self.n, dtype=lib.pyfloat) if 'zt12' in data: zt12 = data['zt12'].astype(lib.pyfloat) else: zt12 = zeros(self.m + self.n, dtype=lib.pyfloat) if 'zprev' in data: zprev = data['zprev'].astype(lib.pyfloat) else: zprev = zeros(self.m + self.n, dtype=lib.pyfloat) if 'rho' in data: rho = lib.pyfloat(data['rho']) else: rho = 1. order = lib.enums.CblasRowMajor if \ A_equil.flags.c_contiguous else lib.enums.CblasColMajor if self.c_solver is not None: self.__unregister_solver() self.__register_solver(lib, lib.pogs_load_solver( A_equil.ctypes.data_as(lib.ok_float_p), LLT_ptr, d.ctypes.data_as(lib.ok_float_p), e.ctypes.data_as(lib.ok_float_p), z.ctypes.data_as(lib.ok_float_p), z12.ctypes.data_as(lib.ok_float_p), zt.ctypes.data_as(lib.ok_float_p), zt12.ctypes.data_as(lib.ok_float_p), zprev.ctypes.data_as(lib.ok_float_p), rho, self.m, self.n, order))
def start(self): self.syn0 = np_load("%s.syn0.npy" % self.path) self.syn1neg = np_load("%s.syn1neg.npy" % self.path) with open("%s.vocab" % self.path, "r") as f: self.vocab = pickle.load(f) self.cum_table = np_load("%s.cum_table.npy" % self.path)
def get_config_dict(): """Return the config dict (create the default one if needed) Returns ------- config_dict: dict Dictionnary gather the parameters of the software """ # dynamic import to avoid loop module = __import__( "pyleecan.definitions", globals=globals(), locals=locals(), fromlist=["USER_DIR", "CONF_PATH"], level=0, ) USER_DIR = module.USER_DIR CONF_PATH = module.CONF_PATH logger = getLogger("Pyleecan") # Initialization to make sure all the parameters exist init_user_dir() config_dict = default_config_dict.copy() if isfile(CONF_PATH): with open(CONF_PATH, "r") as config_file: update_dict(source=config_dict, update=load(config_file)) else: logger.debug("Creating missing config_dict in " + CONF_PATH) save_config_dict(config_dict) # Load the color_dict color_path = join(USER_DIR, "Plot", config_dict["PLOT"]["COLOR_DICT_NAME"]) def_color_path = join(USER_DIR, "Plot", "pyleecan_color.json") if not isfile(color_path): # Default colors logger.warning("Unable to load colors from " + color_path + ", using default colors") color_path = join(USER_DIR, "Plot", "pyleecan_color.json") # Load default to make sure that the keys are all there with open(def_color_path, "r") as color_file: config_dict["PLOT"]["COLOR_DICT"] = load(color_file) if color_path != def_color_path: with open(color_path, "r") as color_file: update_dict(source=config_dict["PLOT"]["COLOR_DICT"], update=load(color_file)) # Register the colormap cmap_name = config_dict["PLOT"]["COLOR_DICT"]["COLOR_MAP"] if "." not in cmap_name: cmap_path = join(USER_DIR, "Plot", cmap_name) + ".npy" try: get_cmap(name=cmap_name) except: if not isfile(cmap_path): # Default colormap config_dict["PLOT"]["COLOR_DICT"]["COLOR_MAP"] = "RdBu_r" else: cmap = np_load(cmap_path) cmp = ListedColormap(cmap) register_cmap(name=config_dict["PLOT"]["COLOR_DICT"]["COLOR_MAP"], cmap=cmp) return config_dict
args.add_argument('-n', '--cv-fold-id', help='CV fold id', type=int, required=True) args.add_argument('-m', '--methods', help='Dimensionality reduction methods (comma-sep)', default='svd,nmf,plsa,lda,slda') args = args.parse_args() return args if __name__=="__main__": args = interface() num_dims = args.num_dims prefix = 'CV_%d' % (args.cv_fold_id) techniques = get_methods(args.methods) output_dir = args.output_dir data_matrix = np_load(args.input_file) labels = np_load(args.input_labels) testing = np_load(args.testing_vector) training = np_load(args.training_vector) test_labels = labels[testing] training_labels = labels[training] test_matrix = data_matrix[testing, :] training_matrix = data_matrix[training, :] for technique, technique_name in techniques: dim_redux = technique(n_components=num_dims) file_label = '%s_%s_%s_' % (prefix, technique_name, str(num_dims)) file_prefix = path.join(output_dir, file_label)