def main(): # Get label encoder lb = LabelBinarizer() lbenc = lb.fit(utils.get_classes()) # Get train data X_train, y_train, train_filenames = utils.get_train( '../input/train', list(lbenc.classes_), img_width, img_height) # Create and train model model = train(X_train, y_train, epochs=100, batch_size=32) print("+++++++++++++++++++++++++++++++++++++++++++") # Load model ... #model = load_model('../models/'+ 'model2_f0.86/'+ 'model2-64-0.341.h5') # Get test data X_test, X_test_id = utils.get_test('../input/test', img_width, img_height) # Predict on test data preds = model.predict(X_test, verbose=1) # Create submission utils.create_submission(lbenc.inverse_transform(preds), X_test_id, output_path="../submissions/", filename=modelname, isSubmission=True) utils.to_csv_ens(lbenc.inverse_transform(preds), preds, X_test_id, utils.get_classes(), output_path="../submissions/", filename=modelname) print('Finished.')
def eval(): # Get classes lb = LabelBinarizer() lbenc = lb.fit(utils.get_classes()) # Load model model = load_model('../models/' + 'model.h5') # Get test data X_test, X_test_id = utils.get_test('../input/test', img_width, img_height) # Predict on test data preds = model.predict(X_test, verbose=1) # Create ensembling file df_csv = utils.to_csv_ens(lbenc.inverse_transform(preds), preds, X_test_id, utils.get_classes(), output_path="../submissions/", filename=modelname) # Create submission file subm = utils.create_submission(lbenc.inverse_transform(preds), X_test_id, output_path="../submissions/", filename=modelname, isSubmission=True)
def _main(args): os.chdir(os.path.join(os.getcwd(), 'yad2k-em3d')) # Parse input arguments anchors_path = os.path.expanduser(args.anchors_path) classes_path = os.path.expanduser(args.classes_path) data_path = os.path.expanduser(args.data_path) test_results = os.path.expanduser(args.test_results) # Extract anchors, classes, images, and boxes from input files anchors = utils.get_anchors(anchors_path) classes = utils.get_classes(classes_path) images, boxes = utils.get_data(data_path) test_results = scipy.io.loadmat(test_results) t = test_results['output'] cv2.imshow("TESTING", images[20]) cv2.waitKey(0) ipdb.set_trace() classes = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 ] class_names = [ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z' ] drawn = utils.draw_boxes(images[0], t[:1], classes, class_names, scores=t[2]) cv2.imshow('drawm', drawn) cv2.waitKey(0)
def test(): """Plot all defined classes in fuzzy.set package""" import fuzzy.set objects = get_classes(fuzzy.set) # add demo sets from fuzzy.set.Polygon import Polygon objects["Polygon (Demo)"] = Polygon([ (-1.2,0), (-1.2,1), (-0.8,0.3), (-0.3,0.2), (-0.2,0.4), (-0.1,0.0), (0.0,0.0), (0.3,1), (0.6,0.5), (0.6,0.1), (1.3,0.6), ]) for name in sorted(objects): if name in ["Set", "Function","Polygon"]: continue obj = objects[name] try: plotSet(obj,name) except: import traceback traceback.print_exc()
def get_val_loader(args, val_index, batch_size=32, dev_mode=False, val_num=3000): classes, stoi = get_classes(args.cls_type, args.start_index, args.end_index) _, val_meta = get_train_val_meta(args.cls_type, args.start_index, args.end_index) # filter, keep label counts <= args.max_labels val_meta = val_meta[val_meta['obj_num'] <= args.max_labels] if len(classes) < 7172: classes_set = set(classes) val_meta['tmp_label_count'] = val_meta['LabelName'].map(lambda x: len(set(x.split()) & classes_set)) val_meta = val_meta[val_meta['tmp_label_count'] > 0] #print(val_meta.shape) #print(val_meta['LabelName'].str.split().apply(pd.Series).stack().nunique()) val_meta = shuffle(val_meta, random_state=1234).iloc[:val_num] #print(val_meta.shape) if dev_mode: val_meta = val_meta.iloc[:10] img_dir = settings.TRAIN_IMG_DIR val_set = ImageDataset(False, val_meta['ImageID'].values.tolist(), img_dir, classes, stoi, val_index, val_meta['LabelName'].values.tolist()) val_loader = data.DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=4, drop_last=False) val_loader.num = val_set.num return val_loader
def main(data_file, sep): classes_column = "Hogwarts House" data, _ = read_data(data_file, sep) num_data = get_numerics(data, get_str=False) class_list = get_classes(data, classes_column) matter = find_most_equal(num_data, class_list) print("Matter with the most homogeneous repartition between houses : %s" % matter) fig = plt.figure("Histogram") # for each matter for i, key in enumerate(num_data.keys()): ax = fig.add_subplot(4, 4, i + 1) ax.set_title(key) classes = [] # for each class for c in class_list: c_tab = class_tab(num_data[key], class_list[c]) classes.append(c_tab) for cat in classes: clean_cat = [] for elem in cat: if not math.isnan(elem): clean_cat.append(elem) ax.hist(clean_cat, alpha=0.5) fig.legend(class_list.keys(), loc = (0.8, 0)) fig.tight_layout() plt.show(block = True)
def interactive(name, params): """interactive use: plot complement using given params""" import fuzzy.complement objects = get_classes(fuzzy.complement) try: complement = objects[name] except KeyError: print "%s is unknown." % name return g = getGnuplot() from fuzzy.set.Triangle import Triangle set = Triangle() set_name = "Triangle" if len(params) > 0: plotComplement(complement, name, set, set_name, params, gnuplot=g, interactive=True) else: plotComplement(complement, name, set, set_name, gnuplot=g, interactive=True) g.close()
def get_train_loader(args, batch_size=32, dev_mode=False, train_shuffle=True): classes, stoi = get_classes(args.cls_type, args.start_index, args.end_index) train_meta, _ = get_train_val_meta(args.cls_type, args.start_index, args.end_index) # filter, keep label counts <= args.max_labels train_meta = train_meta[train_meta['obj_num'] <= args.max_labels] print(train_meta.shape) if len(classes) < 7172: classes_set = set(classes) train_meta['tmp_label_count'] = train_meta['LabelName'].map(lambda x: len(set(x.split()) & classes_set)) train_meta = train_meta[train_meta['tmp_label_count'] > 0] #tuning_labels['LabelName'].map(lambda x: sum([cls_counts[c] for c in x.split()])) #print('>>', train_meta.shape) # resample training data train_img_ids = get_weighted_sample(train_meta, 1024*100) df_sampled = train_meta.set_index('ImageID').loc[train_img_ids] if dev_mode: train_meta = train_meta.iloc[:10] train_shuffle = False img_dir = settings.TRAIN_IMG_DIR train_set = ImageDataset(True, train_img_ids, img_dir, classes, stoi, None, df_sampled['LabelName'].values.tolist()) train_loader = data.DataLoader(train_set, batch_size=batch_size, shuffle=train_shuffle, num_workers=4, drop_last=True)#, collate_fn=train_set.collate_fn, drop_last=True) train_loader.num = train_set.num return train_loader
def _main(args): # Parse input arguments model_path = os.path.expanduser(args.model_path) data_path = os.path.expanduser(args.data_path) anchors_path = os.path.expanduser(args.anchors_path) classes_path = os.path.expanduser(args.classes_path) output_path = os.path.expanduser(args.output_path) batch = args.batch assert model_path.endswith('.h5'), 'model_path must have .h5 extension' assert output_path.endswith('.mat'), 'output_path must have .mat extension' # Extract anchors and classes from input files anchors = utils.get_anchors(anchors_path) classes = utils.get_classes(classes_path) images, boxes = utils.get_data(data_path) # Create model and load weights from file model_body, model = create_model(images.shape[1:-1], int(boxes.shape[-1]), anchors, classes) model_body.load_weights(model_path) model.summary() # Pass input data through the network in batches output = model_body.predict(images[0:batch, :, :, :, :]) for i in range(batch, images.shape[0], batch): output = np.concatenate((output, model_body.predict(images[i:i + batch, :, :, :, :]))) # Save output file if output_path != '': os.makedirs(os.path.dirname(output_path), exist_ok=True) scipy.io.savemat(output_path, mdict={'output': output}) print('Results saved to file: {}'.format(output_path))
def load(): """ 加载模型,回传模型和模型参数 """ # 模型参数 class_names = get_classes(classes_path) num_classes = len(class_names) anchors = get_anchors(anchors_path) num_anchors = len(anchors) input_shape = (416, 416) train_weights_path = log_dir + '/ep169-loss17.356-val_loss6.844.h5' # 获取模型结构,加载权重 image_input = Input(shape=(None, None, 3)) model = yolo_body(image_input, num_anchors//3, num_classes) print('Get YOLOv3 model with {} anchors and {} classes.'.format(num_anchors, num_classes)) model.load_weights(train_weights_path, by_name=True, skip_mismatch=True) print('Load weights {}.'.format(train_weights_path)) # 回传参数 param = [class_names, num_classes, anchors, input_shape] return model, param
def get_train_val_loaders(args, batch_size=32, dev_mode=False, train_shuffle=True, val_num=4000): classes, stoi = get_classes(args.cls_type, args.start_index, args.end_index) train_meta, val_meta = get_train_val_meta(args.cls_type, args.start_index, args.end_index) #sampler = BalancedSammpler(train_meta, classes, stoi, balanced=args.balanced, min_label_num=500, max_label_num=700) #df1 = train_meta.set_index('ImageID') #sampled_train_meta = df1.loc[sampler.img_ids] train_meta = train_meta[train_meta['obj_num'] <= 10] val_meta = val_meta[val_meta['obj_num'] <= 10] # resample training data train_img_ids = get_weighted_sample(train_meta, 1024*100) df_sampled = train_meta.set_index('ImageID').loc[train_img_ids] #print(df_sampled.shape) if val_num is not None: val_meta = val_meta.iloc[:val_num] #if dev_mode: # train_meta = train_meta.iloc[:10] # val_meta = val_meta.iloc[:10] img_dir = settings.TRAIN_IMG_DIR #train_set = ImageDataset(True, sampled_train_meta.index.values.tolist(), img_dir, classes, stoi, sampled_train_meta['LabelName'].values.tolist()) train_set = ImageDataset(True, train_img_ids, img_dir, classes, stoi, df_sampled['LabelName'].values.tolist()) val_set = ImageDataset(False, val_meta['ImageID'].values.tolist(), img_dir, classes, stoi, val_meta['LabelName'].values.tolist()) train_loader = data.DataLoader(train_set, batch_size=batch_size, shuffle=train_shuffle, num_workers=4, collate_fn=train_set.collate_fn, drop_last=True) train_loader.num = train_set.num val_loader = data.DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=val_set.collate_fn, drop_last=False) val_loader.num = val_set.num return train_loader, val_loader
def get_model2(input_shape=(128, 128, 3)): model = Sequential() model.add(BatchNormalization(input_shape=input_shape)) # Convolution + Pooling Layer model.add(Conv2D(32, (5, 5), padding='same', activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) # Convolution + Pooling Layer model.add(Conv2D(32, (5, 5), padding='same', activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) # Convolution + Pooling Layer model.add(Conv2D(64, (5, 5), padding='same', activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) # Convolution + Pooling Layer model.add(Conv2D(64, (5, 5), padding='same', activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) # Flatten model.add(Flatten()) # Fully-Connection model.add(Dense(64, activation='relu')) model.add(BatchNormalization()) model.add(Dropout(0.4)) model.add(Dense(32, activation='relu')) model.add(BatchNormalization()) model.add(Dropout(0.2)) # Output model.add(Dense(len(utils.get_classes()), activation='softmax')) optimizer = Adam(1e-4) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) return model
def ensemble_np(args): if args.th < 0: raise AssertionError('Please specify threshold') np_files = args.ensemble_np.split(',') if len(np_files) < 1: raise AssertionError('no np files') outputs = [] for np_file in np_files: if not os.path.exists(np_file): raise AssertionError('np file does not exist') output = np.load(np_file) print(np_file, output.shape) outputs.append(output) ensemble_outputs = np.mean(outputs, 0) preds = (ensemble_outputs > args.th).astype(np.uint8) classes, _ = get_classes(args.cls_type, args.start_index, args.end_index) label_names = [] for row in preds: label_names.append(get_label_names(row, classes)) if args.check: print(label_names[:10]) return create_submission(args, label_names, args.sub_file)
def save_error(label, prediction, file): classes = utils.get_classes() #classes = train_model_mp.get_classes() path = "./errors/%s_classified_as_%s" % (classes[label], classes[prediction]) if not os.path.isdir(path): os.makedirs(path) shutil.copyfile(file, path + "/" + os.path.basename(file))
def evaluate_model_fast(model_name, path): model = keras.models.load_model(model_name) images, labels, files = utils.load_images_classes(path) predictions = [] errors = [] num_classes = len(utils.get_classes()) confusion_matrix = np.zeros((num_classes, num_classes)) print("Evaluating model (fast version) on %i instances..." % images.shape[0]) start = time.time() prediction_probs = model.predict(images) print("Elased time: %f seconds" % (time.time() - start)) for i in range(0,images.shape[0]): predicted_class = np.argmax(prediction_probs[i, :]) predictions.append(predicted_class) label = int(labels[i]) confusion_matrix[label,predicted_class] += 1 if label == predicted_class: errors.append(0) else: save_error(label, predicted_class, files[i]) errors.append(1) print_statistics(confusion_matrix)
def evaluate_model(model_filepath, path): model_name = model_filepath[28:-3] model = keras.models.load_model(model_filepath) images, labels, files = utils.load_images_classes(path) predictions = [] errors = [] num_classes = len(utils.get_classes()) confusion_matrix = np.zeros((num_classes, num_classes)) print("Evaluating model...") if os.path.isdir("./errors"): shutil.rmtree("./errors") for i in range(0,images.shape[0]): if i % 1000 == 0: print(i) prediction_probs = predict_image_set_with_augmentation(images[i:i+1, :, :, :],model)[0] predicted_class = np.argmax(prediction_probs) predictions.append(predicted_class) label = int(labels[i]) confusion_matrix[label, predicted_class] += 1 if label == predicted_class: errors.append(0) else: errors.append(1) np.save('./Results/Testing_Metrics/Confusion_Matrix/' + model_name + '_cm.npy', confusion_matrix) print('./Results/Testing_Metrics/Confusion_Matrix/' + model_name + '_cm.npy') print_statistics(confusion_matrix)
def predict_softmax(args): model, _ = create_model(args) model = model.cuda() model.eval() test_loader = get_test_loader(args, batch_size=args.batch_size, dev_mode=args.dev_mode) preds = None with torch.no_grad(): for i, x in enumerate(test_loader): x = x.cuda() #output = torch.sigmoid(model(x)) output, _ = model(x) output = F.softmax(output, dim=1) pred = (output > 0.03).byte() # use threshold if preds is None: preds = pred.cpu() else: preds = torch.cat([preds, pred.cpu()], 0) print('{}/{}'.format(args.batch_size*(i+1), test_loader.num), end='\r') classes, _ = get_classes(args.cls_type, args.start_index, args.end_index) label_names = [] preds = preds.numpy() print(preds.shape) n_classes = 7172 for row in preds: label_names.append(' '.join([classes[i] for i in range(n_classes) if row[i] == 1])) if args.dev_mode: print(len(label_names)) print(label_names) create_submission(args, label_names, args.sub_file)
def test(): """Plot all defined classes in fuzzy.set package""" import fuzzy.set objects = get_classes(fuzzy.set) # add demo sets from fuzzy.set.Polygon import Polygon objects["Polygon (Demo)"] = Polygon([ (-1.2, 0), (-1.2, 1), (-0.8, 0.3), (-0.3, 0.2), (-0.2, 0.4), (-0.1, 0.0), (0.0, 0.0), (0.3, 1), (0.6, 0.5), (0.6, 0.1), (1.3, 0.6), ]) for name in sorted(objects): if name in ["Set", "Function", "Polygon"]: continue obj = objects[name] try: plotSet(obj, name) except: import traceback traceback.print_exc()
def predict(image_path, model_path, top_k, class_names_json): ''' Function takes inputs and prints top predicted class name, label and probability for flower image. It also prints the top k results for flower image. INPUT: image_path - (str) path to image. model_path - (str) path to tensorflow model (h5). top_k - (int) Top K results requested. class_names (json_file) Dict [class names : class ids] OUTPUT: NONE ''' #Getting mapping file for class index and class names class_names = get_classes(class_names_json) #Reads Tensorflow model. model = tf.keras.models.load_model( model_path, custom_objects={'KerasLayer': hub.KerasLayer}) #open image img = Image.open(image_path) #put image into array image_numpy = np.asarray(img) #resize image for processing processed_image = process_image(image_numpy) #Predict image using tensorflow model prob_preds = model.predict(np.expand_dims(processed_image, axis=0)) prob_preds = prob_preds[0] #Get top_k results as tensors. values, index = tf.math.top_k(prob_preds, k=top_k) #Conver tensors to numpy for use. probs = values.numpy().tolist() class_index = index.numpy().tolist() #Map class ids to class names. pred_label_names = [] for i in class_index: pred_label_names.append(class_names[str(i)]) #1 Result print( f"""\n\n Class most likely based on the following {image_path} with the highest probaility as listed: \n class_id: {class_index[0]} \n class_label: {pred_label_names[0]} \n probability: {str(round(float(probs[0]) *100, 2)) + '%'} \n\n\n """) if top_k > 1: print(f"\n Top {top_k} probs", probs) print(f"\n Top {top_k} class names", pred_label_names) print(f"\n Top {top_k} class ids", class_index) print("\n\n")
def create_sub_from_raw_csv(args, csv_file): classes, _ = get_classes() df = pd.read_csv(csv_file) df = df[classes] outputs = torch.from_numpy(df.values) _, preds = outputs.topk(3, 1, True, True) preds = preds.numpy() create_submission(args, preds, args.sub_file)
def create_submission(preds, outfile): classes, _ = get_classes() label_names = [] for row in preds: label_names.append(' '.join([classes[i] for i in row])) meta = pd.read_csv(settings.SAMPLE_SUBMISSION) meta['word'] = label_names meta.to_csv(outfile, index=False)
def get(self): user = users.get_current_user() classes = utils.get_classes(user) lessons, lessons_map = utils.get_lessons(user) template = jinja_env.get_template("templates/index.html") self.response.out.write(template.render({'lessons': lessons, 'user_email': user.email(), 'user_name': user.nickname(), 'classes': classes, 'logout_url': users.create_logout_url("/")}))
def train(): data_file_path = 'shuf_train_file' log_dir = 'logs/' classes_path = 'my_class.txt' classes_name = get_classes(classes_path) num_classes = len(classes_name) batch_size = 32 width = 331 hight = 331 base_model = NASNetLarge(input_shape=(width, hight, 3), weights='imagenet', include_top=False, pooling='avg') input_tensor = Input(shape=(None, None, 3)) x = input_tensor # x = Lambda(preprocess_input)(x) x = base_model(x) x = Dropout(0.5)(x) x = Dense(num_classes, activation='softmax')(x) model = Model(input_tensor, x) logging = TensorBoard(log_dir=log_dir) checkpoint = ModelCheckpoint(log_dir + 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5', monitor='val_loss', save_weights_only=True, save_best_only=True, period=3) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1) early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=1) val_split = 0.1 with open(data_file_path, 'r') as f1: lines = f1.readlines() np.random.seed(10101) np.random.shuffle(lines) np.random.seed(None) num_val = int(len(lines) * val_split) num_train = len(lines) - num_val if True: for i in range(len(model.layers) - 2): model.layers[i].trainable = False model.summary() model.compile(optimizer=Adam(lr=1e-3), loss='categorical_crossentropy', metrics=['acc']) model.fit_generator(data_generator_wrapper(lines[:num_train], batch_size=batch_size, input_shape=(width, hight), num_classes=num_classes), steps_per_epoch=max(1, num_train // batch_size), validation_data=data_generator_wrapper(lines[num_train:], batch_size=batch_size, input_shape=(width, hight), num_classes=num_classes), validation_steps=max(1, num_val // batch_size), epochs=10, initial_epoch=0, callbacks=[logging, checkpoint, reduce_lr, early_stopping]) model.save(log_dir + 'trained_weights_stage_1.h5') pass
def create_submission(args, preds, outfile): classes, _ = get_classes() label_names = [] for row in preds: label_names.append(' '.join([classes[i] for i in row])) meta = pd.read_csv(settings.SAMPLE_SUBMISSION) if args.dev_mode: meta = meta.iloc[:len(label_names)] # for dev mode meta['word'] = label_names meta.to_csv(outfile, index=False)
def get_test_loader(args, batch_size=8, dev_mode=False, tta_index=0): img_ids = get_test_ids() classes, stoi = get_classes(args.cls_type, args.start_index, args.end_index) img_dir = settings.TEST_IMG_DIR if dev_mode: img_ids = img_ids[:10] dset = ImageDataset(False, img_ids, img_dir, classes, stoi, tta_index=tta_index) dloader = data.DataLoader(dset, batch_size=batch_size, shuffle=False, num_workers=4, collate_fn=dset.collate_fn, drop_last=False) dloader.num = dset.num return dloader
def save_raw_csv(np_file): df = pd.read_csv(settings.SAMPLE_SUBMISSION) np_dir = os.path.dirname(np_file) csv_file_name = os.path.join(np_dir, 'raw.csv') outputs = np.load(np_file) classes, _ = get_classes() for i, c in enumerate(classes): df[c] = outputs[:, i] col_names = ['key_id', *classes] df.to_csv(csv_file_name, index=False, columns=col_names)
def test(): """Show examples for all norm in package fuzzy.norm""" objects = get_classes(fuzzy.norm) for name in sorted(objects): if name in ["Norm","ParametricNorm"]: continue try: norm = objects[name] plotNorm(norm,name) except: import traceback traceback.print_exc()
def test(): """Show examples for all complements in package fuzzy.complement""" import fuzzy.set import fuzzy.complement objects = get_classes(fuzzy.set) # add demo sets #from fuzzy.set.Polygon import Polygon #objects["Polygon (Demo)"] = Polygon([ # (-1.2,0), # (-1.2,1), # (-0.8,0.3), # (-0.3,0.2), # (-0.2,0.4), # (-0.1,0.0), # (0.0,0.0), # (0.3,1), # (0.6,0.5), # (0.6,0.1), # (1.3,0.6), # ]) complements = get_classes(fuzzy.complement) #print complements for name in sorted(objects): if name in ["Set", "Function", "Polygon", "Singleton"]: continue obj = objects[name] for name2 in sorted(complements): if name2 in ["Base","Parametric"]: continue try: complement = complements[name2] plotComplement(complement,name2,obj,name) except: import traceback traceback.print_exc()
def test(): """Show examples for all norm in package fuzzy.norm""" objects = get_classes(fuzzy.norm) for name in sorted(objects): if name in ["Norm", "ParametricNorm"]: continue try: norm = objects[name] plotNorm(norm, name) except: import traceback traceback.print_exc()
def check_classes(): tmp = get_classes() print(tmp[:10]) classes = set(tmp) print(len(classes)) bbox = build_bbox_dict() v = bbox.values() print(len(v)) for x in v: for c, _ in x: if not (c in classes): print(c) print('done')
def test(): """Show examples for all complements in package fuzzy.complement""" import fuzzy.set import fuzzy.complement objects = get_classes(fuzzy.set) # add demo sets #from fuzzy.set.Polygon import Polygon #objects["Polygon (Demo)"] = Polygon([ # (-1.2,0), # (-1.2,1), # (-0.8,0.3), # (-0.3,0.2), # (-0.2,0.4), # (-0.1,0.0), # (0.0,0.0), # (0.3,1), # (0.6,0.5), # (0.6,0.1), # (1.3,0.6), # ]) complements = get_classes(fuzzy.complement) #print complements for name in sorted(objects): if name in ["Set", "Function", "Polygon", "Singleton"]: continue obj = objects[name] for name2 in sorted(complements): if name2 in ["Base", "Parametric"]: continue try: complement = complements[name2] plotComplement(complement, name2, obj, name) except: import traceback traceback.print_exc()
def __init__(self, base_path, image_shape=(320, 240, 3), batch_size=32): self.base_path = base_path self.training_data_path = os.path.join(base_path, "training_set") self.test_data_path = os.path.join(base_path, "testing_set") self.classes = get_classes(self.training_data_path) self.batch_size = batch_size self.image_shape = image_shape self.train_ds = tf.keras.preprocessing.image_dataset_from_directory( self.training_data_path, validation_split=0.2, subset="training", seed=345, image_size=(self.image_shape[0], self.image_shape[1]), batch_size=self.batch_size, label_mode='categorical', class_names=self.classes) self.val_ds = tf.keras.preprocessing.image_dataset_from_directory( self.training_data_path, validation_split=0.2, subset="validation", seed=345, image_size=(self.image_shape[0], self.image_shape[1]), batch_size=self.batch_size, label_mode='categorical', class_names=self.classes) self.test_ds = tf.keras.preprocessing.image_dataset_from_directory( self.test_data_path, seed=123, image_size=(self.image_shape[0], self.image_shape[1]), batch_size=self.batch_size, label_mode='categorical', class_names=self.classes) normalization_layer = tf.keras.layers.experimental.preprocessing.Rescaling( 1. / 255) self.train_ds = self.train_ds.map(lambda x, y: (normalization_layer(x), y)) self.val_ds = self.train_ds.map(lambda x, y: (normalization_layer(x), y)) self.test_ds = self.test_ds.map(lambda x, y: (normalization_layer(x), y)) # optimizing performances autotune = tf.data.experimental.AUTOTUNE self.train_ds = self.train_ds.cache().prefetch(buffer_size=autotune) self.val_ds = self.train_ds.cache().prefetch(buffer_size=autotune) self.test_ds = self.train_ds.cache().prefetch(buffer_size=autotune)
def get(self): user = users.get_current_user() classes = utils.get_classes(user) lessons, lessons_map = utils.get_lessons(user) logging.info(" Current LESSON" + self.request.get("lesson")) lesson_key=ndb.Key(urlsafe=self.request.get("lesson")) questions_for_lesson_query = Question.query(ancestor=lesson_key) template = jinja_env.get_template("templates/index.html") self.response.out.write(template.render({'lessons': lessons, 'user_email': user.email(), 'user_name': user.nickname(), 'classes': classes, 'logout_url': users.create_logout_url("/"), 'questions_for_lesson': questions_for_lesson_query}))
def main(data_file, sep): classes_column = "Hogwarts House" data, _ = read_data(data_file, sep) num_data = get_numerics(data, get_str=False) class_list = get_classes(data, classes_column) fig = plt.figure("Pair plot") # for each matter for j, key_a in enumerate(num_data.keys()): for i, key_b in enumerate(num_data.keys()): ax = fig.add_subplot(len(num_data), len(num_data), len(num_data) * j + i + 1) if j == len(num_data) - 1: ax.set_xlabel(key_b, rotation=45) if i == 0: ax.set_ylabel(key_a, rotation=45) plt.setp(ax.get_xticklabels(), visible=False) plt.setp(ax.get_yticklabels(), visible=False) ax.tick_params(axis='both', which='both', length=0) if key_b != key_a: classes = [] # for each class for c in class_list: c_tab_a = [] c_tab_b = [] # for each note for idx, val in enumerate(data): if val[classes_column] == c: c_tab_a.append(float(num_data[key_a][idx])) c_tab_b.append(float(num_data[key_b][idx])) classes.append((c_tab_a, c_tab_b)) for cat in classes: ax.scatter(cat[0], cat[1], alpha=0.5) else: # for each class classes = [] for c in class_list: c_tab = [] # for each note for idx, val in enumerate(num_data[key_a]): if idx in class_list[c]: c_tab.append(float(val)) classes.append(c_tab) for cat in classes: clean_cat = [] for elem in cat: if not math.isnan(elem): clean_cat.append(elem) ax.hist(clean_cat, alpha=0.5) plt.show(block=True)
def interactive(name): """interactive use: plot set of given name""" import fuzzy.set objects = get_classes(fuzzy.set) try: set = objects[name] except KeyError: print "%s is unknown." % name return g = getGnuplot() plotSet(set,name,gnuplot=g,interactive=True) g.close()
def ensemble_csvs(csv_files, weights, sub_file): print(csv_files) classes, _ = get_classes() results = [] for filename in csv_files: print(filename) df = pd.read_csv(filename) df = df[classes] results.append(df.values) outputs = np.average(results, axis=0, weights=weights) outputs = torch.from_numpy(outputs) _, preds = outputs.topk(3, 1, True, True) preds = preds.numpy() create_submission(preds, sub_file)
def interactive(name,params): """interactive use: plot norm using given params""" objects = get_classes(fuzzy.norm) try: norm = objects[name] except KeyError: print "%s is unknown." % name return g = getGnuplot() if len(params) > 0: plotNorm(norm,name,params,gnuplot=g,interactive=True) else: plotNorm(norm,name,gnuplot=g,interactive=True) g.close()
def interactive(name,params): """interactive use: plot complement using given params""" import fuzzy.complement objects = get_classes(fuzzy.complement) try: complement = objects[name] except KeyError: print "%s is unknown." % name return g = getGnuplot() from fuzzy.set.Triangle import Triangle set = Triangle() set_name = "Triangle" if len(params) > 0: plotComplement(complement,name,set,set_name,params,gnuplot=g,interactive=True) else: plotComplement(complement,name,set,set_name,gnuplot=g,interactive=True) g.close()
def profile(): classes = utils.get_classes(session["username"]) return render_template("profile.html", username=session['username'], classes=classes)
def test(): """test all found set classes with defuzzyfication method in specific kind of output variable class""" import types import fuzzy.set import fuzzy.defuzzify import fuzzy.OutputVariable import fuzzy.Adjective import fuzzy.set.Polygon # sizes of rows row1 = 10 row2 = 25 sets = get_classes(fuzzy.set) # Add tests of a special sets. sets["""~ _ _ / \ _ ___/ \/ \/ \___ """] = fuzzy.set.Polygon.Polygon([(-2,0),(-1.5,0.5),(-1.,0.5),(-0.5,0.0),(-0.25,1.),(0.25,1.),(0.5,0.0),(1.,0.5),(1.5,0.5),(2,0)]) sets["""~ ___ ___ / \/ """] = fuzzy.set.Polygon.Polygon([(-1,0.5),(0.,0.),(1.,1.)]) sets["""~ __ \ ___ \/ """] = fuzzy.set.Polygon.Polygon([(-1,1.0),(0.,0.),(1.,0.5)]) defuzzy = get_classes(fuzzy.defuzzify) for o in sorted(sets): set = sets[o] # filter out classes without default values if o in ["Set","Function","Polygon"]: continue print "Defuzzification of %s:" % o print "%-*s | %s" % (row1,"method","value") print "%s-+-%s" % ("-"*row1,"-"*row2) for d in sorted(defuzzy): defuzzy_ = defuzzy[d] # filter out abstract base classes if d in ["Base"]: continue v = fuzzy.OutputVariable.OutputVariable(defuzzify=defuzzy_) try: a = fuzzy.Adjective.Adjective(set) a.setMembership(1.0) v.adjectives["test"] = a result = v.getValue() if isinstance(result,types.FloatType): result = "%.3g" % result else: result = str(result) print "%-*s | %s" % (row1,d,result) except: print "%-*s | >>> %s <<<" % (row1,d,sys.exc_info()[1]) print