def main(epochs=5, learning_rate=0.01): # Avoid OMP error and allow multiple OpenMP runtime os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' warnings.filterwarnings("ignore") print(mlflow.__version__) # Download and untar the MNIST data set path = untar_data(URLs.MNIST_SAMPLE) # Prepare, transform, and normalize the data data = ImageDataBunch.from_folder(path, ds_tfms=(rand_pad(2, 28), []), bs=64) data.normalize(imagenet_stats) # Train and fit the Learner model learn = cnn_learner(data, models.resnet18, metrics=accuracy) # Start MLflow session with mlflow.start_run() as run: learn.fit(epochs, learning_rate) mlflow.fastai.log_model(learn, 'model') # fetch the logged model artifacts artifacts = [ f.path for f in MlflowClient().list_artifacts(run.info.run_id, 'model') ] print("artifacts: {}".format(artifacts))
def main(epochs=5, learning_rate=0.01): # Avoid OMP error and allow multiple OpenMP runtime os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' warnings.filterwarnings("ignore") print(mlflow.__version__) # Download and untar the MNIST data set path = untar_data(URLs.MNIST_SAMPLE) # Prepare, transform, and normalize the data data = ImageDataBunch.from_folder(path, ds_tfms=(rand_pad(2, 28), []), bs=64) data.normalize(imagenet_stats) # Create CNN the Learner model learn = cnn_learner(data, models.resnet18, metrics=accuracy) # Start MLflow session with mlflow.start_run() as run: learn.fit(epochs, learning_rate) mlflow.fastai.log_model(learn, "model") # load the model for scoring model_uri = "runs:/{}/model".format(run.info.run_id) loaded_model = mlflow.fastai.load_model(model_uri) predict_data = ... loaded_model.predict(predict_data)
def load(): path = Path('.') global model global learn global classes model = CONFIG['model_name'] # Check if we need to download Model file if CONFIG[model]['url'] != "": try: logging.info(f"Downloading model file from: {CONFIG[model]['url']}") urllib.request.urlretrieve(CONFIG[model]['url'], f"models/{model}.pth") logging.info(f"Downloaded model file and stored at path: models/{model}.pth") except HTTPError as e: logging.critical(f"Failed in downloading file from: {CONFIG[model]['url']}, Exception: '{e}'") sys.exit(4) init_data = ImageDataBunch.single_from_classes( path, CONFIG[model]['classes'], tfms=get_transforms(), size=CONFIG[model]['size'] ).normalize(imagenet_stats) classes = CONFIG[model]['classes'] logging.info(f"Loading model: {CONFIG['model_name']}, architecture: {CONFIG[model]['arch']}, file: models/{model}.pth") learn = create_cnn(init_data, eval(f"models.{CONFIG[model]['arch']}")) learn.load(model, device=CONFIG[model]['device']) # Create direcotry to get feedback for this model Path.mkdir(Path(path_to(FEEDBACK_DIR, model)), parents=True, exist_ok=True)
def main(epochs=5, learning_rate=0.01): # Avoid OMP error and allow multiple OpenMP runtime os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' warnings.filterwarnings("ignore") print(mlflow.__version__) # Download and untar the MNIST data set path = untar_data(URLs.MNIST_SAMPLE) # Prepare, transform, and normalize the data data = ImageDataBunch.from_folder(path, ds_tfms=(rand_pad(2, 28), []), bs=64) data.normalize(imagenet_stats) # Train and fit the Learner model learn = cnn_learner(data, models.resnet18, metrics=accuracy) # Start MLflow session with mlflow.start_run() as run: learn.fit(epochs, learning_rate) mlflow.fastai.log_model(learn, "model") # Fetch the default conda environment print("run_id: {}".format(run.info.run_id)) env = mlflow.fastai.get_default_conda_env() print("conda environment: {}".format(env))
async def setup_learner(): await download_file(model_file_url, path/'models'/f'{model_file_name}.pth') data_bunch = ImageDataBunch.single_from_classes(path, classes, tfms=get_transforms(), size=224).normalize(imagenet_stats) learn = create_cnn(data_bunch, models.resnet34, pretrained=False) learn.load(model_file_name) return learn
def load_dataset(*, sample_size=600, image_size=224) -> ImageDataBunch: labels = pd.read_csv(config.PROCESSED_DATA_DIR / 'labels_full.csv') classes = ['pneumonia', 'normal', 'COVID-19'] ds_types = ['train'] selected = [] for c in classes: for t in ds_types: selected.extend( labels[(labels.label == c) & (labels.ds_type == t)][:sample_size].values.tolist()) subset = pd.DataFrame(selected, columns=labels.columns) subset[['name', 'label']].to_csv(config.PROCESSED_DATA_DIR / 'labels.csv', index=False) tfms = _get_image_transformation() # will read from "labels.csv" in the data directory data = ImageDataBunch.from_csv(config.PROCESSED_DATA_DIR, ds_tfms=tfms, csv_labels=config.PROCESSED_DATA_DIR / 'labels.csv', valid_pct=0.1, seed=config.SEED, size=image_size, bs=21) data.normalize(imagenet_stats) return data
def create_data_bunch(bs, img_sz, train_sz=None, valid_sz=None, use_cache=False): train_ds = QuickDraw(PREPARED, train=True, take_subset=True, use_cache=use_cache, subset_size=train_sz) valid_ds = QuickDraw(PREPARED, train=False, take_subset=True, use_cache=use_cache, subset_size=valid_sz) bunch = ImageDataBunch.create(train_ds, valid_ds, bs=bs, size=img_sz, ds_tfms=get_transforms()) bunch.normalize(imagenet_stats) return bunch
def authenticate(): path = '/lib/Auth/RecFace/images/models/' root_models = [f for f in listdir(path) if isfile(join(path, f))] if 'tmp.pth' in root_models: root_models.remove('tmp.pth') classes = ["Test", "Train"] data = ImageDataBunch.single_from_classes('/lib/Auth/RecFace/images/', classes, ds_tfms=None, size = 224) data.normalize(imagenet_stats) learn = cnn_learner(data, models.vgg16_bn) imgs = getFaces.getFaces() if len(imgs)==0: return False for img in imgs: img = resize(img, (224,224), interpolation = INTER_AREA) imwrite('temp.jpeg', img) img = open_image('temp.jpeg') for mod in root_models: if compare(mod.split('.')[0], img, learn): return True return False
def get_databunch(): df = get_decorated_df() data = (ImageDataBunch.from_df(df, size=224) .random_split_by_pct() .transform()) return data
def get_data_from_path(path_image: Path): # tfms = get_transforms(do_flip=False) data = (ImageDataBunch.from_folder(path_image).random_split_by_pct(). label_from_func(get_float_labels).transform()) # data.normalize(imagenet_stats) return data
def getDataBunch(self, tfm = True, val_p = 0, sz = 224, b_sz = 8): if tfm: data = ImageDataBunch.from_folder('/lib/Auth/RecFace/images/', ds_tfms=get_transforms(do_flip=False), valid_pct=val_p, size=sz, bs=b_sz) else: data = ImageDataBunch.from_folder('/lib/Auth/RecFace/images/', ds_tfms=None, valid_pct=val_p, size=sz, bs=b_sz) return data
def create_data_bunch(path): np.random.seed(42) return ImageDataBunch.from_folder(path, train=".", valid_pct=0.2, ds_tfms=get_transforms(), size=224, num_workers=4).normalize(imagenet_stats)
def get_data(image_set='train', path='~/.fastai/data/stanford-cars', path_devkit='~/.fastai/data/devkit', normalization_type='default'): """ Returns an ImageDataBunch for the specified image_set Parameters ---------- image_set: str One of {'train', 'test'} path: PosixPath or str Image directory path path_devkit: PosixPath or str Devkit directory path normalization_type: str One of {'default', 'imagenet'} Returns ------- data: ImageDataBunch obj """ if type(path) == str: path = PosixPath(os.path.expanduser(path)) if type(path_devkit) == str: path_devkit = PosixPath(os.path.expanduser(path_devkit)) path_images = path / 'cars_{}_p'.format(image_set) path_annos = path_devkit / 'cars_{}_annos.mat'.format(image_set) path_labels_names = path_devkit / 'cars_meta.mat' annotations = scipy.io.loadmat(path_annos)['annotations'][0] label_names = scipy.io.loadmat(path_labels_names)['class_names'][0] images = [path_images / x[5][0] for x in annotations] labels = [label_names[int(x[4][0]) - 1][0] for x in annotations] if normalization_type == 'default': normalization_stats = [ torch.tensor([0.5, 0.5, 0.5]), torch.tensor([0.5, 0.5, 0.5]) ] elif normalization_type == 'imagenet': normalization_stats = imagenet_stats else: raise ValueError("invalid normalization_type provided") data = ImageDataBunch.from_lists(path_images, fnames=images, labels=labels, ds_tfms=get_transforms(), size=331, bs=8).normalize(normalization_stats) return data
async def setup_learner(): await download_file(model_file_url, path / 'models' / f'{model_file_name}.pth') data_bunch = ImageDataBunch.single_from_classes( path, classes, tfms=get_transforms(), size=224).normalize(imagenet_stats) learn = create_cnn(data_bunch, models.resnet34, pretrained=False) learn.load(model_file_name) return learn
def setup_data_loader() -> str: from fastai.vision import ImageDataBunch import dill import codecs bunch = ImageDataBunch.from_folder("test/images", train="training", valid="test", size=112) return codecs.encode(dill.dumps(bunch), "base64").decode()
def main(): # Set data transformations if args.ds_tfms: ds_tfms = get_transforms(do_flip=True, flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.1) else: ds_tfms = None # Load data data = ImageDataBunch.from_folder(args.data_path, train=args.train_dirname, valid=args.test_dirname, ds_tfms=ds_tfms, size=args.image_size, bs=args.batch_size).normalize() if args.verbose: print(len(data.valid_ds), "test images") # Get tile IDS, image IDs and image targets tile_ids, image_ids, image_targets = get_tile_filename_info(data) # Compute tile prediction scores using trained model tile_scores, tile_targets = predict_tiles(data, model_type[args.model], args.model_path, args.ps, args.wd, mixup=args.mixup) # Make tile predictions using prediction scores tile_preds = torch.argmax(tile_scores, 1) # Create output directory if it does not exist if not os.path.isdir(args.output_path): os.makedirs(args.output_path, exist_ok=True) # Create numpy array holding the tile output tile_output = tile_output_to_array(tile_ids, image_ids, tile_scores, tile_preds, tile_targets, image_targets) # Save tile IDs, image IDs, tile prediction scores, tile predictions and tile true labels to file np.savetxt( os.path.join(args.output_path, args.output_filename), tile_output, fmt=['%s', '%s', '%f', '%d', '%d', '%d'], delimiter=',', newline='\n', header='tile_id,image_id,tile_score,tile_pred,tile_target,image_target', comments='') if args.verbose: print("Completed execution.")
def bearsInference(): #Get querystring to figure out which model to load ic=request.args.get('imageclassifier') print(ic) classes=[] path=None if ic=='KDEF': path=os.path.join(HERE, "tmp/KDEF") classes = ['afraid', 'angry', 'disgusted', 'happy', 'neutral', 'sad', 'surprised'] elif ic=='teddys': path=os.path.join(HERE, "tmp/bears") classes = ['black', 'grizzly', 'teddys'] learn = create_cnn(ImageDataBunch.single_from_classes(path, classes, tfms=get_transforms(), size=224).normalize(imagenet_stats), models.resnet34) learn.load('stage-2') fp = request.files['file'] #img=open_image(bearspath + '/models/00000014.jpg') # Read EXIF data exifData={} imgTemp=Image.open(fp) exifDataRaw=imgTemp._getexif() angle=0 if not exifDataRaw==None: for orientation in ExifTags.TAGS.keys(): if ExifTags.TAGS[orientation]=='Orientation': break exif=dict(exifDataRaw.items()) # print(exif[orientation]) if exif[orientation] == 3: angle=180 elif exif[orientation] == 6: angle=270 elif exif[orientation] == 8: angle=90 img=open_image(fp) rotate(img,angle) pred_class,pred_idx,outputs = learn.predict(img) img_data = encode(img) body = { 'label': str(pred_class), 'image': img_data } resp= Response(response=json.dumps({"response": body}), status=200, mimetype='application/json') #print (str(pred_class)) return resp
def train(imgs_root, model_dir): my_tfms = get_transforms() print(f"Transforms on Train set: {my_tfms[0]}") print(f"Transforms on Validation set: {my_tfms[1]}") np.random.seed(42) ### '/home/user/tmp/pycharm_project_310/1_detectron2/Furniture-Style-Classifier-master/Data' # imgs_root = '/home/user/tmp/pycharm_project_310/1_detectron2/Furniture-Style-Classifier-master/MyData' data = ImageDataBunch.from_folder(path=Path(imgs_root), train=".", valid_pct=0.2, ds_tfms=my_tfms, size=224, num_workers=4, bs=64).normalize(imagenet_stats) print(f"BatchSize: {data.batch_size}") print(f"Train Dataset size: {len(data.train_ds)}") print(f"Validataion Dataset size: {len(data.valid_ds)}") print(f"Classes: {data.classes}") print(f"Number of Classes : {data.c}") num_epochs = 5 lr = 4.37E-03 learn = cnn_learner(data, models.resnet34, metrics=error_rate, pretrained=True, true_wd=False, train_bn=False) learn.fit(epochs=num_epochs, lr=lr) #### 模型评估 report = learn.interpret() matrix = report.confusion_matrix().tolist() print("confusion_matrix:\n{}".format(matrix)) learn.model = learn.model.cpu() ### 转化为cpu模型 # model_dir = os.path.join(os.getcwd(),"./models") # model_dir = '/home/user/tmp/pycharm_project_310/1_detectron2/ImageDetectionAPI/image_style_classifier/models/' weight_path = os.path.join(model_dir, 'resnet34_scene_detection') inference_path = os.path.join(model_dir, 'export.pkl') learn.save(weight_path) learn.export(file=Path(inference_path)) if os.path.exists(inference_path): print("model save to :{}".format(model_dir))
def main(epochs): Task.init(project_name="examples", task_name="fastai v1") path = untar_data(URLs.MNIST_SAMPLE) data = ImageDataBunch.from_folder(path, ds_tfms=(rand_pad(2, 28), []), bs=64, num_workers=0) data.normalize(imagenet_stats) learn = cnn_learner(data, models.resnet18, metrics=accuracy) accuracy(*learn.get_preds()) learn.fit_one_cycle(epochs, 0.01)
def export_to_fastAI(self, imagedatabunch_args={}): # https://docs.fast.ai/vision.data.html#ImageDataBunch.from_df # from fastai.vision import ImageDataBunch # data = ImageDataBunch.from_df(path, df, ds_tfms=tfms, size=24) # return data try: from fastai.vision import ImageDataBunch, ImageList from fastai.vision.transform import get_transforms except ModuleNotFoundError as e: ERROR['missing_module']('fastai') df = self.filesnames_to_ml_df(withAbsPath=False) fastai_idb = ImageDataBunch.from_df(path='', df=df, **imagedatabunch_args) return fastai_idb
def load_w(self, location): self.location = location # res = models.resnet50 #hyperparameters batch_size = 10 epochs = 50 save_loc = 'resnet50model_trainedonHerlevsetandSipakmed' + str( epochs) + "batch" + str(batch_size) #location to save the model ## Declaring path of dataset path_img = Path("/cluster/home/cantoniou/deep_project/sipakmedFormat") ## Loading data data = ImageDataBunch.from_folder(path=path_img, train='train', valid='val', ds_tfms=get_transforms(), size=224, bs=batch_size) #, check_ext=False) ## Normalizing data based on Image net parameters #data.normalize(imagenet_stats) #normalize now according to the batch data and not imagenet data.normalize() #defaults to batch 'stats' print(data.classes) len(data.classes), data.c #LOAD THE TRANSFER LEARNING MODEL ## To create a ResNET 50 with pretrained weights based on the new dataset trans_model = cnn_learner( data, models.resnet50, metrics=[accuracy, FBeta(average="weighted")]) #print(trans_model) #check the architecture of the loaded model to make sure it was loaded with a head of 5 ---to match the data classes trans_model = trans_model.load( location ) #load the previous pretrained model weights form the harlev dataset print("Start training") #find best learning rate trans_model.lr_find() # Train the model trans_model.fit_one_cycle(epochs, callbacks=[ SaveModelCallback(trans_model, every='improvement', mode='max', monitor='accuracy', name=save_loc) ])
def train_classifier(model, config, x_train, y_train, x_val, y_val, train_tfms=None): loss_func = torch.nn.CrossEntropyLoss() if train_tfms is None: train_tfms = [] train_ds = ImageArrayDS(x_train, y_train, train_tfms) val_ds = ImageArrayDS(x_val, y_val) data = ImageDataBunch.create(train_ds, val_ds, bs=config['batch_size']) callbacks = [partial(EarlyStoppingCallback, min_delta=1e-3, patience=config['patience'])] learner = Learner(data, model, metrics=accuracy, loss_func=loss_func, callback_fns=callbacks) learner.fit(config['epochs'], config['start_lr'], wd=config['weight_decay']) return learner
def load_data(): dataset = build_dataset('cifar_10', val_size=val_size) x_train, y_train = dataset.dataset('train') x_val, y_val = dataset.dataset('val') shape = (-1, 3, 32, 32) x_train = ((x_train - 128) / 128).reshape(shape) x_val = ((x_val - 128) / 128).reshape(shape) train_tfms = [*rand_pad(4, 32), flip_lr(p=0.5)] train_ds = ImageArrayDS(x_train, y_train, train_tfms) val_ds = ImageArrayDS(x_val, y_val) data = ImageDataBunch.create(train_ds, val_ds, bs=256) return data, x_train, y_train, x_val, y_val
def search(bot, update): """Send reply of user's message.""" photo_file = bot.get_file(update.message.photo[-1].file_id) photo_file.download('testing.jpeg') try: bs = 32 path = "classes" np.random.seed(42) data = ImageDataBunch.from_folder( path, train='.', valid_pct=0.2, ds_tfms=get_transforms(), size=224, num_workers=4).normalize(imagenet_stats) learn = cnn_learner(data, models.resnet34, metrics=error_rate).load("stage-1") learn.export() learn = load_learner("classes") cat, tensor, probs = learn.predict(open_image("testing.jpeg")) l = list(probs) a = tensor.__str__() a = int(a.strip("tensor" "()")) l = list(probs)[a] l = l.__str__() b = float(l.strip("tensor" "()")) if b >= 0.9: update.message.reply_text( '`' + str(cat) + '`', parse_mode=ParseMode.MARKDOWN, reply_to_message_id=update.message.message_id) # print("prediction :") # print(cat) else: cat = "sry I am not sure " update.message.reply_text( '`' + str(cat) + '`', parse_mode=ParseMode.MARKDOWN, reply_to_message_id=update.message.message_id) # print("prediction :") # print("Not Sure") except Exception as e: update.message.reply_text(e)
def main(): # Set data transformations if args.ds_tfms: ds_tfms = get_transforms(do_flip=True, flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.1) else: ds_tfms = None # Load data data = ImageDataBunch.from_folder(args.data_path, train=args.train_dirname, valid=args.validation_dirname, test=args.test_dirname, ds_tfms=ds_tfms, size=args.image_size, bs=args.batch_size).normalize() if args.verbose: print(len(data.train_ds), "training images", len(data.valid_ds), "validation images and", len(data.test_ds), "test images") # Explore optimal learning rates lrs, losses = tune_lr(data, model_dict[args.model], args.ps, args.wd, args.start_lr, args.end_lr, args.num_lr_iters, pretrained=args.pretrained, mixup=args.mixup) # Create output directory if it does not exist if not os.path.isdir(args.output_path): os.makedirs(args.output_path, exist_ok=True) # Save learning rates and associated loss values np.savetxt(os.path.join(args.output_path, args.output_filename), np.column_stack((lrs, losses)), delimiter=',', newline='\n', header='lrs,losses', comments='') if args.verbose: print("Completed execution.")
def main(): # Load data data = ImageDataBunch.from_folder( args.data_path, train=args.train_dirname, valid=args.test_dirname, ds_tfms=get_transforms(do_flip=True, flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.1), size=args.image_size, bs=args.batch_size ).normalize() if args.verbose: print(len(data.valid_ds), "test images") # Compute F1, TPR and FPR for different thresholds and find optimal threshold optimal_thres, optimal_thres_idx, thres_metrics = tune_thres( data, model_type[args.model], args.model_path, args.ps, args.thres, tpr_lb=args.tpr_lb, fpr_ub=args.fpr_ub, verbose=args.verbose ) # Save optimal threshold and associated F1 score, TPF and FPR to file np.savetxt( os.path.join(args.output_path, args.output_optimal_thres_filename), [[ optimal_thres, thres_metrics['f1'][optimal_thres_idx], thres_metrics['tpr'][optimal_thres_idx], thres_metrics['fpr'][optimal_thres_idx] ]], fmt=['%f', '%f', '%f', '%f'], delimiter=',', newline='\n', header='thres,f1,tpr,fpr', comments='' ) # Save thresholds, F1 score, TPR and FPR to file np.savetxt( os.path.join(args.output_path, args.output_thres_metrics_filename), thres_metrics, fmt=['%f', '%f', '%f', '%f'], delimiter=',', newline='\n', header='thres,f1,tpr,fpr', comments='' ) if args.verbose: print("Completed execution.")
def fine_tune_convnet(path: str): """ Path must be in the format of a FastAI data bunch, ie Train/Test/Valid dirs and then one subdirectory per class in each of those dirs. """ tfms = get_transforms(do_flip=False) data = ImageDataBunch.from_folder(path, ds_tfms=tfms, size=64) learn = cnn_learner(data, models.resnet34, metrics=error_rate) learn.fit_one_cycle(5) learn.save('stage-1') learn.unfreeze() learn.fit_one_cycle(1, max_lr=slice(1e-6, 1e-3)) learn.save('stage-2') learn.export(file="two-stage-model.pkl")
def main(): # Load data data = ImageDataBunch.from_folder(args.data_path, train=args.train_dirname, valid=args.validation_dirname, test=args.test_dirname, ds_tfms=get_transforms(do_flip=True, flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.1), size=args.image_size, bs=args.batch_size).normalize() if args.verbose: print(len(data.train_ds), "training images", len(data.valid_ds), "validation images and", len(data.test_ds), "test images") # Create output directory if it does not exist if not Path(args.output_path).exists(): Path(args.output_path).mkdir(parents=True, exist_ok=True) # Fit model using one-cycle policy recorder = fit_model(data, model_dict[args.model], args.pretrained, args.ps, args.cyc_len, args.lr_lower, args.lr_upper, args.output_path, args.output_model_filename, args.verbose) # Save loss value for each processed batch to file if args.save_loss: np.savetxt(Path(args.output_path).joinpath( args.output_training_loss_filename), [l.item() for l in recorder.losses], newline='\n', header='losses', comments='') # Save training and validation losses per epoch np.savetxt(Path(args.output_path).joinpath(args.output_loss_filename), all_losses_to_array(get_all_losses_per_epoch(recorder)), fmt=['%d', '%f', '%f'], delimiter=',', newline='\n', header='nb_batches,train_losses,val_losses', comments='') if args.verbose: print("Completed execution.")
def __init__(self, picture_file): self.classes = [ 'Afghan_hound', 'African_hunting_dog', 'Airedale', 'American_Staffordshire_terrier', 'Appenzeller', 'Australian_terrier', 'Bedlington_terrier', 'Bernese_mountain_dog', 'Blenheim_spaniel', 'Border_collie', 'Border_terrier', 'Boston_bull', 'Bouvier_des_Flandres', 'Brabancon_griffon', 'Brittany_spaniel', 'Cardigan', 'Chesapeake_Bay_retriever', 'Chihuahua', 'Dandie_Dinmont', 'Doberman', 'English_foxhound', 'English_setter', 'English_springer', 'EntleBucher', 'Eskimo_dog', 'French_bulldog', 'German_shepherd', 'German_short-haired_pointer', 'Gordon_setter', 'Great_Dane', 'Great_Pyrenees', 'Greater_Swiss_Mountain_dog', 'Ibizan_hound', 'Irish_setter', 'Irish_terrier', 'Irish_water_spaniel', 'Irish_wolfhound', 'Italian_greyhound', 'Japanese_spaniel', 'Kerry_blue_terrier', 'Labrador_retriever', 'Lakeland_terrier', 'Leonberg', 'Lhasa', 'Maltese_dog', 'Mexican_hairless', 'Newfoundland', 'Norfolk_terrier', 'Norwegian_elkhound', 'Norwich_terrier', 'Old_English_sheepdog', 'Pekinese', 'Pembroke', 'Pomeranian', 'Rhodesian_ridgeback', 'Rottweiler', 'Saint_Bernard', 'Saluki', 'Samoyed', 'Scotch_terrier', 'Scottish_deerhound', 'Sealyham_terrier', 'Shetland_sheepdog', 'Shih-Tzu', 'Siberian_husky', 'Staffordshire_bullterrier', 'Sussex_spaniel', 'Tibetan_mastiff', 'Tibetan_terrier', 'Walker_hound', 'Weimaraner', 'Welsh_springer_spaniel', 'West_Highland_white_terrier', 'Yorkshire_terrier', 'affenpinscher', 'basenji', 'basset', 'beagle', 'black-and', 'bloodhound', 'bluetick', 'borzoi', 'boxer', 'briard', 'bull_mastiff', 'cairn', 'chow', 'clumber', 'cocker_spaniel', 'collie', 'curly-coated_retriever', 'dhole', 'dingo', 'flat-coated_retriever', 'giant_schnauzer', 'golden_retriever', 'groenendael', 'keeshond', 'kelpie', 'komondor', 'kuvasz', 'malamute', 'malinois', 'miniature_pinscher', 'miniature_poodle', 'miniature_schnauzer', 'otterhound', 'papillon', 'pug', 'redbone', 'schipperke', 'silky_terrier', 'soft-coated_wheaten_terrier', 'standard_poodle', 'standard_schnauzer', 'toy_poodle', 'toy_terrier', 'vizsla', 'whippet', 'wire-haired_fox_terrier' ] f_name, f_ext = os.path.splitext(picture_file) img_path = os.path.join(app.root_path, 'static', 'images', f_name + f_ext) self.img = open_image(img_path) self.data = ImageDataBunch.single_from_classes( "./", self.classes, ds_tfms=get_transforms(), size=224).normalize(imagenet_stats) self.learner = cnn_learner(self.data, models.resnet50) self.learner.load('stage-2-rerun')
def train_model(model_name, dataset_name, arch, img_size, epochs): data = ImageDataBunch.from_folder(f'{BASE_DIR}datasets/{dataset_name}', valid_pct=0.2, ds_tfms=get_transforms(), size=img_size, num_workers=6, bs=BATCH_SIZE) arch = model_mapping[arch] learner = cnn_learner(data, arch, metrics=[error_rate, accuracy]) learner.fit_one_cycle(epochs) learner.export( os.path.join(os.getcwd(), '..', 'static', 'models', model_name + '.pkl')) meta = {} meta['metrics'] = [[i.item() for i in e] for e in learner.recorder.metrics] meta['loss'] = [i.item() for i in learner.recorder.losses] meta['lr'] = [i.item() for i in learner.recorder.lrs] return meta
def test_freeze_unfreeze_effnet(): this_tests(cnn_learner) def get_number_of_trainable_params(model: nn.Module): return sum(p.numel() for p in model.parameters() if p.requires_grad) base_arch = EfficientNetB1 path = untar_data(URLs.MNIST_TINY) data = ImageDataBunch.from_folder(path, size=64) data.c = 1000 # Set number of class to be 1000 to stay in line with the pretrained model. cnn_learn = cnn_learner(data, base_arch, pretrained=True) ref_learn = Learner(data, EfficientNet.from_pretrained("efficientnet-b1")) # By default the neural net in cnn learner is freezed. assert get_number_of_trainable_params( cnn_learn.model) < get_number_of_trainable_params(ref_learn.model) cnn_learn.unfreeze() assert get_number_of_trainable_params( cnn_learn.model) == get_number_of_trainable_params(ref_learn.model)