def main(epochs=5, learning_rate=0.01):
# Avoid OMP error and allow multiple OpenMP runtime
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
warnings.filterwarnings("ignore")
print(mlflow.__version__)
# Download and untar the MNIST data set
path = untar_data(URLs.MNIST_SAMPLE)
# Prepare, transform, and normalize the data
data = ImageDataBunch.from_folder(path,
ds_tfms=(rand_pad(2, 28), []),
bs=64)
data.normalize(imagenet_stats)
# Train and fit the Learner model
learn = cnn_learner(data, models.resnet18, metrics=accuracy)
# Start MLflow session
with mlflow.start_run() as run:
learn.fit(epochs, learning_rate)
mlflow.fastai.log_model(learn, 'model')
# fetch the logged model artifacts
artifacts = [
f.path for f in MlflowClient().list_artifacts(run.info.run_id, 'model')
]
print("artifacts: {}".format(artifacts))
def main(epochs=5, learning_rate=0.01):
# Avoid OMP error and allow multiple OpenMP runtime
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
warnings.filterwarnings("ignore")
print(mlflow.__version__)
# Download and untar the MNIST data set
path = untar_data(URLs.MNIST_SAMPLE)
# Prepare, transform, and normalize the data
data = ImageDataBunch.from_folder(path,
ds_tfms=(rand_pad(2, 28), []),
bs=64)
data.normalize(imagenet_stats)
# Create CNN the Learner model
learn = cnn_learner(data, models.resnet18, metrics=accuracy)
# Start MLflow session
with mlflow.start_run() as run:
learn.fit(epochs, learning_rate)
mlflow.fastai.log_model(learn, "model")
# load the model for scoring
model_uri = "runs:/{}/model".format(run.info.run_id)
loaded_model = mlflow.fastai.load_model(model_uri)
predict_data = ...
loaded_model.predict(predict_data)
def load():
path = Path('.')
global model
global learn
global classes
model = CONFIG['model_name']
# Check if we need to download Model file
if CONFIG[model]['url'] != "":
try:
logging.info(f"Downloading model file from: {CONFIG[model]['url']}")
urllib.request.urlretrieve(CONFIG[model]['url'], f"models/{model}.pth")
logging.info(f"Downloaded model file and stored at path: models/{model}.pth")
except HTTPError as e:
logging.critical(f"Failed in downloading file from: {CONFIG[model]['url']}, Exception: '{e}'")
sys.exit(4)
init_data = ImageDataBunch.single_from_classes(
path, CONFIG[model]['classes'], tfms=get_transforms(),
size=CONFIG[model]['size']
).normalize(imagenet_stats)
classes = CONFIG[model]['classes']
logging.info(f"Loading model: {CONFIG['model_name']}, architecture: {CONFIG[model]['arch']}, file: models/{model}.pth")
learn = create_cnn(init_data, eval(f"models.{CONFIG[model]['arch']}"))
learn.load(model, device=CONFIG[model]['device'])
# Create direcotry to get feedback for this model
Path.mkdir(Path(path_to(FEEDBACK_DIR, model)), parents=True, exist_ok=True)
def main(epochs=5, learning_rate=0.01):
# Avoid OMP error and allow multiple OpenMP runtime
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
warnings.filterwarnings("ignore")
print(mlflow.__version__)
# Download and untar the MNIST data set
path = untar_data(URLs.MNIST_SAMPLE)
# Prepare, transform, and normalize the data
data = ImageDataBunch.from_folder(path,
ds_tfms=(rand_pad(2, 28), []),
bs=64)
data.normalize(imagenet_stats)
# Train and fit the Learner model
learn = cnn_learner(data, models.resnet18, metrics=accuracy)
# Start MLflow session
with mlflow.start_run() as run:
learn.fit(epochs, learning_rate)
mlflow.fastai.log_model(learn, "model")
# Fetch the default conda environment
print("run_id: {}".format(run.info.run_id))
env = mlflow.fastai.get_default_conda_env()
print("conda environment: {}".format(env))
async def setup_learner():
await download_file(model_file_url, path/'models'/f'{model_file_name}.pth')
data_bunch = ImageDataBunch.single_from_classes(path, classes,
tfms=get_transforms(), size=224).normalize(imagenet_stats)
learn = create_cnn(data_bunch, models.resnet34, pretrained=False)
learn.load(model_file_name)
return learn
def load_dataset(*, sample_size=600, image_size=224) -> ImageDataBunch:
labels = pd.read_csv(config.PROCESSED_DATA_DIR / 'labels_full.csv')
classes = ['pneumonia', 'normal', 'COVID-19']
ds_types = ['train']
selected = []
for c in classes:
for t in ds_types:
selected.extend(
labels[(labels.label == c)
& (labels.ds_type == t)][:sample_size].values.tolist())
subset = pd.DataFrame(selected, columns=labels.columns)
subset[['name', 'label']].to_csv(config.PROCESSED_DATA_DIR / 'labels.csv',
index=False)
tfms = _get_image_transformation()
# will read from "labels.csv" in the data directory
data = ImageDataBunch.from_csv(config.PROCESSED_DATA_DIR,
ds_tfms=tfms,
csv_labels=config.PROCESSED_DATA_DIR /
'labels.csv',
valid_pct=0.1,
seed=config.SEED,
size=image_size,
bs=21)
data.normalize(imagenet_stats)
return data
def create_data_bunch(bs,
img_sz,
train_sz=None,
valid_sz=None,
use_cache=False):
train_ds = QuickDraw(PREPARED,
train=True,
take_subset=True,
use_cache=use_cache,
subset_size=train_sz)
valid_ds = QuickDraw(PREPARED,
train=False,
take_subset=True,
use_cache=use_cache,
subset_size=valid_sz)
bunch = ImageDataBunch.create(train_ds,
valid_ds,
bs=bs,
size=img_sz,
ds_tfms=get_transforms())
bunch.normalize(imagenet_stats)
return bunch
def authenticate():
path = '/lib/Auth/RecFace/images/models/'
root_models = [f for f in listdir(path) if isfile(join(path, f))]
if 'tmp.pth' in root_models:
root_models.remove('tmp.pth')
classes = ["Test", "Train"]
data = ImageDataBunch.single_from_classes('/lib/Auth/RecFace/images/',
classes,
ds_tfms=None,
size = 224)
data.normalize(imagenet_stats)
learn = cnn_learner(data, models.vgg16_bn)
imgs = getFaces.getFaces()
if len(imgs)==0:
return False
for img in imgs:
img = resize(img, (224,224), interpolation = INTER_AREA)
imwrite('temp.jpeg', img)
img = open_image('temp.jpeg')
for mod in root_models:
if compare(mod.split('.')[0], img, learn):
return True
return False
def get_databunch():
df = get_decorated_df()
data = (ImageDataBunch.from_df(df, size=224)
.random_split_by_pct()
.transform())
return data
def get_data_from_path(path_image: Path):
# tfms = get_transforms(do_flip=False)
data = (ImageDataBunch.from_folder(path_image).random_split_by_pct().
label_from_func(get_float_labels).transform())
# data.normalize(imagenet_stats)
return data
def getDataBunch(self, tfm = True, val_p = 0, sz = 224, b_sz = 8):
if tfm:
data = ImageDataBunch.from_folder('/lib/Auth/RecFace/images/',
ds_tfms=get_transforms(do_flip=False),
valid_pct=val_p,
size=sz,
bs=b_sz)
else:
data = ImageDataBunch.from_folder('/lib/Auth/RecFace/images/',
ds_tfms=None,
valid_pct=val_p,
size=sz,
bs=b_sz)
return data
def create_data_bunch(path):
np.random.seed(42)
return ImageDataBunch.from_folder(path,
train=".",
valid_pct=0.2,
ds_tfms=get_transforms(),
size=224,
num_workers=4).normalize(imagenet_stats)
def get_data(image_set='train',
path='~/.fastai/data/stanford-cars',
path_devkit='~/.fastai/data/devkit',
normalization_type='default'):
"""
Returns an ImageDataBunch for the specified image_set
Parameters
----------
image_set: str
One of {'train', 'test'}
path: PosixPath or str
Image directory path
path_devkit: PosixPath or str
Devkit directory path
normalization_type: str
One of {'default', 'imagenet'}
Returns
-------
data: ImageDataBunch obj
"""
if type(path) == str:
path = PosixPath(os.path.expanduser(path))
if type(path_devkit) == str:
path_devkit = PosixPath(os.path.expanduser(path_devkit))
path_images = path / 'cars_{}_p'.format(image_set)
path_annos = path_devkit / 'cars_{}_annos.mat'.format(image_set)
path_labels_names = path_devkit / 'cars_meta.mat'
annotations = scipy.io.loadmat(path_annos)['annotations'][0]
label_names = scipy.io.loadmat(path_labels_names)['class_names'][0]
images = [path_images / x[5][0] for x in annotations]
labels = [label_names[int(x[4][0]) - 1][0] for x in annotations]
if normalization_type == 'default':
normalization_stats = [
torch.tensor([0.5, 0.5, 0.5]),
torch.tensor([0.5, 0.5, 0.5])
]
elif normalization_type == 'imagenet':
normalization_stats = imagenet_stats
else:
raise ValueError("invalid normalization_type provided")
data = ImageDataBunch.from_lists(path_images,
fnames=images,
labels=labels,
ds_tfms=get_transforms(),
size=331,
bs=8).normalize(normalization_stats)
return data
async def setup_learner():
await download_file(model_file_url,
path / 'models' / f'{model_file_name}.pth')
data_bunch = ImageDataBunch.single_from_classes(
path, classes, tfms=get_transforms(),
size=224).normalize(imagenet_stats)
learn = create_cnn(data_bunch, models.resnet34, pretrained=False)
learn.load(model_file_name)
return learn
def setup_data_loader() -> str:
from fastai.vision import ImageDataBunch
import dill
import codecs
bunch = ImageDataBunch.from_folder("test/images",
train="training",
valid="test",
size=112)
return codecs.encode(dill.dumps(bunch), "base64").decode()
def main():
# Set data transformations
if args.ds_tfms:
ds_tfms = get_transforms(do_flip=True,
flip_vert=True,
max_lighting=0.1,
max_zoom=1.05,
max_warp=0.1)
else:
ds_tfms = None
# Load data
data = ImageDataBunch.from_folder(args.data_path,
train=args.train_dirname,
valid=args.test_dirname,
ds_tfms=ds_tfms,
size=args.image_size,
bs=args.batch_size).normalize()
if args.verbose:
print(len(data.valid_ds), "test images")
# Get tile IDS, image IDs and image targets
tile_ids, image_ids, image_targets = get_tile_filename_info(data)
# Compute tile prediction scores using trained model
tile_scores, tile_targets = predict_tiles(data,
model_type[args.model],
args.model_path,
args.ps,
args.wd,
mixup=args.mixup)
# Make tile predictions using prediction scores
tile_preds = torch.argmax(tile_scores, 1)
# Create output directory if it does not exist
if not os.path.isdir(args.output_path):
os.makedirs(args.output_path, exist_ok=True)
# Create numpy array holding the tile output
tile_output = tile_output_to_array(tile_ids, image_ids, tile_scores,
tile_preds, tile_targets, image_targets)
# Save tile IDs, image IDs, tile prediction scores, tile predictions and tile true labels to file
np.savetxt(
os.path.join(args.output_path, args.output_filename),
tile_output,
fmt=['%s', '%s', '%f', '%d', '%d', '%d'],
delimiter=',',
newline='\n',
header='tile_id,image_id,tile_score,tile_pred,tile_target,image_target',
comments='')
if args.verbose:
print("Completed execution.")
def bearsInference():
#Get querystring to figure out which model to load
ic=request.args.get('imageclassifier')
print(ic)
classes=[]
path=None
if ic=='KDEF':
path=os.path.join(HERE, "tmp/KDEF")
classes = ['afraid', 'angry', 'disgusted', 'happy', 'neutral', 'sad', 'surprised']
elif ic=='teddys':
path=os.path.join(HERE, "tmp/bears")
classes = ['black', 'grizzly', 'teddys']
learn = create_cnn(ImageDataBunch.single_from_classes(path, classes, tfms=get_transforms(), size=224).normalize(imagenet_stats), models.resnet34)
learn.load('stage-2')
fp = request.files['file']
#img=open_image(bearspath + '/models/00000014.jpg')
# Read EXIF data
exifData={}
imgTemp=Image.open(fp)
exifDataRaw=imgTemp._getexif()
angle=0
if not exifDataRaw==None:
for orientation in ExifTags.TAGS.keys():
if ExifTags.TAGS[orientation]=='Orientation':
break
exif=dict(exifDataRaw.items())
# print(exif[orientation])
if exif[orientation] == 3:
angle=180
elif exif[orientation] == 6:
angle=270
elif exif[orientation] == 8:
angle=90
img=open_image(fp)
rotate(img,angle)
pred_class,pred_idx,outputs = learn.predict(img)
img_data = encode(img)
body = { 'label': str(pred_class), 'image': img_data }
resp= Response(response=json.dumps({"response": body}), status=200, mimetype='application/json')
#print (str(pred_class))
return resp
def train(imgs_root, model_dir):
my_tfms = get_transforms()
print(f"Transforms on Train set: {my_tfms[0]}")
print(f"Transforms on Validation set: {my_tfms[1]}")
np.random.seed(42)
### '/home/user/tmp/pycharm_project_310/1_detectron2/Furniture-Style-Classifier-master/Data'
# imgs_root = '/home/user/tmp/pycharm_project_310/1_detectron2/Furniture-Style-Classifier-master/MyData'
data = ImageDataBunch.from_folder(path=Path(imgs_root),
train=".",
valid_pct=0.2,
ds_tfms=my_tfms,
size=224,
num_workers=4,
bs=64).normalize(imagenet_stats)
print(f"BatchSize: {data.batch_size}")
print(f"Train Dataset size: {len(data.train_ds)}")
print(f"Validataion Dataset size: {len(data.valid_ds)}")
print(f"Classes: {data.classes}")
print(f"Number of Classes : {data.c}")
num_epochs = 5
lr = 4.37E-03
learn = cnn_learner(data,
models.resnet34,
metrics=error_rate,
pretrained=True,
true_wd=False,
train_bn=False)
learn.fit(epochs=num_epochs, lr=lr)
#### 模型评估
report = learn.interpret()
matrix = report.confusion_matrix().tolist()
print("confusion_matrix:\n{}".format(matrix))
learn.model = learn.model.cpu() ### 转化为cpu模型
# model_dir = os.path.join(os.getcwd(),"./models")
# model_dir = '/home/user/tmp/pycharm_project_310/1_detectron2/ImageDetectionAPI/image_style_classifier/models/'
weight_path = os.path.join(model_dir, 'resnet34_scene_detection')
inference_path = os.path.join(model_dir, 'export.pkl')
learn.save(weight_path)
learn.export(file=Path(inference_path))
if os.path.exists(inference_path):
print("model save to :{}".format(model_dir))
def main(epochs):
Task.init(project_name="examples", task_name="fastai v1")
path = untar_data(URLs.MNIST_SAMPLE)
data = ImageDataBunch.from_folder(path, ds_tfms=(rand_pad(2, 28), []), bs=64, num_workers=0)
data.normalize(imagenet_stats)
learn = cnn_learner(data, models.resnet18, metrics=accuracy)
accuracy(*learn.get_preds())
learn.fit_one_cycle(epochs, 0.01)
def export_to_fastAI(self, imagedatabunch_args={}):
# https://docs.fast.ai/vision.data.html#ImageDataBunch.from_df
# from fastai.vision import ImageDataBunch
# data = ImageDataBunch.from_df(path, df, ds_tfms=tfms, size=24)
# return data
try:
from fastai.vision import ImageDataBunch, ImageList
from fastai.vision.transform import get_transforms
except ModuleNotFoundError as e:
ERROR['missing_module']('fastai')
df = self.filesnames_to_ml_df(withAbsPath=False)
fastai_idb = ImageDataBunch.from_df(path='', df=df, **imagedatabunch_args)
return fastai_idb
def load_w(self, location):
self.location = location
# res = models.resnet50
#hyperparameters
batch_size = 10
epochs = 50
save_loc = 'resnet50model_trainedonHerlevsetandSipakmed' + str(
epochs) + "batch" + str(batch_size) #location to save the model
## Declaring path of dataset
path_img = Path("/cluster/home/cantoniou/deep_project/sipakmedFormat")
## Loading data
data = ImageDataBunch.from_folder(path=path_img,
train='train',
valid='val',
ds_tfms=get_transforms(),
size=224,
bs=batch_size) #, check_ext=False)
## Normalizing data based on Image net parameters
#data.normalize(imagenet_stats)
#normalize now according to the batch data and not imagenet
data.normalize() #defaults to batch 'stats'
print(data.classes)
len(data.classes), data.c
#LOAD THE TRANSFER LEARNING MODEL
## To create a ResNET 50 with pretrained weights based on the new dataset
trans_model = cnn_learner(
data,
models.resnet50,
metrics=[accuracy, FBeta(average="weighted")])
#print(trans_model) #check the architecture of the loaded model to make sure it was loaded with a head of 5 ---to match the data classes
trans_model = trans_model.load(
location
) #load the previous pretrained model weights form the harlev dataset
print("Start training")
#find best learning rate
trans_model.lr_find()
# Train the model
trans_model.fit_one_cycle(epochs,
callbacks=[
SaveModelCallback(trans_model,
every='improvement',
mode='max',
monitor='accuracy',
name=save_loc)
])
def train_classifier(model, config, x_train, y_train, x_val, y_val, train_tfms=None):
loss_func = torch.nn.CrossEntropyLoss()
if train_tfms is None:
train_tfms = []
train_ds = ImageArrayDS(x_train, y_train, train_tfms)
val_ds = ImageArrayDS(x_val, y_val)
data = ImageDataBunch.create(train_ds, val_ds, bs=config['batch_size'])
callbacks = [partial(EarlyStoppingCallback, min_delta=1e-3, patience=config['patience'])]
learner = Learner(data, model, metrics=accuracy, loss_func=loss_func, callback_fns=callbacks)
learner.fit(config['epochs'], config['start_lr'], wd=config['weight_decay'])
return learner
def load_data():
dataset = build_dataset('cifar_10', val_size=val_size)
x_train, y_train = dataset.dataset('train')
x_val, y_val = dataset.dataset('val')
shape = (-1, 3, 32, 32)
x_train = ((x_train - 128) / 128).reshape(shape)
x_val = ((x_val - 128) / 128).reshape(shape)
train_tfms = [*rand_pad(4, 32), flip_lr(p=0.5)]
train_ds = ImageArrayDS(x_train, y_train, train_tfms)
val_ds = ImageArrayDS(x_val, y_val)
data = ImageDataBunch.create(train_ds, val_ds, bs=256)
return data, x_train, y_train, x_val, y_val
def search(bot, update):
"""Send reply of user's message."""
photo_file = bot.get_file(update.message.photo[-1].file_id)
photo_file.download('testing.jpeg')
try:
bs = 32
path = "classes"
np.random.seed(42)
data = ImageDataBunch.from_folder(
path,
train='.',
valid_pct=0.2,
ds_tfms=get_transforms(),
size=224,
num_workers=4).normalize(imagenet_stats)
learn = cnn_learner(data, models.resnet34,
metrics=error_rate).load("stage-1")
learn.export()
learn = load_learner("classes")
cat, tensor, probs = learn.predict(open_image("testing.jpeg"))
l = list(probs)
a = tensor.__str__()
a = int(a.strip("tensor" "()"))
l = list(probs)[a]
l = l.__str__()
b = float(l.strip("tensor" "()"))
if b >= 0.9:
update.message.reply_text(
'`' + str(cat) + '`',
parse_mode=ParseMode.MARKDOWN,
reply_to_message_id=update.message.message_id)
# print("prediction :")
# print(cat)
else:
cat = "sry I am not sure "
update.message.reply_text(
'`' + str(cat) + '`',
parse_mode=ParseMode.MARKDOWN,
reply_to_message_id=update.message.message_id)
# print("prediction :")
# print("Not Sure")
except Exception as e:
update.message.reply_text(e)
def main():
# Set data transformations
if args.ds_tfms:
ds_tfms = get_transforms(do_flip=True,
flip_vert=True,
max_lighting=0.1,
max_zoom=1.05,
max_warp=0.1)
else:
ds_tfms = None
# Load data
data = ImageDataBunch.from_folder(args.data_path,
train=args.train_dirname,
valid=args.validation_dirname,
test=args.test_dirname,
ds_tfms=ds_tfms,
size=args.image_size,
bs=args.batch_size).normalize()
if args.verbose:
print(len(data.train_ds), "training images", len(data.valid_ds),
"validation images and", len(data.test_ds), "test images")
# Explore optimal learning rates
lrs, losses = tune_lr(data,
model_dict[args.model],
args.ps,
args.wd,
args.start_lr,
args.end_lr,
args.num_lr_iters,
pretrained=args.pretrained,
mixup=args.mixup)
# Create output directory if it does not exist
if not os.path.isdir(args.output_path):
os.makedirs(args.output_path, exist_ok=True)
# Save learning rates and associated loss values
np.savetxt(os.path.join(args.output_path, args.output_filename),
np.column_stack((lrs, losses)),
delimiter=',',
newline='\n',
header='lrs,losses',
comments='')
if args.verbose:
print("Completed execution.")
def main():
# Load data
data = ImageDataBunch.from_folder(
args.data_path,
train=args.train_dirname,
valid=args.test_dirname,
ds_tfms=get_transforms(do_flip=True, flip_vert=True, max_lighting=0.1, max_zoom=1.05, max_warp=0.1),
size=args.image_size,
bs=args.batch_size
).normalize()
if args.verbose:
print(len(data.valid_ds), "test images")
# Compute F1, TPR and FPR for different thresholds and find optimal threshold
optimal_thres, optimal_thres_idx, thres_metrics = tune_thres(
data, model_type[args.model], args.model_path, args.ps, args.thres,
tpr_lb=args.tpr_lb, fpr_ub=args.fpr_ub, verbose=args.verbose
)
# Save optimal threshold and associated F1 score, TPF and FPR to file
np.savetxt(
os.path.join(args.output_path, args.output_optimal_thres_filename),
[[
optimal_thres,
thres_metrics['f1'][optimal_thres_idx],
thres_metrics['tpr'][optimal_thres_idx],
thres_metrics['fpr'][optimal_thres_idx]
]],
fmt=['%f', '%f', '%f', '%f'],
delimiter=',',
newline='\n',
header='thres,f1,tpr,fpr',
comments=''
)
# Save thresholds, F1 score, TPR and FPR to file
np.savetxt(
os.path.join(args.output_path, args.output_thres_metrics_filename),
thres_metrics,
fmt=['%f', '%f', '%f', '%f'],
delimiter=',',
newline='\n',
header='thres,f1,tpr,fpr',
comments=''
)
if args.verbose:
print("Completed execution.")
def fine_tune_convnet(path: str):
"""
Path must be in the format of a FastAI data bunch, ie Train/Test/Valid dirs
and then one subdirectory per class in each of those dirs.
"""
tfms = get_transforms(do_flip=False)
data = ImageDataBunch.from_folder(path, ds_tfms=tfms, size=64)
learn = cnn_learner(data, models.resnet34, metrics=error_rate)
learn.fit_one_cycle(5)
learn.save('stage-1')
learn.unfreeze()
learn.fit_one_cycle(1, max_lr=slice(1e-6, 1e-3))
learn.save('stage-2')
learn.export(file="two-stage-model.pkl")
def main():
# Load data
data = ImageDataBunch.from_folder(args.data_path,
train=args.train_dirname,
valid=args.validation_dirname,
test=args.test_dirname,
ds_tfms=get_transforms(do_flip=True,
flip_vert=True,
max_lighting=0.1,
max_zoom=1.05,
max_warp=0.1),
size=args.image_size,
bs=args.batch_size).normalize()
if args.verbose:
print(len(data.train_ds), "training images", len(data.valid_ds),
"validation images and", len(data.test_ds), "test images")
# Create output directory if it does not exist
if not Path(args.output_path).exists():
Path(args.output_path).mkdir(parents=True, exist_ok=True)
# Fit model using one-cycle policy
recorder = fit_model(data, model_dict[args.model], args.pretrained,
args.ps, args.cyc_len, args.lr_lower, args.lr_upper,
args.output_path, args.output_model_filename,
args.verbose)
# Save loss value for each processed batch to file
if args.save_loss:
np.savetxt(Path(args.output_path).joinpath(
args.output_training_loss_filename),
[l.item() for l in recorder.losses],
newline='\n',
header='losses',
comments='')
# Save training and validation losses per epoch
np.savetxt(Path(args.output_path).joinpath(args.output_loss_filename),
all_losses_to_array(get_all_losses_per_epoch(recorder)),
fmt=['%d', '%f', '%f'],
delimiter=',',
newline='\n',
header='nb_batches,train_losses,val_losses',
comments='')
if args.verbose:
print("Completed execution.")
def __init__(self, picture_file):
self.classes = [
'Afghan_hound', 'African_hunting_dog', 'Airedale',
'American_Staffordshire_terrier', 'Appenzeller',
'Australian_terrier', 'Bedlington_terrier', 'Bernese_mountain_dog',
'Blenheim_spaniel', 'Border_collie', 'Border_terrier',
'Boston_bull', 'Bouvier_des_Flandres', 'Brabancon_griffon',
'Brittany_spaniel', 'Cardigan', 'Chesapeake_Bay_retriever',
'Chihuahua', 'Dandie_Dinmont', 'Doberman', 'English_foxhound',
'English_setter', 'English_springer', 'EntleBucher', 'Eskimo_dog',
'French_bulldog', 'German_shepherd', 'German_short-haired_pointer',
'Gordon_setter', 'Great_Dane', 'Great_Pyrenees',
'Greater_Swiss_Mountain_dog', 'Ibizan_hound', 'Irish_setter',
'Irish_terrier', 'Irish_water_spaniel', 'Irish_wolfhound',
'Italian_greyhound', 'Japanese_spaniel', 'Kerry_blue_terrier',
'Labrador_retriever', 'Lakeland_terrier', 'Leonberg', 'Lhasa',
'Maltese_dog', 'Mexican_hairless', 'Newfoundland',
'Norfolk_terrier', 'Norwegian_elkhound', 'Norwich_terrier',
'Old_English_sheepdog', 'Pekinese', 'Pembroke', 'Pomeranian',
'Rhodesian_ridgeback', 'Rottweiler', 'Saint_Bernard', 'Saluki',
'Samoyed', 'Scotch_terrier', 'Scottish_deerhound',
'Sealyham_terrier', 'Shetland_sheepdog', 'Shih-Tzu',
'Siberian_husky', 'Staffordshire_bullterrier', 'Sussex_spaniel',
'Tibetan_mastiff', 'Tibetan_terrier', 'Walker_hound', 'Weimaraner',
'Welsh_springer_spaniel', 'West_Highland_white_terrier',
'Yorkshire_terrier', 'affenpinscher', 'basenji', 'basset',
'beagle', 'black-and', 'bloodhound', 'bluetick', 'borzoi', 'boxer',
'briard', 'bull_mastiff', 'cairn', 'chow', 'clumber',
'cocker_spaniel', 'collie', 'curly-coated_retriever', 'dhole',
'dingo', 'flat-coated_retriever', 'giant_schnauzer',
'golden_retriever', 'groenendael', 'keeshond', 'kelpie',
'komondor', 'kuvasz', 'malamute', 'malinois', 'miniature_pinscher',
'miniature_poodle', 'miniature_schnauzer', 'otterhound',
'papillon', 'pug', 'redbone', 'schipperke', 'silky_terrier',
'soft-coated_wheaten_terrier', 'standard_poodle',
'standard_schnauzer', 'toy_poodle', 'toy_terrier', 'vizsla',
'whippet', 'wire-haired_fox_terrier'
]
f_name, f_ext = os.path.splitext(picture_file)
img_path = os.path.join(app.root_path, 'static', 'images',
f_name + f_ext)
self.img = open_image(img_path)
self.data = ImageDataBunch.single_from_classes(
"./", self.classes, ds_tfms=get_transforms(),
size=224).normalize(imagenet_stats)
self.learner = cnn_learner(self.data, models.resnet50)
self.learner.load('stage-2-rerun')
def train_model(model_name, dataset_name, arch, img_size, epochs):
data = ImageDataBunch.from_folder(f'{BASE_DIR}datasets/{dataset_name}',
valid_pct=0.2,
ds_tfms=get_transforms(),
size=img_size,
num_workers=6,
bs=BATCH_SIZE)
arch = model_mapping[arch]
learner = cnn_learner(data, arch, metrics=[error_rate, accuracy])
learner.fit_one_cycle(epochs)
learner.export(
os.path.join(os.getcwd(), '..', 'static', 'models',
model_name + '.pkl'))
meta = {}
meta['metrics'] = [[i.item() for i in e] for e in learner.recorder.metrics]
meta['loss'] = [i.item() for i in learner.recorder.losses]
meta['lr'] = [i.item() for i in learner.recorder.lrs]
return meta
def test_freeze_unfreeze_effnet():
this_tests(cnn_learner)
def get_number_of_trainable_params(model: nn.Module):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
base_arch = EfficientNetB1
path = untar_data(URLs.MNIST_TINY)
data = ImageDataBunch.from_folder(path, size=64)
data.c = 1000 # Set number of class to be 1000 to stay in line with the pretrained model.
cnn_learn = cnn_learner(data, base_arch, pretrained=True)
ref_learn = Learner(data, EfficientNet.from_pretrained("efficientnet-b1"))
# By default the neural net in cnn learner is freezed.
assert get_number_of_trainable_params(
cnn_learn.model) < get_number_of_trainable_params(ref_learn.model)
cnn_learn.unfreeze()
assert get_number_of_trainable_params(
cnn_learn.model) == get_number_of_trainable_params(ref_learn.model)
