def read_imported_config(project_path, project_name, projects=None):
# Oh god this logic is a disaster, user interfaces are hard
unique_name = False
while unique_name == False:
unique_name = True
if projects is not None:
for project in projects:
if project['name'] == project_name:
print(
colored('Project with this name already exists.',
'red'))
project_name = str_input('Provide a new project name: ')
unique_name = False
project_dest = os.path.expanduser(
str_input('Provide a path for your predictions to be saved: '))
if os.path.isdir(project_dest) == False:
print('Creating directory:', project_dest)
os.makedirs(project_dest, exist_ok=True)
# You don't get to judge me t('-' t)
with open(os.path.join(project_path, 'config.yaml'), 'r') as fp:
import_project = yaml.load(fp.read())
import_project['name'] = project_name
import_project['server_weights'] = [
os.path.join(project_path, weight)
for weight in os.listdir(project_path) if weight.find('.hdf5') > 0
]
import_project['path'] = project_dest
return import_project
def select_augmentations():
print('Select augmentations:')
print(colored('Note: defaults are all zero or false.', 'cyan'))
rounds = int_input('number of augmentation rounds', 1, 100)
# These three are not implemented because they require training and then that would
# need to get propogated over which is complicated for prediction
featurewise_center = False #bool_input('featurewise_center: set input mean to 0 over the dataset.')
featurewise_std_normalization = False #bool_input('featurewise_std_normalization: divide inputs by std of the dataset.')
zca_whitening = False #bool_input('zca_whitening: apply ZCA whitening.')
samplewise_center = False #bool_input('samplewise_center: set each sample mean to 0.')
samplewise_std_normalization = False #bool_input('samplewise_std_normalization: divide each input by its std.')
rotation_range = int_input('rotation_range: degrees', 0, 180, True)
width_shift_range = float_input('width_shift_range: fraction of total width.', 0., 1.)
height_shift_range = float_input('height_shift_range: fraction of total width.', 0., 1.)
shear_range = float_input('shear_range: shear intensity (shear angle in radians)', 0., np.pi/2)
zoom_range_in = float_input('zoom_range: amount of zoom in. 1.0 is no zoom, 0 is full zoom.', 0., 1.)
zoom_range_out = float_input('zoom_range: amount of zoom out. 1.0 is no zoom, 2.0 is full zoom ', 1., 2.)
channel_shift_range = float_input('channel_shift_rangee: shift range for each channels.', 0., 1.)
print('fill_mode: points outside the boundaries are filled according to the given mode.')
fill_mode = str_input('constant, nearest, reflect, or wrap. Default nearest: ',['constant', 'nearest', 'reflect', 'wrap'])
if (fill_mode == 'constant'):
cval = float_input('cval: value used for points outside the boundaries', 0., 1.)
else:
cval = 0.0
horizontal_flip = bool_input('horizontal_flip: whether to randomly flip images horizontally.')
vertical_flip = bool_input('vertical_flip: whether to randomly flip images vertically.')
rescale = None #int_input('rescale: rescaling factor. If None or 0, no rescaling is applied, otherwise we multiply the data by the value provided.', 0, 255)
if rescale == 0:
rescale = None
augmentations = {'rounds': rounds,
'featurewise_center': featurewise_center,
'featurewise_std_normalization': featurewise_std_normalization,
'samplewise_center': samplewise_center,
'samplewise_std_normalization': samplewise_std_normalization,
'zca_whitening': zca_whitening,
'rotation_range': rotation_range,
'width_shift_range': width_shift_range,
'height_shift_range': height_shift_range,
'shear_range': shear_range,
'zoom_range': [zoom_range_in, zoom_range_out],
'channel_shift_range': channel_shift_range,
'fill_mode': fill_mode,
'cval': cval,
'horizontal_flip': horizontal_flip,
'vertical_flip': vertical_flip,
'rescale': rescale}
return augmentations
def select_augmentations():
print('Select augmentations:')
print(colored('Note: defaults are all zero or false.', 'cyan'))
rounds = int_input('number of augmentation rounds', 1, 100)
# These three are not implemented because they require training and then that would
# need to get propogated over which is complicated for prediction
featurewise_center = False #bool_input('featurewise_center: set input mean to 0 over the dataset.')
featurewise_std_normalization = False #bool_input('featurewise_std_normalization: divide inputs by std of the dataset.')
zca_whitening = False #bool_input('zca_whitening: apply ZCA whitening.')
samplewise_center = False #bool_input('samplewise_center: set each sample mean to 0.')
samplewise_std_normalization = False #bool_input('samplewise_std_normalization: divide each input by its std.')
rotation_range = int_input('rotation_range: degrees', 0, 180, True)
width_shift_range = float_input('width_shift_range: fraction of total width.', 0., 1.)
height_shift_range = float_input('height_shift_range: fraction of total width.', 0., 1.)
shear_range = float_input('shear_range: shear intensity (shear angle in radians)', 0., np.pi/2)
zoom_range_in = float_input('zoom_range: amount of zoom in. 1.0 is no zoom, 0 is full zoom.', 0., 1.)
zoom_range_out = float_input('zoom_range: amount of zoom out. 1.0 is no zoom, 2.0 is full zoom ', 1., 2.)
channel_shift_range = float_input('channel_shift_rangee: shift range for each channels.', 0., 1.)
print('fill_mode: points outside the boundaries are filled according to the given mode.')
fill_mode = str_input('constant, nearest, reflect, or wrap. Default nearest: ',['constant', 'nearest', 'reflect', 'wrap'])
if (fill_mode == 'constant'):
cval = float_input('cval: value used for points outside the boundaries', 0., 1.)
else:
cval = 0.0
horizontal_flip = bool_input('horizontal_flip: whether to randomly flip images horizontally.')
vertical_flip = bool_input('vertical_flip: whether to randomly flip images vertically.')
rescale = None #int_input('rescale: rescaling factor. If None or 0, no rescaling is applied, otherwise we multiply the data by the value provided.', 0, 255)
if rescale == 0:
rescale = None
augmentations = {'rounds': rounds,
'featurewise_center': featurewise_center,
'featurewise_std_normalization': featurewise_std_normalization,
'samplewise_center': samplewise_center,
'samplewise_std_normalization': samplewise_std_normalization,
'zca_whitening': zca_whitening,
'rotation_range': rotation_range,
'width_shift_range': width_shift_range,
'height_shift_range': height_shift_range,
'shear_range': shear_range,
'zoom_range': [zoom_range_in, zoom_range_out],
'channel_shift_range': channel_shift_range,
'fill_mode': fill_mode,
'cval': cval,
'horizontal_flip': horizontal_flip,
'vertical_flip': vertical_flip,
'rescale': rescale}
return augmentations
def read_imported_config(project_path, project_name, projects = None):
# Oh god this logic is a disaster, user interfaces are hard
unique_name = False
while unique_name == False:
unique_name = True
if projects is not None:
for project in projects:
if project['name'] == project_name:
print(colored('Project with this name already exists.', 'red'))
project_name = str_input('Provide a new project name: ')
unique_name = False
project_dest = os.path.expanduser(str_input('Provide a path for your predictions to be saved: '))
if os.path.isdir(project_dest) == False:
print('Creating directory:', project_dest)
os.makedirs(project_dest, exist_ok = True)
# You don't get to judge me t('-' t)
with open(os.path.join(project_path, 'config.yaml'), 'r') as fp:
import_project = yaml.load(fp.read())
import_project['name'] = project_name
import_project['server_weights'] = [os.path.join(project_path, weight) for weight in os.listdir(project_path) if weight.find('.hdf5') > 0]
import_project['path'] = project_dest
return import_project
def configure():
'''
Configure the transfer environment and store
'''
completer = Completer()
readline.set_completer_delims('\t')
readline.parse_and_bind('tab: complete')
readline.set_completer(completer.path_completer)
home = os.path.expanduser('~')
if os.path.isfile(os.path.join(home, '.transfer', 'config.yaml')):
with open(os.path.join(home, '.transfer', 'config.yaml'), 'r') as fp:
config = yaml.load(fp.read())
else:
config = []
project_name = input('Name your project: ')
existing_project = None
for project in config:
if project_name == project['name']:
existing_project = project_name
if existing_project is not None:
print(colored('Project ' + project_name + ' already exists', 'red'))
overwrite = str_input(
'Would you like to overwrite this project? (yes or no) ',
['yes', 'no'])
if overwrite == 'no':
return
else:
config = [
project for project in config
if project_name != project['name']
]
image_path = os.path.expanduser(
input('Select parent directory for your images: '))
path_unset = True
while path_unset:
project_path = os.path.expanduser(
input('Select destination for your project: '))
if (project_path.find(image_path) == 0):
print(
'Project destination should not be same or within image directory!'
)
else:
path_unset = False
print('Select architecture:')
print('[0] resnet50')
print('[1] xception')
print('[2] inception_v3')
architecture = int_input('choice', 0, 2, show_range=False)
if architecture == 0:
arch = 'resnet50'
img_dim = 224
conv_dim = 7
final_cutoff = 80
elif architecture == 1:
arch = 'xception'
img_dim = 299
conv_dim = 10
final_cutoff = 80
else:
arch = 'inception_v3'
img_dim = 299
conv_dim = 8
final_cutoff = 80
api_port = int_input('port for local prediction API (suggested: 5000)',
1024, 49151)
kfold = int_input('number of folds to use (suggested: 5)', 3, 10)
kfold_every = bool_input(
'Fit a model for every fold? (if false, just fit one)')
print(
'Warning: if working on a remote computer, you may not be able to plot!'
)
plot_cm = bool_input('Plot a confusion matrix after training?')
batch_size = int_input('batch size (suggested: 8)', 1, 64)
learning_rate = float_input('learning rate (suggested: 0.001)', 0, 1)
learning_rate_decay = float_input(
'learning decay rate (suggested: 0.000001)', 0, 1)
cycle = int_input(
'number of cycles before resetting the learning rate (suggested: 3)',
1, 10)
num_rounds = int_input('number of rounds (suggested: 3)', 1, 100)
print('Select image resolution:')
print('[0] low (' + str(img_dim) + ' px)')
print('[1] mid (' + str(img_dim * 2) + ' px)')
print('[2] high (' + str(img_dim * 4) + ' px)')
img_resolution_index = int_input('choice', 0, 2, show_range=False)
if img_resolution_index == 0:
img_size = 1
elif img_resolution_index == 1:
img_size = 2
else:
img_size = 4
use_augmentation = str_input(
'Would you like to add image augmentation? (yes or no) ',
['yes', 'no'])
if use_augmentation == 'yes':
augmentations = select_augmentations()
else:
augmentations = None
project = {
'name': project_name,
'img_path': image_path,
'path': project_path,
'plot': plot_cm,
'api_port': api_port,
'kfold': kfold,
'kfold_every': kfold_every,
'cycle': cycle,
'seed': np.random.randint(9999),
'batch_size': batch_size,
'learning_rate': learning_rate,
'learning_rate_decay': learning_rate_decay,
'final_cutoff': final_cutoff,
'rounds': num_rounds,
'img_size': img_size,
'augmentations': augmentations,
'architecture': arch,
'img_dim': img_dim,
'conv_dim': conv_dim,
'is_split': False,
'is_array': False,
'is_augmented': False,
'is_pre_model': False,
'is_final': False,
'model_round': 0,
'server_weights': None,
'last_weights': None,
'best_weights': None
}
config.append(project)
store_config(config)
print('')
print(colored('Project configure saved!', 'cyan'))
print('')
print('To run project:')
print('')
print(colored(' transfer --run --project ' + project_name, 'green'))
print('or')
print(colored(' transfer -r -p ' + project_name, 'green'))
def configure_server():
'''
Configure the transfer environment and store
'''
home = os.path.expanduser('~')
if os.path.isfile(os.path.join(home, '.transfer', 'config.yaml')):
with open(os.path.join(home, '.transfer', 'config.yaml'), 'r') as fp:
config = yaml.load(fp.read())
else:
config = []
project_name = input('Name your project: ')
existing_project = None
for project in config:
if project_name == project['name']:
existing_project = project_name
if existing_project is not None:
print(colored('Project ' + project_name + ' already exists', 'red'))
overwrite = str_input(
'Would you like to overwrite this project? (yes or no) ',
['yes', 'no'])
if overwrite == 'no':
return
else:
config = [
project for project in config
if project_name != project['name']
]
api_port = int_input('port for local prediction API (suggested: 5000)',
1024, 49151)
print('Select image resolution:')
print('[0] low (224 px)')
print('[1] mid (448 px)')
print('[2] high (896 px)')
img_resolution_index = int_input('choice', 0, 2, show_range=False)
if img_resolution_index == 0:
img_size = 1
elif img_resolution_index == 1:
img_size = 2
else:
img_size = 4
num_categories = int_input('number of image categories in your model', 0,
10000000)
weights = False
while weights == False:
server_weights = os.path.expanduser(input('Select weights file: '))
if os.path.isfile(server_weights):
weights = True
else:
print('Cannot find the weight file: ', server_weights)
project = {
'name': project_name,
'api_port': api_port,
'img_size': img_size,
'number_categories': num_categories,
'server_weights': server_weights
}
config.append(project)
store_config(config)
print('')
print(colored('Project configure saved!', 'cyan'))
print('')
print('To start the server:')
print('')
print(
colored(' transfer --prediction-rest-api --project ' + project_name,
'green'))
print('or')
print(
colored(' transfer --prediction-rest-api -p ' + project_name,
'green'))
def main(args = None):
'''
Main entry point for transfer command line tool.
This essentially will marshall the user to the functions they need.
'''
parser = argparse.ArgumentParser(description = 'Tool to perform transfer learning')
parser.add_argument('-c','--configure',
action = 'store_true',
help = 'Configure transfer')
parser.add_argument('-e','--export',
action = 'store_true',
dest = 'export_config',
help = 'Export configuration and models')
parser.add_argument('-i','--import',
action = 'store',
type = str,
default = None,
dest = 'import_config',
help = 'Import configuration and models')
parser.add_argument('-p','--project',
action = 'store',
type = str,
default = None,
dest = 'project',
help = 'Specify a project, if not supplied it will be picked from configured projects')
parser.add_argument('-r','--run',
action = 'store_true',
help = 'Run all transfer learning operations')
parser.add_argument('-f','--final',
action = 'store_true',
help = 'Run final training on all layers: Warning SLOW!')
parser.add_argument('-l','--last-weights',
action = 'store_true',
dest = 'last',
help = 'Restart from the last weights, rather than the best intermediate weights')
parser.add_argument('--predict',
action = 'store',
type = str,
default = None,
const = 'default',
dest = 'predict',
nargs='?',
help = 'Predict model on file or directory')
parser.add_argument('--prediction-rest-api',
action = 'store_true',
dest = 'rest_api',
help = 'Start rest api to make predictions on files or directories')
if len(sys.argv) == 1:
parser.print_help()
return
args = parser.parse_args()
if args.import_config is not None:
import_config(args.import_config)
return
elif args.export_config:
project = select_project(args.project)
weights = model_input(project)
ind = model_individual_input(project, weights)
export_config(project, weights, ind)
return
elif args.configure:
configure()
return
else:
project = select_project(args.project)
if args.run:
if project['is_array'] == False:
project = images_to_array(project)
update_config(project)
if project['is_augmented'] == False:
project = augment_arrays(project)
update_config(project)
if project['is_pre_model'] == False:
project = pre_model(project)
update_config(project)
project = train_model(project, final = args.final, last = args.last)
update_config(project)
print('')
print(colored('Completed modeling round: ' + str(project['model_round']), 'cyan'))
print('')
print('Best current model: ', colored(project['best_weights'], 'yellow'))
print('Last current model: ', colored(project['last_weights'], 'yellow'))
print('')
print('To further refine the model, run again with:')
print('')
print(colored(' transfer --run --project ' + project['name'], 'green'))
print('')
print('To make predictions:')
print('')
print(colored(' transfer --predict [optional dir or file] --project ' + project['name'], 'yellow'))
print('')
elif args.rest_api:
if project['server_weights'] is not None:
start_server(project, 'server_weights')
elif project['best_weights'] is not None:
weights = model_input(project)
start_server(project, weights)
else:
print('Model is not trained. Please first run your project:')
print('')
print(colored(' transfer --run', 'green'))
print('')
elif args.predict is not None:
if args.predict == 'default':
args.predict = str_input('Enter a path to file(s): ')
if project['server_weights'] is not None:
predict_model(project, 'server_weights', args.predict)
elif project['best_weights'] is not None:
weights = model_input(project)
print('Predicting on image(s) in: ', colored(args.predict, 'yellow'))
predict_model(project, weights, args.predict)
else:
print('Model is not trained. Please first run your project:')
print('')
print(colored(' transfer --run', 'green'))
print('')
def main(args=None):
'''
Main entry point for transfer command line tool.
This essentially will marshall the user to the functions they need.
'''
parser = argparse.ArgumentParser(
description='Tool to perform transfer learning')
parser.add_argument('-c',
'--configure',
action='store_true',
help='Configure transfer')
parser.add_argument('-e',
'--export',
action='store_true',
dest='export_config',
help='Export configuration and models')
parser.add_argument('-i',
'--import',
action='store',
type=str,
default=None,
dest='import_config',
help='Import configuration and models')
parser.add_argument(
'-p',
'--project',
action='store',
type=str,
default=None,
dest='project',
help=
'Specify a project, if not supplied it will be picked from configured projects'
)
parser.add_argument('-r',
'--run',
action='store_true',
help='Run all transfer learning operations')
parser.add_argument('-f',
'--final',
action='store_true',
help='Run final training on all layers: Warning SLOW!')
parser.add_argument(
'-l',
'--last-weights',
action='store_true',
dest='last',
help=
'Restart from the last weights, rather than the best intermediate weights'
)
parser.add_argument('--predict',
action='store',
type=str,
default=None,
const='default',
dest='predict',
nargs='?',
help='Predict model on file or directory')
parser.add_argument(
'--prediction-rest-api',
action='store_true',
dest='rest_api',
help='Start rest api to make predictions on files or directories')
if len(sys.argv) == 1:
parser.print_help()
return
args = parser.parse_args()
if args.import_config is not None:
import_config(args.import_config)
return
elif args.export_config:
project = select_project(args.project)
weights = model_input(project)
ind = model_individual_input(project, weights)
export_config(project, weights, ind)
return
elif args.configure:
configure()
return
else:
project = select_project(args.project)
if args.run:
if project['is_array'] == False:
project = images_to_array(project)
update_config(project)
if project['is_augmented'] == False:
project = augment_arrays(project)
update_config(project)
if project['is_pre_model'] == False:
project = pre_model(project)
update_config(project)
project = train_model(project, final=args.final, last=args.last)
update_config(project)
print('')
print(
colored('Completed modeling round: ' + str(project['model_round']),
'cyan'))
print('')
print('Best current model: ', colored(project['best_weights'],
'yellow'))
print('Last current model: ', colored(project['last_weights'],
'yellow'))
print('')
print('To further refine the model, run again with:')
print('')
print(
colored(' transfer --run --project ' + project['name'],
'green'))
print('')
print('To make predictions:')
print('')
print(
colored(
' transfer --predict [optional dir or file] --project ' +
project['name'], 'yellow'))
print('')
elif args.rest_api:
if project['server_weights'] is not None:
start_server(project, 'server_weights')
elif project['best_weights'] is not None:
weights = model_input(project)
start_server(project, weights)
else:
print('Model is not trained. Please first run your project:')
print('')
print(colored(' transfer --run', 'green'))
print('')
elif args.predict is not None:
if args.predict == 'default':
args.predict = str_input('Enter a path to file(s): ')
if project['server_weights'] is not None:
predict_model(project, 'server_weights', args.predict)
elif project['best_weights'] is not None:
weights = model_input(project)
print('Predicting on image(s) in: ',
colored(args.predict, 'yellow'))
predict_model(project, weights, args.predict)
else:
print('Model is not trained. Please first run your project:')
print('')
print(colored(' transfer --run', 'green'))
print('')
def configure_server():
'''
Configure the transfer environment and store
'''
home = os.path.expanduser('~')
if os.path.isfile(os.path.join(home, '.transfer', 'config.yaml')):
with open(os.path.join(home, '.transfer', 'config.yaml'), 'r') as fp:
config = yaml.load(fp.read())
else:
config = []
project_name = input('Name your project: ')
existing_project = None
for project in config:
if project_name == project['name']:
existing_project = project_name
if existing_project is not None:
print(colored('Project ' + project_name + ' already exists', 'red'))
overwrite = str_input('Would you like to overwrite this project? (yes or no) ', ['yes', 'no'])
if overwrite == 'no':
return
else:
config = [project for project in config if project_name != project['name']]
api_port = int_input('port for local prediction API (suggested: 5000)', 1024, 49151)
print('Select image resolution:')
print('[0] low (224 px)')
print('[1] mid (448 px)')
print('[2] high (896 px)')
img_resolution_index = int_input('choice', 0, 2, show_range = False)
if img_resolution_index == 0:
img_size = 1
elif img_resolution_index == 1:
img_size = 2
else:
img_size = 4
num_categories = int_input('number of image categories in your model', 0, 10000000)
weights = False
while weights == False:
server_weights = os.path.expanduser(input('Select weights file: '))
if os.path.isfile(server_weights):
weights = True
else:
print('Cannot find the weight file: ', server_weights)
project = {'name': project_name,
'api_port': api_port,
'img_size': img_size,
'number_categories': num_categories,
'server_weights': server_weights}
config.append(project)
store_config(config)
print('')
print(colored('Project configure saved!', 'cyan'))
print('')
print('To start the server:')
print('')
print(colored(' transfer --prediction-rest-api --project ' + project_name, 'green'))
print('or')
print(colored(' transfer --prediction-rest-api -p ' + project_name, 'green'))
def configure():
'''
Configure the transfer environment and store
'''
home = os.path.expanduser('~')
if os.path.isfile(os.path.join(home, '.transfer', 'config.yaml')):
with open(os.path.join(home, '.transfer', 'config.yaml'), 'r') as fp:
config = yaml.load(fp.read())
else:
config = []
project_name = input('Name your project: ')
existing_project = None
for project in config:
if project_name == project['name']:
existing_project = project_name
if existing_project is not None:
print(colored('Project ' + project_name + ' already exists', 'red'))
overwrite = str_input('Would you like to overwrite this project? (yes or no) ', ['yes', 'no'])
if overwrite == 'no':
return
else:
config = [project for project in config if project_name != project['name']]
image_path = os.path.expanduser(input('Select parent directory for your images: '))
path_unset = True
while path_unset:
project_path = os.path.expanduser(input('Select destination for your project: '))
if (project_path.find(image_path) == 0):
print('Project destination should not be same or within image directory!')
else:
path_unset = False
print('Select architecture:')
print('[0] resnet50')
print('[1] xception')
print('[2] inception_v3')
architecture = int_input('choice', 0, 2, show_range = False)
if architecture == 0:
arch = 'resnet50'
img_dim = 224
conv_dim = 7
final_cutoff = 80
elif architecture == 1:
arch = 'xception'
img_dim = 299
conv_dim = 10
final_cutoff = 80
else:
arch = 'inception_v3'
img_dim = 299
conv_dim = 8
final_cutoff = 80
api_port = int_input('port for local prediction API (suggested: 5000)', 1024, 49151)
kfold = int_input('number of folds to use (suggested: 5)', 3, 10)
kfold_every = bool_input('Fit a model for every fold? (if false, just fit one)')
print('Warning: if working on a remote computer, you may not be able to plot!')
plot_cm = bool_input('Plot a confusion matrix after training?')
batch_size = int_input('batch size (suggested: 8)', 1, 64)
learning_rate = float_input('learning rate (suggested: 0.001)', 0, 1)
learning_rate_decay = float_input('learning decay rate (suggested: 0.000001)', 0, 1)
cycle = int_input('number of cycles before resetting the learning rate (suggested: 3)', 1, 10)
num_rounds = int_input('number of rounds (suggested: 3)', 1, 100)
print('Select image resolution:')
print('[0] low (' + str(img_dim) + ' px)')
print('[1] mid (' + str(img_dim * 2) + ' px)')
print('[2] high (' + str(img_dim * 4) + ' px)')
img_resolution_index = int_input('choice', 0, 2, show_range = False)
if img_resolution_index == 0:
img_size = 1
elif img_resolution_index == 1:
img_size = 2
else:
img_size = 4
use_augmentation = str_input('Would you like to add image augmentation? (yes or no) ', ['yes', 'no'])
if use_augmentation == 'yes':
augmentations = select_augmentations()
else:
augmentations = None
project = {'name': project_name,
'img_path': image_path,
'path': project_path,
'plot': plot_cm,
'api_port': api_port,
'kfold': kfold,
'kfold_every': kfold_every,
'cycle': cycle,
'seed': np.random.randint(9999),
'batch_size': batch_size,
'learning_rate': learning_rate,
'learning_rate_decay': learning_rate_decay,
'final_cutoff': final_cutoff,
'rounds': num_rounds,
'img_size': img_size,
'augmentations': augmentations,
'architecture': arch,
'img_dim': img_dim,
'conv_dim': conv_dim,
'is_split': False,
'is_array': False,
'is_augmented': False,
'is_pre_model': False,
'is_final': False,
'model_round': 0,
'server_weights': None,
'last_weights': None,
'best_weights': None}
config.append(project)
store_config(config)
print('')
print(colored('Project configure saved!', 'cyan'))
print('')
print('To run project:')
print('')
print(colored(' transfer --run --project ' + project_name, 'green'))
print('or')
print(colored(' transfer -r -p ' + project_name, 'green'))
def read_imported_config(project_path, project_name, projects=None):
completer = Completer()
readline.set_completer_delims('\t')
readline.parse_and_bind('tab: complete')
readline.set_completer(completer.path_completer)
# Oh god this logic is a disaster, user interfaces are hard
bad_user = True
while bad_user:
relearn_str = str_input(
'Do you want to learn on new starting from these weights? (yes or no) '
)
if relearn_str.lower() == 'yes' or relearn_str.lower() == 'y':
bad_user = False
relearn = True
elif relearn_str.lower() == 'no' or relearn_str.lower() == 'n':
bad_user = False
relearn = False
unique_name = False
while unique_name == False:
unique_name = True
if projects is not None:
for project in projects:
if project['name'] == project_name:
print(
colored('Project with this name already exists.',
'red'))
project_name = str_input('Provide a new project name: ')
unique_name = False
if relearn:
image_path = os.path.expanduser(
input('Select parent directory for your images: '))
path_unset = True
while path_unset:
project_dest = os.path.expanduser(
input('Select destination for your project: '))
if (project_dest.find(image_path) == 0):
print(
'Project destination should not be same or within image directory!'
)
else:
path_unset = False
else:
project_dest = os.path.expanduser(
input('Select destination for your project: '))
if os.path.isdir(project_dest) == False:
print('Creating directory:', project_dest)
os.makedirs(project_dest, exist_ok=True)
# You don't get to judge me t('-' t)
with open(os.path.join(project_path, 'config.yaml'), 'r') as fp:
import_project = yaml.load(fp.read())
import_project['name'] = project_name
import_project['path'] = project_dest
if relearn:
kfold = int_input('number of folds to use (suggested: 5)', 3, 10)
kfold_every = bool_input(
'Fit a model for every fold? (if false, just fit one)')
print(
'Warning: if working on a remote computer, you may not be able to plot!'
)
plot_cm = bool_input('Plot a confusion matrix after training?')
batch_size = int_input('batch size (suggested: 8)', 1, 64)
learning_rate = float_input('learning rate (suggested: 0.001)', 0, 1)
learning_rate_decay = float_input(
'learning decay rate (suggested: 0.000001)', 0, 1)
cycle = int_input(
'number of cycles before resetting the learning rate (suggested: 3)',
1, 10)
num_rounds = int_input('number of rounds (suggested: 3)', 1, 100)
import_project['img_path'] = image_path
import_project['best_weights'] = [
os.path.join(project_path, weight)
for weight in os.listdir(project_path) if weight.find('.hdf5') > 0
]
import_project['last_weights'] = import_project['best_weights']
import_project['server_weights'] = None
import_project['kfold'] = kfold
import_project['kfold_every'] = kfold_every
import_project['cycle'] = cycle
import_project['seed'] = np.random.randint(9999)
import_project['batch_size'] = batch_size
import_project['learning_rate'] = learning_rate
import_project['learning_rate_decay'] = learning_rate_decay
if 'final_cutoff' not in import_project.keys():
import_project['final_cutoff'] = 80
import_project['rounds'] = num_rounds
import_project['is_split'] = False
import_project['is_array'] = False
import_project['is_augmented'] = False
import_project['is_pre_model'] = False
import_project['model_round'] = 1
import_project['plot'] = plot_cm
print('')
print('To re-learn new images with project:')
print('')
print(colored(' transfer --run --project ' + project_name, 'green'))
print('or')
print(colored(' transfer -r -p ' + project_name, 'green'))
print('')
else:
import_project['server_weights'] = [
os.path.join(project_path, weight)
for weight in os.listdir(project_path) if weight.find('.hdf5') > 0
]
return import_project
