def select_augmentations(): print('Select augmentations:') print(colored('Note: defaults are all zero or false.', 'cyan')) rounds = int_input('number of augmentation rounds', 1, 100) # These three are not implemented because they require training and then that would # need to get propogated over which is complicated for prediction featurewise_center = False #bool_input('featurewise_center: set input mean to 0 over the dataset.') featurewise_std_normalization = False #bool_input('featurewise_std_normalization: divide inputs by std of the dataset.') zca_whitening = False #bool_input('zca_whitening: apply ZCA whitening.') samplewise_center = False #bool_input('samplewise_center: set each sample mean to 0.') samplewise_std_normalization = False #bool_input('samplewise_std_normalization: divide each input by its std.') rotation_range = int_input('rotation_range: degrees', 0, 180, True) width_shift_range = float_input('width_shift_range: fraction of total width.', 0., 1.) height_shift_range = float_input('height_shift_range: fraction of total width.', 0., 1.) shear_range = float_input('shear_range: shear intensity (shear angle in radians)', 0., np.pi/2) zoom_range_in = float_input('zoom_range: amount of zoom in. 1.0 is no zoom, 0 is full zoom.', 0., 1.) zoom_range_out = float_input('zoom_range: amount of zoom out. 1.0 is no zoom, 2.0 is full zoom ', 1., 2.) channel_shift_range = float_input('channel_shift_rangee: shift range for each channels.', 0., 1.) print('fill_mode: points outside the boundaries are filled according to the given mode.') fill_mode = str_input('constant, nearest, reflect, or wrap. Default nearest: ',['constant', 'nearest', 'reflect', 'wrap']) if (fill_mode == 'constant'): cval = float_input('cval: value used for points outside the boundaries', 0., 1.) else: cval = 0.0 horizontal_flip = bool_input('horizontal_flip: whether to randomly flip images horizontally.') vertical_flip = bool_input('vertical_flip: whether to randomly flip images vertically.') rescale = None #int_input('rescale: rescaling factor. If None or 0, no rescaling is applied, otherwise we multiply the data by the value provided.', 0, 255) if rescale == 0: rescale = None augmentations = {'rounds': rounds, 'featurewise_center': featurewise_center, 'featurewise_std_normalization': featurewise_std_normalization, 'samplewise_center': samplewise_center, 'samplewise_std_normalization': samplewise_std_normalization, 'zca_whitening': zca_whitening, 'rotation_range': rotation_range, 'width_shift_range': width_shift_range, 'height_shift_range': height_shift_range, 'shear_range': shear_range, 'zoom_range': [zoom_range_in, zoom_range_out], 'channel_shift_range': channel_shift_range, 'fill_mode': fill_mode, 'cval': cval, 'horizontal_flip': horizontal_flip, 'vertical_flip': vertical_flip, 'rescale': rescale} return augmentations
def select_augmentations(): print('Select augmentations:') print(colored('Note: defaults are all zero or false.', 'cyan')) rounds = int_input('number of augmentation rounds', 1, 100) # These three are not implemented because they require training and then that would # need to get propogated over which is complicated for prediction featurewise_center = False #bool_input('featurewise_center: set input mean to 0 over the dataset.') featurewise_std_normalization = False #bool_input('featurewise_std_normalization: divide inputs by std of the dataset.') zca_whitening = False #bool_input('zca_whitening: apply ZCA whitening.') samplewise_center = False #bool_input('samplewise_center: set each sample mean to 0.') samplewise_std_normalization = False #bool_input('samplewise_std_normalization: divide each input by its std.') rotation_range = int_input('rotation_range: degrees', 0, 180, True) width_shift_range = float_input('width_shift_range: fraction of total width.', 0., 1.) height_shift_range = float_input('height_shift_range: fraction of total width.', 0., 1.) shear_range = float_input('shear_range: shear intensity (shear angle in radians)', 0., np.pi/2) zoom_range_in = float_input('zoom_range: amount of zoom in. 1.0 is no zoom, 0 is full zoom.', 0., 1.) zoom_range_out = float_input('zoom_range: amount of zoom out. 1.0 is no zoom, 2.0 is full zoom ', 1., 2.) channel_shift_range = float_input('channel_shift_rangee: shift range for each channels.', 0., 1.) print('fill_mode: points outside the boundaries are filled according to the given mode.') fill_mode = str_input('constant, nearest, reflect, or wrap. Default nearest: ',['constant', 'nearest', 'reflect', 'wrap']) if (fill_mode == 'constant'): cval = float_input('cval: value used for points outside the boundaries', 0., 1.) else: cval = 0.0 horizontal_flip = bool_input('horizontal_flip: whether to randomly flip images horizontally.') vertical_flip = bool_input('vertical_flip: whether to randomly flip images vertically.') rescale = None #int_input('rescale: rescaling factor. If None or 0, no rescaling is applied, otherwise we multiply the data by the value provided.', 0, 255) if rescale == 0: rescale = None augmentations = {'rounds': rounds, 'featurewise_center': featurewise_center, 'featurewise_std_normalization': featurewise_std_normalization, 'samplewise_center': samplewise_center, 'samplewise_std_normalization': samplewise_std_normalization, 'zca_whitening': zca_whitening, 'rotation_range': rotation_range, 'width_shift_range': width_shift_range, 'height_shift_range': height_shift_range, 'shear_range': shear_range, 'zoom_range': [zoom_range_in, zoom_range_out], 'channel_shift_range': channel_shift_range, 'fill_mode': fill_mode, 'cval': cval, 'horizontal_flip': horizontal_flip, 'vertical_flip': vertical_flip, 'rescale': rescale} return augmentations
def select_project(user_provided_project): ''' Select a project from configuration to run transfer on args: user_provided_project (str): Project name that should match a project in the config returns: project (dict): Configuration settings for a user selected project ''' home = os.path.expanduser('~') if os.path.isfile(os.path.join(home, '.transfer', 'config.yaml')): with open(os.path.join(home, '.transfer', 'config.yaml'), 'r') as fp: projects = yaml.load(fp.read()) if len(projects) == 1: project = projects[0] else: if user_provided_project in [ project['name'] for project in projects ]: for inner_project in projects: if user_provided_project == inner_project['name']: project = inner_project else: print('Select your project') for i, project in enumerate(projects): print('[' + str(i) + ']: ' + project['name']) project_index = int_input('project', -1, len(projects), show_range=False) project = projects[project_index] else: print('Transfer is not configured.') print('Please run:') print('') print(colored(' transfer --configure', 'green')) return print(colored('Project selected: ' + project['name'], 'cyan')) return project
def select_project(user_provided_project): ''' Select a project from configuration to run transfer on args: user_provided_project (str): Project name that should match a project in the config returns: project (dict): Configuration settings for a user selected project ''' home = os.path.expanduser('~') if os.path.isfile(os.path.join(home, '.transfer', 'config.yaml')): with open(os.path.join(home, '.transfer', 'config.yaml'), 'r') as fp: projects = yaml.load(fp.read()) if len(projects) == 1: project = projects[0] else: if user_provided_project in [project['name'] for project in projects]: for inner_project in projects: if user_provided_project == inner_project['name']: project = inner_project else: print('Select your project') for i, project in enumerate(projects): print('[' + str(i) + ']: ' + project['name']) project_index = int_input('project', -1, len(projects), show_range = False) project = projects[project_index] else: print('Transfer is not configured.') print('Please run:') print('') print(colored(' transfer --configure', 'green')) return print(colored('Project selected: ' + project['name'], 'cyan')) return project
def configure(): ''' Configure the transfer environment and store ''' completer = Completer() readline.set_completer_delims('\t') readline.parse_and_bind('tab: complete') readline.set_completer(completer.path_completer) home = os.path.expanduser('~') if os.path.isfile(os.path.join(home, '.transfer', 'config.yaml')): with open(os.path.join(home, '.transfer', 'config.yaml'), 'r') as fp: config = yaml.load(fp.read()) else: config = [] project_name = input('Name your project: ') existing_project = None for project in config: if project_name == project['name']: existing_project = project_name if existing_project is not None: print(colored('Project ' + project_name + ' already exists', 'red')) overwrite = str_input( 'Would you like to overwrite this project? (yes or no) ', ['yes', 'no']) if overwrite == 'no': return else: config = [ project for project in config if project_name != project['name'] ] image_path = os.path.expanduser( input('Select parent directory for your images: ')) path_unset = True while path_unset: project_path = os.path.expanduser( input('Select destination for your project: ')) if (project_path.find(image_path) == 0): print( 'Project destination should not be same or within image directory!' ) else: path_unset = False print('Select architecture:') print('[0] resnet50') print('[1] xception') print('[2] inception_v3') architecture = int_input('choice', 0, 2, show_range=False) if architecture == 0: arch = 'resnet50' img_dim = 224 conv_dim = 7 final_cutoff = 80 elif architecture == 1: arch = 'xception' img_dim = 299 conv_dim = 10 final_cutoff = 80 else: arch = 'inception_v3' img_dim = 299 conv_dim = 8 final_cutoff = 80 api_port = int_input('port for local prediction API (suggested: 5000)', 1024, 49151) kfold = int_input('number of folds to use (suggested: 5)', 3, 10) kfold_every = bool_input( 'Fit a model for every fold? (if false, just fit one)') print( 'Warning: if working on a remote computer, you may not be able to plot!' ) plot_cm = bool_input('Plot a confusion matrix after training?') batch_size = int_input('batch size (suggested: 8)', 1, 64) learning_rate = float_input('learning rate (suggested: 0.001)', 0, 1) learning_rate_decay = float_input( 'learning decay rate (suggested: 0.000001)', 0, 1) cycle = int_input( 'number of cycles before resetting the learning rate (suggested: 3)', 1, 10) num_rounds = int_input('number of rounds (suggested: 3)', 1, 100) print('Select image resolution:') print('[0] low (' + str(img_dim) + ' px)') print('[1] mid (' + str(img_dim * 2) + ' px)') print('[2] high (' + str(img_dim * 4) + ' px)') img_resolution_index = int_input('choice', 0, 2, show_range=False) if img_resolution_index == 0: img_size = 1 elif img_resolution_index == 1: img_size = 2 else: img_size = 4 use_augmentation = str_input( 'Would you like to add image augmentation? (yes or no) ', ['yes', 'no']) if use_augmentation == 'yes': augmentations = select_augmentations() else: augmentations = None project = { 'name': project_name, 'img_path': image_path, 'path': project_path, 'plot': plot_cm, 'api_port': api_port, 'kfold': kfold, 'kfold_every': kfold_every, 'cycle': cycle, 'seed': np.random.randint(9999), 'batch_size': batch_size, 'learning_rate': learning_rate, 'learning_rate_decay': learning_rate_decay, 'final_cutoff': final_cutoff, 'rounds': num_rounds, 'img_size': img_size, 'augmentations': augmentations, 'architecture': arch, 'img_dim': img_dim, 'conv_dim': conv_dim, 'is_split': False, 'is_array': False, 'is_augmented': False, 'is_pre_model': False, 'is_final': False, 'model_round': 0, 'server_weights': None, 'last_weights': None, 'best_weights': None } config.append(project) store_config(config) print('') print(colored('Project configure saved!', 'cyan')) print('') print('To run project:') print('') print(colored(' transfer --run --project ' + project_name, 'green')) print('or') print(colored(' transfer -r -p ' + project_name, 'green'))
def configure_server(): ''' Configure the transfer environment and store ''' home = os.path.expanduser('~') if os.path.isfile(os.path.join(home, '.transfer', 'config.yaml')): with open(os.path.join(home, '.transfer', 'config.yaml'), 'r') as fp: config = yaml.load(fp.read()) else: config = [] project_name = input('Name your project: ') existing_project = None for project in config: if project_name == project['name']: existing_project = project_name if existing_project is not None: print(colored('Project ' + project_name + ' already exists', 'red')) overwrite = str_input( 'Would you like to overwrite this project? (yes or no) ', ['yes', 'no']) if overwrite == 'no': return else: config = [ project for project in config if project_name != project['name'] ] api_port = int_input('port for local prediction API (suggested: 5000)', 1024, 49151) print('Select image resolution:') print('[0] low (224 px)') print('[1] mid (448 px)') print('[2] high (896 px)') img_resolution_index = int_input('choice', 0, 2, show_range=False) if img_resolution_index == 0: img_size = 1 elif img_resolution_index == 1: img_size = 2 else: img_size = 4 num_categories = int_input('number of image categories in your model', 0, 10000000) weights = False while weights == False: server_weights = os.path.expanduser(input('Select weights file: ')) if os.path.isfile(server_weights): weights = True else: print('Cannot find the weight file: ', server_weights) project = { 'name': project_name, 'api_port': api_port, 'img_size': img_size, 'number_categories': num_categories, 'server_weights': server_weights } config.append(project) store_config(config) print('') print(colored('Project configure saved!', 'cyan')) print('') print('To start the server:') print('') print( colored(' transfer --prediction-rest-api --project ' + project_name, 'green')) print('or') print( colored(' transfer --prediction-rest-api -p ' + project_name, 'green'))
def configure_server(): ''' Configure the transfer environment and store ''' home = os.path.expanduser('~') if os.path.isfile(os.path.join(home, '.transfer', 'config.yaml')): with open(os.path.join(home, '.transfer', 'config.yaml'), 'r') as fp: config = yaml.load(fp.read()) else: config = [] project_name = input('Name your project: ') existing_project = None for project in config: if project_name == project['name']: existing_project = project_name if existing_project is not None: print(colored('Project ' + project_name + ' already exists', 'red')) overwrite = str_input('Would you like to overwrite this project? (yes or no) ', ['yes', 'no']) if overwrite == 'no': return else: config = [project for project in config if project_name != project['name']] api_port = int_input('port for local prediction API (suggested: 5000)', 1024, 49151) print('Select image resolution:') print('[0] low (224 px)') print('[1] mid (448 px)') print('[2] high (896 px)') img_resolution_index = int_input('choice', 0, 2, show_range = False) if img_resolution_index == 0: img_size = 1 elif img_resolution_index == 1: img_size = 2 else: img_size = 4 num_categories = int_input('number of image categories in your model', 0, 10000000) weights = False while weights == False: server_weights = os.path.expanduser(input('Select weights file: ')) if os.path.isfile(server_weights): weights = True else: print('Cannot find the weight file: ', server_weights) project = {'name': project_name, 'api_port': api_port, 'img_size': img_size, 'number_categories': num_categories, 'server_weights': server_weights} config.append(project) store_config(config) print('') print(colored('Project configure saved!', 'cyan')) print('') print('To start the server:') print('') print(colored(' transfer --prediction-rest-api --project ' + project_name, 'green')) print('or') print(colored(' transfer --prediction-rest-api -p ' + project_name, 'green'))
def configure(): ''' Configure the transfer environment and store ''' home = os.path.expanduser('~') if os.path.isfile(os.path.join(home, '.transfer', 'config.yaml')): with open(os.path.join(home, '.transfer', 'config.yaml'), 'r') as fp: config = yaml.load(fp.read()) else: config = [] project_name = input('Name your project: ') existing_project = None for project in config: if project_name == project['name']: existing_project = project_name if existing_project is not None: print(colored('Project ' + project_name + ' already exists', 'red')) overwrite = str_input('Would you like to overwrite this project? (yes or no) ', ['yes', 'no']) if overwrite == 'no': return else: config = [project for project in config if project_name != project['name']] image_path = os.path.expanduser(input('Select parent directory for your images: ')) path_unset = True while path_unset: project_path = os.path.expanduser(input('Select destination for your project: ')) if (project_path.find(image_path) == 0): print('Project destination should not be same or within image directory!') else: path_unset = False print('Select architecture:') print('[0] resnet50') print('[1] xception') print('[2] inception_v3') architecture = int_input('choice', 0, 2, show_range = False) if architecture == 0: arch = 'resnet50' img_dim = 224 conv_dim = 7 final_cutoff = 80 elif architecture == 1: arch = 'xception' img_dim = 299 conv_dim = 10 final_cutoff = 80 else: arch = 'inception_v3' img_dim = 299 conv_dim = 8 final_cutoff = 80 api_port = int_input('port for local prediction API (suggested: 5000)', 1024, 49151) kfold = int_input('number of folds to use (suggested: 5)', 3, 10) kfold_every = bool_input('Fit a model for every fold? (if false, just fit one)') print('Warning: if working on a remote computer, you may not be able to plot!') plot_cm = bool_input('Plot a confusion matrix after training?') batch_size = int_input('batch size (suggested: 8)', 1, 64) learning_rate = float_input('learning rate (suggested: 0.001)', 0, 1) learning_rate_decay = float_input('learning decay rate (suggested: 0.000001)', 0, 1) cycle = int_input('number of cycles before resetting the learning rate (suggested: 3)', 1, 10) num_rounds = int_input('number of rounds (suggested: 3)', 1, 100) print('Select image resolution:') print('[0] low (' + str(img_dim) + ' px)') print('[1] mid (' + str(img_dim * 2) + ' px)') print('[2] high (' + str(img_dim * 4) + ' px)') img_resolution_index = int_input('choice', 0, 2, show_range = False) if img_resolution_index == 0: img_size = 1 elif img_resolution_index == 1: img_size = 2 else: img_size = 4 use_augmentation = str_input('Would you like to add image augmentation? (yes or no) ', ['yes', 'no']) if use_augmentation == 'yes': augmentations = select_augmentations() else: augmentations = None project = {'name': project_name, 'img_path': image_path, 'path': project_path, 'plot': plot_cm, 'api_port': api_port, 'kfold': kfold, 'kfold_every': kfold_every, 'cycle': cycle, 'seed': np.random.randint(9999), 'batch_size': batch_size, 'learning_rate': learning_rate, 'learning_rate_decay': learning_rate_decay, 'final_cutoff': final_cutoff, 'rounds': num_rounds, 'img_size': img_size, 'augmentations': augmentations, 'architecture': arch, 'img_dim': img_dim, 'conv_dim': conv_dim, 'is_split': False, 'is_array': False, 'is_augmented': False, 'is_pre_model': False, 'is_final': False, 'model_round': 0, 'server_weights': None, 'last_weights': None, 'best_weights': None} config.append(project) store_config(config) print('') print(colored('Project configure saved!', 'cyan')) print('') print('To run project:') print('') print(colored(' transfer --run --project ' + project_name, 'green')) print('or') print(colored(' transfer -r -p ' + project_name, 'green'))
def read_imported_config(project_path, project_name, projects=None): completer = Completer() readline.set_completer_delims('\t') readline.parse_and_bind('tab: complete') readline.set_completer(completer.path_completer) # Oh god this logic is a disaster, user interfaces are hard bad_user = True while bad_user: relearn_str = str_input( 'Do you want to learn on new starting from these weights? (yes or no) ' ) if relearn_str.lower() == 'yes' or relearn_str.lower() == 'y': bad_user = False relearn = True elif relearn_str.lower() == 'no' or relearn_str.lower() == 'n': bad_user = False relearn = False unique_name = False while unique_name == False: unique_name = True if projects is not None: for project in projects: if project['name'] == project_name: print( colored('Project with this name already exists.', 'red')) project_name = str_input('Provide a new project name: ') unique_name = False if relearn: image_path = os.path.expanduser( input('Select parent directory for your images: ')) path_unset = True while path_unset: project_dest = os.path.expanduser( input('Select destination for your project: ')) if (project_dest.find(image_path) == 0): print( 'Project destination should not be same or within image directory!' ) else: path_unset = False else: project_dest = os.path.expanduser( input('Select destination for your project: ')) if os.path.isdir(project_dest) == False: print('Creating directory:', project_dest) os.makedirs(project_dest, exist_ok=True) # You don't get to judge me t('-' t) with open(os.path.join(project_path, 'config.yaml'), 'r') as fp: import_project = yaml.load(fp.read()) import_project['name'] = project_name import_project['path'] = project_dest if relearn: kfold = int_input('number of folds to use (suggested: 5)', 3, 10) kfold_every = bool_input( 'Fit a model for every fold? (if false, just fit one)') print( 'Warning: if working on a remote computer, you may not be able to plot!' ) plot_cm = bool_input('Plot a confusion matrix after training?') batch_size = int_input('batch size (suggested: 8)', 1, 64) learning_rate = float_input('learning rate (suggested: 0.001)', 0, 1) learning_rate_decay = float_input( 'learning decay rate (suggested: 0.000001)', 0, 1) cycle = int_input( 'number of cycles before resetting the learning rate (suggested: 3)', 1, 10) num_rounds = int_input('number of rounds (suggested: 3)', 1, 100) import_project['img_path'] = image_path import_project['best_weights'] = [ os.path.join(project_path, weight) for weight in os.listdir(project_path) if weight.find('.hdf5') > 0 ] import_project['last_weights'] = import_project['best_weights'] import_project['server_weights'] = None import_project['kfold'] = kfold import_project['kfold_every'] = kfold_every import_project['cycle'] = cycle import_project['seed'] = np.random.randint(9999) import_project['batch_size'] = batch_size import_project['learning_rate'] = learning_rate import_project['learning_rate_decay'] = learning_rate_decay if 'final_cutoff' not in import_project.keys(): import_project['final_cutoff'] = 80 import_project['rounds'] = num_rounds import_project['is_split'] = False import_project['is_array'] = False import_project['is_augmented'] = False import_project['is_pre_model'] = False import_project['model_round'] = 1 import_project['plot'] = plot_cm print('') print('To re-learn new images with project:') print('') print(colored(' transfer --run --project ' + project_name, 'green')) print('or') print(colored(' transfer -r -p ' + project_name, 'green')) print('') else: import_project['server_weights'] = [ os.path.join(project_path, weight) for weight in os.listdir(project_path) if weight.find('.hdf5') > 0 ] return import_project