def __init__(self, verbose=False, path=None, resume=False, searcher_args=None, search_type=BayesianSearcher): """Initialize the instance. The classifier will be loaded from the files in 'path' if parameter 'resume' is True. Otherwise it would create a new one. Args: verbose: A boolean of whether the search process will be printed to stdout. path: A string. The path to a directory, where the intermediate results are saved. resume: A boolean. If True, the classifier will continue to previous work saved in path. Otherwise, the classifier will start a new search. searcher_args: A dictionary containing the parameters for the searcher's __init__ function. search_type: A constant denoting the type of hyperparameter search algorithm that must be used. """ super().__init__(verbose) if searcher_args is None: searcher_args = {} if path is None: path = rand_temp_folder_generator() self.path = path ensure_dir(path) if resume: classifier = pickle_from_file(os.path.join(self.path, 'classifier')) self.__dict__ = classifier.__dict__ self.cnn = pickle_from_file(os.path.join(self.path, 'module')) else: self.y_encoder = None self.data_transformer = None self.verbose = verbose self.cnn = CnnModule(self.loss, self.metric, searcher_args, path, verbose, search_type)
def __init__(self, verbose=False, path=None, resume=False, searcher_args=None, search_type=BayesianSearcher): """Initialize the instance. The classifier will be loaded from the files in 'path' if parameter 'resume' is True. Otherwise it would create a new one. Args: verbose: A boolean of whether the search process will be printed to stdout. path: A string. The path to a directory, where the intermediate results are saved. resume: A boolean. If True, the classifier will continue to previous work saved in path. Otherwise, the classifier will start a new search. searcher_args: A dictionary containing the parameters for the searcher's __init__ function. search_type: A constant denoting the type of hyperparameter search algorithm that must be used. """ super().__init__(verbose) if searcher_args is None: searcher_args = {} if path is None: path = rand_temp_folder_generator() self.path = path ensure_dir(path) if resume: classifier = pickle_from_file(os.path.join(self.path, 'classifier')) self.__dict__ = classifier.__dict__ self.cnn = pickle_from_file(os.path.join(self.path, 'module')) else: self.y_encoder = None self.data_transformer = None self.verbose = verbose self.cnn = CnnModule(self.loss, self.metric, searcher_args, path, verbose, search_type)
def text_preprocess(x_train): """This is the text preprocess main method. It takes an raw string, clean it and processing it into tokenlized numpy array. """ if Constant.STORE_PATH == '': temp_path = temp_path_generator() path = temp_path + '_store' else: path = Constant.STORE_PATH ensure_dir(path) x_train = [clean_str(x) for x in x_train] x_train, word_index = tokenlize_text( max_seq_length=Constant.MAX_SEQUENCE_LENGTH, max_num_words=Constant.MAX_NB_WORDS, x_train=x_train) print("generating preprocessing model...") x_train = processing(path=path, word_index=word_index, input_length=Constant.MAX_SEQUENCE_LENGTH, x_train=x_train) return x_train
def __init__(self, verbose=False, path=constant.DEFAULT_SAVE_PATH, resume=False, searcher_args=None): """Initialize the instance. The classifier will be loaded from the files in 'path' if parameter 'resume' is True. Otherwise it would create a new one. Args: verbose: An boolean of whether the search process will be printed to stdout. path: A string. The path to a directory, where the intermediate results are saved. resume: An boolean. If True, the classifier will continue to previous work saved in path. Otherwise, the classifier will start a new search. """ if searcher_args is None: searcher_args = {} if has_file(os.path.join(path, 'classifier')) and resume: classifier = pickle_from_file(os.path.join(path, 'classifier')) self.__dict__ = classifier.__dict__ self.path = path else: self.y_encoder = None self.data_transformer = None self.verbose = verbose self.searcher = False self.path = path self.searcher_args = searcher_args ensure_dir(path)
def __init__(self, model_path=None, overwrite=False): super(VoiceGenerator, self).__init__() self.model_path = model_path if model_path is not None else temp_path_generator() ensure_dir(self.model_path) self.checkpoint_path = os.path.join(self.model_path, Constant.PRE_TRAIN_VOICE_GENERATOR_MODEL_NAME) self.sample_rate = 0 self.hop_length = 0 self.overwrite = overwrite self.device = get_device() self.load()
def __init__(self, model_path=None, overwrite=False): super(VoiceGenerator, self).__init__() if model_path is None: model_path = temp_path_generator() self.model_path = model_path ensure_dir(self.model_path) self.checkpoint_path = os.path.join(self.model_path, Constant.PRE_TRAIN_VOICE_GENERATOR_MODEL_NAME) self.sample_rate = 0 self.hop_length = 0 self.overwrite = overwrite self.load()
def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False, search_type=BayesianSearcher): self.searcher_args = searcher_args if searcher_args is not None else {} self.searcher = None self.path = path if path is not None else rand_temp_folder_generator() ensure_dir(self.path) if verbose: print('Saving Directory:', self.path) self.verbose = verbose self.loss = loss self.metric = metric self.generators = [] self.search_type = search_type
def __init__(self, loss, metric, searcher_args=None, path=None, verbose=False, search_type=BayesianSearcher): self.searcher_args = searcher_args if searcher_args is not None else {} self.searcher = None self.path = path if path is not None else rand_temp_folder_generator() ensure_dir(self.path) if verbose: print('Saving Directory:', self.path) self.verbose = verbose self.loss = loss self.metric = metric self.generators = [] self.search_type = search_type
def load(self, model_path=None): temp_path = temp_path_generator() ensure_dir(temp_path) model_paths = [ f'{temp_path}/{file_name}' for file_name in Constant.FACE_DETECTOR['MODEL_NAMES'] ] for google_id, file_name in zip( Constant.FACE_DETECTOR['MODEL_GOOGLE_ID'], Constant.FACE_DETECTOR['MODEL_NAMES']): download_file_from_google_drive( file_id=google_id, dest_path=f'{temp_path}/{file_name}') return model_paths
def __init__(self, path=None, **kwargs): """ Initialization function for tabular supervised learner. """ super().__init__(**kwargs) self.is_trained = False self.clf = None self.objective = None self.tabular_preprocessor = None self.path = path if path is not None else rand_temp_folder_generator() ensure_dir(self.path) if self.verbose: print('Path:', path) self.save_filename = os.path.join(self.path, 'lgbm.txt') self.time_limit = None self.lgbm = None
def __init__(self, path=None, **kwargs): """ Initialization function for tabular supervised learner. """ super().__init__(**kwargs) self.is_trained = False self.clf = None self.objective = None self.tabular_preprocessor = None self.path = path if path is not None else rand_temp_folder_generator() ensure_dir(self.path) if self.verbose: print('Path:', path) self.save_filename = os.path.join(self.path, 'lgbm.txt') self.time_limit = None self.lgbm = None
def __init__(self, verbose=False, path=None, resume=False, searcher_args=None, augment=None): """Initialize the instance. The classifier will be loaded from the files in 'path' if parameter 'resume' is True. Otherwise it would create a new one. Args: verbose: A boolean of whether the search process will be printed to stdout. path: A string. The path to a directory, where the intermediate results are saved. resume: A boolean. If True, the classifier will continue to previous work saved in path. Otherwise, the classifier will start a new search. augment: A boolean value indicating whether the data needs augmentation. If not define, then it will use the value of Constant.DATA_AUGMENTATION which is True by default. """ super().__init__(verbose) if searcher_args is None: searcher_args = {} if path is None: path = temp_folder_generator() self.cnn = CnnModule(self.loss, self.metric, searcher_args, path, verbose) if augment is None: augment = Constant.DATA_AUGMENTATION if has_file(os.path.join(path, 'classifier')) and resume: classifier = pickle_from_file(os.path.join(path, 'classifier')) self.__dict__ = classifier.__dict__ self.path = path else: self.y_encoder = None self.data_transformer = None self.verbose = verbose self.searcher = False self.path = path self.searcher_args = searcher_args self.augment = augment ensure_dir(path)
def __init__(self, verbose=True, model_path=None): """Initialize the instance.""" self.verbose = verbose self.model = None self.device = get_device() self.model_path = model_path if model_path is not None else temp_path_generator( ) ensure_dir(self.model_path) self.local_paths = [ os.path.join(self.model_path, x.local_name) for x in self._google_drive_files ] for path, x in zip(self.local_paths, self._google_drive_files): if not os.path.exists(path): download_file_from_google_drive(file_id=x.google_drive_id, dest_path=path, verbose=True)
def load(self, model_path=None): if model_path is None: model_file_name = Constant.OBJECT_DETECTOR['MODEL_NAME'] temp_path = temp_path_generator() ensure_dir(temp_path) model_path = f'{temp_path}/{model_file_name}' download_file_from_google_drive(file_id=Constant.OBJECT_DETECTOR['MODEL_GOOGLE_ID'], dest_path=model_path) # load net num_classes = len(VOC_CLASSES) + 1 # +1 for background self.model = self._build_ssd('test', 300, num_classes) # initialize SSD if self.device.startswith("cuda"): self.model.load_state_dict(torch.load(model_path)) else: self.model.load_state_dict(torch.load(model_path, map_location=lambda storage, loc: storage)) self.model.eval() print('Finished loading model!') self.model = self.model.to(self.device)
def text_preprocess(x_train): """This is the text preprocess main method. It takes an raw string, clean it and processing it into tokenlized numpy array. """ if Constant.STORE_PATH == '': temp_path = temp_path_generator() path = temp_path + '_store' else: path = Constant.STORE_PATH ensure_dir(path) x_train = [clean_str(x) for x in x_train] x_train, word_index = tokenlize_text(max_seq_length=Constant.MAX_SEQUENCE_LENGTH, max_num_words=Constant.MAX_NB_WORDS, x_train=x_train) print("generating preprocessing model...") x_train = processing(path=path, word_index=word_index, input_length=Constant.MAX_SEQUENCE_LENGTH, x_train=x_train) return x_train
def __init__(self, verbose=False, searcher_type=None, path=constant.DEFAULT_SAVE_PATH, resume=False): """Initialize the instance. The classifier will be loaded from file if the directory in 'path' has a saved classifier. Otherwise it would create a new one. """ if has_file(os.path.join(path, 'classifier')) and resume: classifier = pickle.load( open(os.path.join(path, 'classifier'), 'rb')) self.__dict__ = classifier.__dict__ else: self.y_encoder = None self.verbose = verbose self.searcher = False self.searcher_type = searcher_type self.path = path ensure_dir(path)
def __init__(self, verbose=False, path=None, resume=False, searcher_args=None, augment=None): """Initialize the instance. The classifier will be loaded from the files in 'path' if parameter 'resume' is True. Otherwise it would create a new one. Args: verbose: A boolean of whether the search process will be printed to stdout. path: A string. The path to a directory, where the intermediate results are saved. resume: A boolean. If True, the classifier will continue to previous work saved in path. Otherwise, the classifier will start a new search. augment: A boolean value indicating whether the data needs augmentation. If not define, then it will use the value of Constant.DATA_AUGMENTATION which is True by default. """ super().__init__(verbose) if searcher_args is None: searcher_args = {} if path is None: path = temp_folder_generator() if augment is None: augment = Constant.DATA_AUGMENTATION if has_file(os.path.join(path, 'classifier')) and resume: classifier = pickle_from_file(os.path.join(path, 'classifier')) self.__dict__ = classifier.__dict__ self.path = path else: self.y_encoder = None self.data_transformer = None self.verbose = verbose self.searcher = False self.path = path self.searcher_args = searcher_args self.augment = augment ensure_dir(path)
def load(self, model_path=None): temp_path = temp_path_generator() ensure_dir(temp_path) for model_link, file_path in zip(Constant.FACE_DETECTION_PRETRAINED['PRETRAINED_MODEL_LINKS'], Constant.FACE_DETECTION_PRETRAINED['FILE_NAMES']): download_file(model_link, f'{temp_path}/{file_path}')
from keras.datasets import mnist import os from pandas import DataFrame from PIL import Image from autokeras.utils import ensure_dir ensure_dir('mnist/train') ensure_dir('mnist/test') (x_train, y_train), (x_test, y_test) = mnist.load_data() # x_train = x_train.reshape(x_train.shape + (1,)) # x_test = x_test.reshape(x_test.shape + (1,)) # file_names = [] # for i in range(len(x_train)): # file_name = ("%05d" % (i, )) + '.jpg' # Image.fromarray(x_train[i]).save(os.path.join('mnist', 'train', file_name)) # file_names.append(file_name) # # csv_data = {'File Name': file_names, 'Label': y_train} # DataFrame(csv_data).to_csv('mnist/train/label.csv', index=False) file_names = [] for i in range(len(x_test)): file_name = ("%05d" % (i, )) + '.jpg' Image.fromarray(x_test[i]).save(os.path.join('mnist', 'test', file_name)) file_names.append(file_name) csv_data = {'File Name': file_names, 'Label': y_test} DataFrame(csv_data).to_csv('mnist/test/label.csv', index=False)