def create_geo_alt_names(geo_type): geo_alt_names = {} for locale in settings.locales: name_format = geo_type.__name__ + '{}' + locale.capitalize() name = name_format.format('AltName') geo_alt_names[locale] = create_model( name = name, fields = { 'geo': models.ForeignKey(geo_type, # Related geo type related_name = 'alt_names_' + locale), 'name': models.CharField(max_length=200, db_index=True), # Alternate name 'is_preferred': models.BooleanField(), # True if this alternate name is an official / preferred name 'is_short': models.BooleanField(), # True if this is a short name like 'California' for 'State of California' 'objects': GeoAltNameManager(), '__unicode__': lambda self: force_unicode(self.name), }, app_label = 'cities', module = 'cities.models', options = { 'db_table': 'cities_' + un_camel(name), 'verbose_name': un_camel(name).replace('_', ' '), 'verbose_name_plural': un_camel(name_format.format('AltNames')).replace('_', ' '), }, ) return geo_alt_names
def __init__(self, args, clients, test_loader=None): self.args = args self.comm_rounds = args.comm_rounds self.num_clients = args.num_clients self.frac = args.frac self.clients = clients self.client_data_num = [] self.elapsed_comm_rounds = 0 self.accuracies = np.zeros(args.comm_rounds) self.client_accuracies = np.zeros( (self.args.num_clients, self.args.comm_rounds)) self.selected_client_tally = np.zeros( (self.args.comm_rounds, self.args.num_clients)) self.test_loader = test_loader for client in self.clients: self.client_data_num.append(len(client.train_loader)) self.client_data_num = np.array(self.client_data_num) # The extra 1 entry in client_models and global_models are used to store # the results after last communication round self.client_models = np.zeros((self.comm_rounds + 1, self.num_clients), dtype='object') self.global_models = None # np.zeros((self.comm_rounds + 1,), dtype='object') self.init_model = create_model(args.dataset, args.arch) self.global_models = self.init_model self.global_init_model = copy_model(self.init_model, args.dataset, args.arch) assert self.num_clients == len( clients ), "Number of client objects does not match command line input"
def main(): args = parse_args('predict') device = torch.device("cuda" if torch.cuda.is_available() else "cpu") transform = transforms.Compose([ transforms.Resize(224), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), ]) test_set = DataLoader(args.ext_test, transform=transform) test_loader = torch.utils.data.DataLoader(test_set, batch_size=args.batch_size, shuffle=False) print("==> creating model") num_classes = args.num_classes model = create_model(num_classes, args).to(device) if args.average: record = pd.read_csv(os.path.join(args.output_csv_dir, 'output.csv'), index_col=0) sorted_r = record.sort_values('f1', ascending=False) model_list = list(sorted_r['epoch_num'].astype(int))[:5] for idx, epoch in enumerate(model_list): print('\nPredict with ' f'epoch{epoch} checkpoint') model = load_checkpoint(model, os.path.join(args.save_model_dir, f'epoch{epoch}_checkpoint.pth.tar')).to(device) predict(test_loader, model, device, args, idx) else: predict(test_loader, model, device, args) predict(test_loader, model, device, args)
def test5_test_model_list_size(): import os from util import log_obj, create_model models = [create_model('mnist', 'mlp') for _ in range(400)] log_obj('./log_test/server/400_models_list.model_list', models) filesize = os.path.getsize('./log_test/server/400_models_list.model_list') byte_per_gb = 1E9 print(f'The log file size is {filesize/byte_per_gb}GB')
def main(): args = parse_args('test') device = torch.device("cuda" if torch.cuda.is_available() else "cpu") transform_test = transforms.Compose([ transforms.Resize(224), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), ]) test_set = DataLoader(args.test_list, transform=transform_test) test_loader = torch.utils.data.DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) # Model print("==> creating model") num_classes = args.num_classes model = create_model(num_classes, args).to(device) criterion = nn.CrossEntropyLoss() # testing expname = args.expname.split('/')[-1] print('\n' + expname + ': TESTING!') train_set = os.path.basename(args.train_list).split('.')[0] val_set = os.path.basename(args.val_list).split('.')[0] test_set = os.path.basename(args.test_list).split('.')[0] if args.average: record = pd.read_csv(os.path.join(args.output_csv_dir, 'output.csv'), index_col=0) sorted_r = record.sort_values('f1', ascending=False) model_list = list(sorted_r['epoch_num'].astype(int))[:5] df = pd.DataFrame(columns=['exp', 'train', 'val', 'test', 'test_loss', 'test_acc', 'f1']) for idx, epoch in enumerate(model_list): model = load_checkpoint(model, os.path.join(args.save_model_dir, f'epoch{epoch}_checkpoint.pth.tar')) test_loss, test_acc, f1, _ = test(test_loader, model, criterion, device, args, epoch=idx+1) df.loc[len(df)] = [expname, train_set, val_set, test_set, test_loss, test_acc, f1] output_csv_file = os.path.join(args.output_csv_dir, 'test_acc.csv') df.to_csv(output_csv_file, index=False) else: df = pd.DataFrame(columns=['exp', 'train', 'val', 'test', 'test_loss', 'test_acc', 'f1']) model = load_checkpoint(model, os.path.join(save_model_dir, 'model_best_f1.pth.tar')) test_loss, test_acc, f1, _ = test(test_loader, model, criterion, device, args) df.loc[len(df)] = [expname, train_set, val_set, test_set, test_loss, test_acc, f1] output_csv_file = os.path.join(output_csv_dir, 'test_f1.csv') df.to_csv(output_csv_file, index=False)
def main(args): config = create_config() model = create_model(args.mode) dm = create_data_module(config) print(f'Training for {args.max_epochs} epochs') # Output training parameters config.display() trainer = Trainer.from_argparse_args(args) trainer.fit(model, dm)
def __init__(self, args, train_loader, test_loader, client_id=None): self.args = args print("Creating model for client " + str(client_id)) self.model = create_model(self.args.dataset, self.args.arch) print("Copying model for client " + str(client_id)) self.init_model = copy_model(self.model, self.args.dataset, self.args.arch) print("Done Copying model for client " + str(client_id)) self.test_loader = test_loader self.train_loader = train_loader self.client_id = client_id self.elapsed_comm_rounds = 0 self.accuracies = np.zeros((args.comm_rounds, self.args.client_epoch)) self.losses = np.zeros((args.comm_rounds, self.args.client_epoch)) self.prune_rates = np.zeros(args.comm_rounds) assert self.model, "Something went wrong and the model cannot be initialized"
def main(args): with open('classes.json', 'r') as f: classes = json.load(f) # Convert key values to ints classes = {int(k): v for k, v in classes.items()} model = create_model(args.mode, path_to_checkpoint=args.checkpoint_path) model.eval() image = Image.open(args.image_path) tensor = ToTensor()(image).unsqueeze(0) prediction = torch.argmax(model(tensor), dim=1).item() print(f'Predicted class: {classes[prediction]}')
def main(**kwargs): config_name = kwargs.get("config_name", "config.json") path = osp.join(osp.dirname(osp.realpath(__file__)), config_name) config_file = open(path) config = json.load(config_file, object_hook=lambda d: namedtuple('x', d.keys()) (*d.values())) config_file.seek(0) logger.info(str(json.load(config_file))) config_file.close() num_unrolls = config.num_steps // config.unroll_length with tf.Session() as sess: # tf.get_default_graph().finalize() model = util.create_model(sess, config, logger) step, loss, reset, fx_array, x_array = model.step() best_cost = [float('inf')] * 3 epoch_cost = 0 total_cost = 0 for e in range(config.num_epochs): cost, _ = util.run_epoch(sess, loss, [step], reset, num_unrolls) epoch_cost += cost total_cost += cost if (e + 1) % config.log_period == 0: lm_e = epoch_cost / config.log_period logger.info('Epoch {}, Mean Error: {:.3f}'.format(e, lm_e)) epoch_cost = 0 if (e + 1) % config.evaluation_period == 0: elm_e = total_cost / config.evaluation_period logger.info('Current {} epochs, Mean Error: {:.3f}'.format( config.evaluation_period, elm_e)) mbc = max(best_cost) if config.save_path is not None and total_cost < mbc: save_path = osp.join(osp.dirname(osp.realpath(__file__)), config.save_path) best_cost.remove(mbc) best_cost.append(total_cost) logger.info('Save current model ...') model.saver.save(sess, save_path, global_step=e) total_cost = 0
def main(): config_file = open('./config.json') config = json.load(config_file, object_hook=lambda d: namedtuple('x', d.keys()) (*d.values())) config_file.seek(0) logger.info(str(json.load(config_file))) config_file.close() num_unrolls = config.num_steps // config.unroll_length with tf.Session() as sess: # tf.get_default_graph().finalize() model = util.create_model(sess, config, logger) step, loss, reset, fx_array, x_array = model.step() best_evaluation = float('inf') total_cost = 0 for e in range(config.num_epochs): cost, _ = util.run_epoch(sess, loss, [step], reset, num_unrolls) total_cost += cost if (e + 1) % config.log_period == 0: lm_e = total_cost / config.log_period logger.info('Epoch {}, Mean Error: {:.3f}'.format(e, lm_e)) total_cost = 0 if (e + 1) % config.evaluation_period == 0: eval_cost = 0 for _ in range(config.evaluation_epochs): cost, _ = util.run_epoch(sess, loss, [ step, ], reset, num_unrolls) eval_cost += cost elm_e = eval_cost / config.evaluation_epochs logger.info('EVALUATION, Mean Error: {:.3f}'.format(elm_e)) if config.save_path is not None and eval_cost < best_evaluation: logger.info('Save current model ...') model.saver.save(sess, config.save_path, global_step=e) best_evaluation = eval_cost
def AFCNN(x_train, y_train, x_test, y_test): class_num = 4 # 分类的种类 x_train = x_train.reshape(x_train.shape[0], 1, x_train.shape[1], 1) x_test = x_test.reshape(x_test.shape[0], 1, x_test.shape[1], 1) # one-hot Y_train_one_hot = to_categorical(y_train, int(class_num)) # four labels Y_test_one_hot = to_categorical(y_test, int(class_num)) # four labels model = create_model() # 创建CNN模型 # model.fit(x_train, y_train,batch_size = 32, epochs = 10,verbose = 1,validation_data=(test_data,test_target),callbacks=callbacks_list) model.fit(x_train, Y_train_one_hot, batch_size=32, epochs=5, verbose=1, validation_data=(x_test, Y_test_one_hot)) # test_pred,train_pred = stacking(train_data=x_train,train_target=Y_train_one_hot,test_data=x_test,test_target=Y_test_one_hot, n_fold=5) test_prob = model.predict(x_test) test_p = [np.argmax(item) for item in test_prob] # 将矩阵按列合并 score = model.evaluate(x_test, Y_test_one_hot, verbose=1) # print('acc==', score) # tmp_target = [np.argmax(item) for item in Y_test_one_hot] # print(confusion_matrix(tmp_target, test_p)) # return test_p, test_prob return test_p
def main(): print("==> loading model") num_classes = args.num_classes model = create_model(num_classes, args).to(device) model = load_checkpoint(model, checkpoint).to(device) transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), ]) input_data = args.input if os.path.isdir(input_data): input_folder_lst = [ os.path.join(input_data, i) for i in os.listdir(input_data) ] for input_folder in input_folder_lst: print(os.path.basename(input_folder)) input_data = os.path.join(input_folder, 'patch.json') filter_patches(model, input_data, transform) else: filter_patches(model, input_data, transform)
def create_geo_alt_names(geo_type): geo_alt_names = {} for locale in settings.locales: name_format = geo_type.__name__ + "{}" + locale.capitalize() name = name_format.format("AltName") geo_alt_names[locale] = create_model( name=name, fields={ "geo": models.ForeignKey(geo_type, related_name="alt_names_" + locale), # Related geo type "name": models.CharField(max_length=200, db_index=True), # Alternate name "is_preferred": models.BooleanField(), # True if this alternate name is an official / preferred name "is_short": models.BooleanField(), # True if this is a short name like 'California' for 'State of California' "objects": GeoAltNameManager(), "__unicode__": lambda self: force_unicode(self.name), }, app_label="cities", module="cities.models", options={ "db_table": "cities_" + un_camel(name), "verbose_name": un_camel(name).replace("_", " "), "verbose_name_plural": un_camel(name_format.format("AltNames")).replace("_", " "), }, ) return geo_alt_names
default='gru', choices=['gru', 'lstm'], help='type of recurrent layer to use (gru or lstm)') parser.add_argument('--model_weights', help='pickle file of trained weights') args = parser.parse_args(gen_be=False) # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) be.bsz = 1 # load the bAbI dataset babi = babi_handler(args.data_dir, args.task) valid_set = QA(*babi.test) # create model model_inference = create_model(babi.vocab_size, args.rlayer_type) model_inference.load_params(args.model_weights) model_inference.initialize(dataset=valid_set) ex_story, ex_question, ex_answer = babi.test_parsed[0] stitch_sentence = lambda words: \ " ".join(words).replace(" ?", "?").replace(" .", ".\n").replace("\n ", "\n") print "\nThe vocabulary set from this task has {} words:".format( babi.vocab_size) print stitch_sentence(babi.vocab) print "\nExample from test set:" print "\nStory" print stitch_sentence(ex_story) print "Question" print stitch_sentence(ex_question) print "\nAnswer"
def create_postal_codes(): @property def parent(self): for parent_name in reversed(['country'] + RegionBase.levels): parent_obj = getattr(self, parent_name) if parent_obj: return parent_obj return None @property def hierarchy(self): """Get hierarchy, root first""" list = self.parent.hierarchy list.append(self) return list @property def names(self): """Get a hierarchy of non-null names, root first""" return [e for e in [ force_unicode(self.country), force_unicode(self.region_name), force_unicode(self.subregion_name), force_unicode(self.district_name), force_unicode(self.name), ] if e] @property def name_full(self): """Get full name including hierarchy""" return u', '.join(reversed(self.names)) postal_codes = {} for country in settings.postal_codes: name_format = "{}" + country name = name_format.format('PostalCode') postal_codes[country] = create_model( name = name, fields = { 'country': models.ForeignKey(Country, related_name = 'postal_codes_' + country), 'code': models.CharField(max_length=20, primary_key=True), 'name': models.CharField(max_length=200, db_index=True), # Region names for each admin level, region may not exist in DB 'region_name': models.CharField(max_length=100, db_index=True), 'subregion_name': models.CharField(max_length=100, db_index=True), 'district_name': models.CharField(max_length=100, db_index=True), 'region': models.ForeignKey(Region, null=True, blank=True, related_name = 'postal_codes_' + country), 'subregion': models.ForeignKey(Subregion, null=True, blank=True, related_name = 'postal_codes_' + country), 'location': models.PointField(), 'objects': models.GeoManager(), 'parent': parent, 'hierarchy': hierarchy, 'names': names, 'name_full': name_full, '__unicode__': lambda self: force_unicode(self.code), }, app_label = 'cities', module = 'cities.models', options = { 'db_table': 'cities_' + un_camel(name), 'verbose_name': un_camel(name).replace('_', ' '), 'verbose_name_plural': un_camel(name_format.format('PostalCodes')).replace('_', ' '), }, ) return postal_codes
# # Model Running # ###################################################### logger.info("building model") config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: initializer = tf.contrib.layers.xavier_initializer() with tf.variable_scope("model", reuse=None, initializer=initializer): model = util.create_model(session=sess, Model=BILSTM_CRF, ckpt_file=save_path, labelKey=labelKey, label2Idx=label2Idx, word2Idx=word2Idx, num_steps=num_steps, num_epochs=num_epochs, embedding_matrix=embeddings, logger=logger) logger.info("training model") logger.info("start training") model.train(sess, save_path, X_train, y_train, X_dev, y_dev, save_model_name, logger) print("final best f1 is: %f" % (model.max_f1)) end_time = time.time() print("time used %f(hour)" % ((end_time - start_time) / 3600))
parser.add_argument('--rlayer_type', default='gru', choices=['gru', 'lstm'], help='type of recurrent layer to use (gru or lstm)') parser.add_argument('--model_weights', help='pickle file of trained weights') args = parser.parse_args(gen_be=False) # setup backend be = gen_backend(**extract_valid_args(args, gen_backend)) be.bsz = 1 # load the bAbI dataset babi = babi_handler(args.data_dir, args.task) valid_set = QA(*babi.test) # create model model_inference = create_model(babi.vocab_size, args.rlayer_type) model_inference.load_params(args.model_weights) model_inference.initialize(dataset=valid_set) ex_story, ex_question, ex_answer = babi.test_parsed[0] stitch_sentence = lambda words: \ " ".join(words).replace(" ?", "?").replace(" .", ".\n").replace("\n ", "\n") print "\nThe vocabulary set from this task has {} words:".format(babi.vocab_size) print stitch_sentence(babi.vocab) print "\nExample from test set:" print "\nStory" print stitch_sentence(ex_story) print "Question" print stitch_sentence(ex_question) print "\nAnswer" print ex_answer
# Set this variable to your MLflow server's DNS name mlflow_server = '178.128.106.223' # Tracking URI mlflow_tracking_uri = 'http://' + mlflow_server + ':5000' print("MLflow Tracking URI: %s" % (mlflow_tracking_uri)) mlflow.set_tracking_uri(mlflow_tracking_uri) if __name__ == "__main__": #init_mlflow() h2o.init() # VARIABLE INITIATION target = str(sys.argv[1]) if len(sys.argv) > 1 else target csv_path = str(sys.argv[2]) if len(sys.argv) > 1 else csv_path delimiter = str(sys.argv[3]) if len(sys.argv) > 1 else delimiter run_time = int(sys.argv[4]) if len(sys.argv) > 1 else run_time export_path = str(sys.argv[5]) if len(sys.argv) > 1 else export_path # LOAD DATA data = util.load_csv_to_pandas(csv_path, delimiter) # MODEL EXPERIMENT model = util.create_model(h2o, mlflow, data, target, run_time) # MODEL EXPORT util.save_model(model, save_directory=export_path, mlflow=mlflow.h2o)
import numpy as np from util import create_model x = np.random.random((3, 7228)) model = create_model(x) model.load_weights('weights.best.hdf5') print(model.predict(x))
# print('x-max-seq-unit: {}'.format(X_max_seq_len)) # print('y-max-seq-unit: {}'.format(y_max_seq_len)) if not os.path.exists(SAVED_DIR): os.mkdir(SAVED_DIR) while EPOCHS > 0: epochs = min(EPOCHS, SAVE_MODEL_AFTER_EPOCH) # load model if os.path.isfile(SAVED_MODEL_PATH): # retrieving model model = load_model(SAVED_MODEL_PATH) with open(SAVED_EPOCH_PATH, 'r') as f: prev_epoch = int(f.read()) else: # creating model model = create_model(X_num_of_unit, X_max_seq_len, y_num_of_unit, y_max_seq_len, NUM_OF_HIDDEN_UNIT) prev_epoch = 0 # printing model summary # model.summary() # shuffle data # to avoid local minima indices = np.arange(len(X)) np.random.shuffle(indices) X = X[indices] y = y[indices] # checkpoint = ModelCheckpoint(filepath=SAVED_MODEL_PATH, monitor='acc', # verbose=1, save_best_only='true', mode='max') # callback_list = [checkpoint]
def create_postal_codes(): @property def parent(self): """Returns region if available, otherwise country""" return self.region if self.region else self.country @property def hierarchy(self): """Get hierarchy, root first""" list = self.parent.hierarchy list.append(self) return list @property def names(self): """Get a hierarchy of non-null names, root first""" return [ e for e in [ force_unicode(self.country), force_unicode(self.region_0_name), force_unicode(self.region_1_name), force_unicode(self.region_2_name), force_unicode(self.name), ] if e ] @property def name_full(self): """Get full name including hierarchy""" return u', '.join(reversed(self.names)) postal_codes = {} for country in settings.postal_codes: name_format = "{}" + country name = name_format.format('PostalCode') postal_codes[country] = create_model( name=name, fields={ 'country': models.ForeignKey(Country, related_name='postal_codes_' + country), 'code': models.CharField(max_length=20, db_index=True), 'name': models.CharField(max_length=200, db_index=True), 'region_0_name': models.CharField(max_length=100, db_index=True, verbose_name="region 0 name (state)"), 'region_1_name': models.CharField(max_length=100, db_index=True, verbose_name="region 1 name (county)"), 'region_2_name': models.CharField(max_length=100, db_index=True, verbose_name="region 2 name (community)"), 'region': models.ForeignKey(Region, null=True, blank=True, related_name='postal_codes_' + country), 'location': models.PointField(), 'objects': models.GeoManager(), 'parent': parent, 'hierarchy': hierarchy, 'names': names, 'name_full': name_full, '__unicode__': lambda self: force_unicode(self.code), }, app_label='cities', module='cities.models', options={ 'db_table': 'cities_' + un_camel(name), 'verbose_name': un_camel(name).replace('_', ' '), 'verbose_name_plural': un_camel(name_format.format('PostalCodes')).replace('_', ' '), }, ) return postal_codes
def create_postal_codes(): @property def parent(self): """Returns region if available, otherwise country""" return self.region if self.region else self.country @property def hierarchy(self): """Get hierarchy, root first""" list = self.parent.hierarchy list.append(self) return list @property def names(self): """Get a hierarchy of non-null names, root first""" return [e for e in [ force_unicode(self.country), force_unicode(self.region_0_name), force_unicode(self.region_1_name), force_unicode(self.region_2_name), force_unicode(self.name), ] if e] @property def name_full(self): """Get full name including hierarchy""" return u', '.join(reversed(self.names)) postal_codes = {} for country in settings.postal_codes: name_format = "{}" + country name = name_format.format('PostalCode') postal_codes[country] = create_model( name = name, fields = { 'country': models.ForeignKey(Country, related_name = 'postal_codes_' + country), 'code': models.CharField(max_length=20, db_index=True), 'name': models.CharField(max_length=200, db_index=True), 'region_0_name': models.CharField(max_length=100, db_index=True, verbose_name="region 0 name (state)"), 'region_1_name': models.CharField(max_length=100, db_index=True, verbose_name="region 1 name (county)"), 'region_2_name': models.CharField(max_length=100, db_index=True, verbose_name="region 2 name (community)"), 'region': models.ForeignKey(Region, null=True, blank=True, related_name = 'postal_codes_' + country), 'location': models.PointField(), 'objects': models.GeoManager(), 'parent': parent, 'hierarchy': hierarchy, 'names': names, 'name_full': name_full, '__unicode__': lambda self: force_unicode(self.code), }, app_label = 'cities', module = 'cities.models', options = { 'db_table': 'cities_' + un_camel(name), 'verbose_name': un_camel(name).replace('_', ' '), 'verbose_name_plural': un_camel(name_format.format('PostalCodes')).replace('_', ' '), }, ) return postal_codes
def train(model, version, train_generator, validation_generator, steps_per_epoch, save=True, csv=True, plot=True, connected=True, dropout=0.0, dense=4096, BN=False, AveragePooling=True, MutiFC=False, report=True, load_model=False): """ training code for pet(cat and dog) emotion. by now only support 'cat' or 'dog' :param report: if show classification report :param test_face: will use pure face test data :param MutiFC: add multiple fully connected layers :param AveragePooling: add global average pooling layer :param BN: if add batch normalization before fully-connected layer :param load_model: if load existed model :param dense: set number of cells in fully-connected layer :param dropout: set dropout ratio (float) :param connected: if add a fully connected layer :param classify: cat or dog :param model: the name of pre-trained model :param version: the current version :param plot: decide if need to plot & save images of acc and loss :param csv: if save the history to csv :param save: if save the model """ # load pre-trained model from keras.application # create model model, modelName = util.create_model(model, connected, dropout, dense, BN, AveragePooling, MutiFC) MODEL_PATH = '../../../../Data for project/New_Data/cat/' # file name for model modelPath = os.sep + modelName + version # compile model model.compile(optimizer=keras.optimizers.Adam(), loss=keras.losses.categorical_crossentropy, metrics=['accuracy']) model.summary() print('Now is running: ', ' Cat Final ', version) if not load_model: # add tensorboard log save_path log_dir = const.LOG_PATH + modelPath tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) # fit the model history = model.fit( train_generator, steps_per_epoch=steps_per_epoch, epochs=const.EPOCH, validation_data=validation_generator, workers=6, # callbacks=[tensorboard_callback] ) else: # show classification report on predicting util.show_predict_report(validation_generator) # save model history to csv for further analyse if csv: pd.DataFrame(history.history).to_csv(MODEL_PATH + modelPath + '.csv') # summarize history for accuracy & loss if plot: util.plotInfo(MODEL_PATH, modelPath, history) # save model if save: try: model.save(MODEL_PATH + modelPath + '.h5') except OSError: # I found a bug in saving model with tf. it shows os error but it did save model print("OSError, is it saved? ") pass # show each class report if report: y_actual = validation_generator.classes y_predicted = model.predict(validation_generator) class_ = util.get_confusion_matrix(y_actual=y_actual, y_predicted=y_predicted) print(class_)
#tokenize the input t,reverse_word_map = word_mapping(df) t.fit_on_texts(df.texts + df.target) print('done tokenizing') #pickle t pickle.dump(t,open('tokenizer.pickle', 'wb')) print('done saving tokenizer') #prepare the data set PREDICTORS,TARGET = prepare_sets(df,t) #split the data set predictors_train,target_train = generate_sets(PREDICTORS,TARGET,1) #experiment parameters: max_list_predictors = len(max(PREDICTORS,key=len)) max_sequence_len=max_list_predictors + 1 total_words = len(t.word_index) + 1 output_dim = np.shape(target_train)[1] neurons = 300 epochs = 1 #create the model model = create_model(max_sequence_len,total_words,neurons,output_dim) #train it and run the experiment model = train_and_run_experiment(model,epochs,predictors_train,target_train)
args.save_path = 'babi.p' if args.callback_args['save_path'] is None: args.callback_args['save_path'] = args.save_path # setup backend args.batch_size = 32 be = gen_backend(**extract_valid_args(args, gen_backend)) # load the bAbI dataset babi = babi_handler(args.data_dir, args.task) train_set = QA(*babi.train) valid_set = QA(*babi.test) # create model model = create_model(babi.vocab_size, args.rlayer_type) # setup callbacks callbacks = Callbacks(model, train_set, eval_set=valid_set, **args.callback_args) # train model model.fit(train_set, optimizer=Adam(), num_epochs=args.epochs, cost=GeneralizedCost(costfunc=CrossEntropyMulti()), callbacks=callbacks) # output accuracies print('Train Accuracy = %.1f%%' % (model.eval(train_set, metric=Accuracy())*100)) print('Test Accuracy = %.1f%%' % (model.eval(valid_set, metric=Accuracy())*100))
type=str, default='none', help='none|std_dev|init_weights') parser.add_argument( '--prune_method', type=str, default='l1', help='l1|old_super_mask|new_super_mask|mix_l1_super_mask') parser.add_argument('--server_prune_threshold', type=float, default=0.8) parser.add_argument('--proj_name', type=str, default='CELL') args = parser.parse_args() seed_everything(seed=args.seed, workers=True) model = create_model(cls=models[args.dataset][args.arch], device=args.device) train_loaders, test_loaders, class_idxs = DataLoaders( num_users=args.n_clients, dataset_name=args.dataset, n_class=args.n_classes, nsamples=args.n_samples, mode=args.dataset_mode, batch_size=args.batch_size, rate_unbalance=args.rate_unbalance, num_workers=args.n_workers) clients = [] for i in range(args.n_clients): client = Client(i, args, train_loaders[i], test_loaders[i], class_idxs[i]) clients.append(client)
def main(config, checkpoint_dir=None): args = parse_args('train') device = torch.device("cuda" if torch.cuda.is_available() else "cpu") torch.manual_seed(10) if tune_hyperparam: batch_size = config['batch_size'] lr = config['lr'] else: batch_size = args.batch_size lr = args.lr gamma = config['gamma'] best_f1 = 0 transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomRotation(30), transforms.RandomHorizontalFlip(p=0.5), transforms.RandomVerticalFlip(p=0.5), transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), GaussianBlur(), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), RandomErasing() ]) transform_val = transforms.Compose([ transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), ]) train_set = DataLoader(args.train_list, transform=transform_train) if args.resample: train_loader = torch.utils.data.DataLoader( train_set, batch_size=batch_size, sampler=ImbalancedDatasetSampler( train_set, num_samples=len(train_set), callback_get_label=train_set.data), num_workers=args.num_workers) else: train_loader = torch.utils.data.DataLoader( train_set, batch_size=batch_size, shuffle=True, num_workers=args.num_workers) val_set = DataLoader(args.val_list, transform=transform_val) val_loader = torch.utils.data.DataLoader(val_set, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) # Load model print("==> Creating model") num_classes = args.num_classes model = create_model(num_classes, args).to(device) # choose loss function if args.weighted_loss: targets = [i['target'] for i in train_set.data] weights = compute_class_weight('balanced', classes=np.unique(targets), y=np.array(targets)) criterion = select_loss_func(choice=args.loss, weights=torch.tensor(weights, dtype=torch.float), gamma=gamma) else: criterion = select_loss_func(choice=args.loss, gamma=gamma) # choose optimizer print('==> {} optimizer'.format(args.optimizer)) if args.optimizer == 'SAM': base_optimizer = torch.optim.SGD optimizer = SAM(model.parameters(), base_optimizer, lr=lr, momentum=0.9) elif args.optimizer == 'ADAM': optimizer = torch.optim.Adam(model.parameters(), lr=lr) else: optimizer = torch.optim.SGD(model.parameters(), lr, momentum=0.9, weight_decay=1e-3, nesterov=True) # set up logger writer = SummaryWriter(log_dir=args.log_dir) start_epoch = args.start_epoch if args.dataset == 'renal': df = pd.DataFrame(columns=[ 'model', 'lr', 'epoch_num', 'train_loss', 'val_loss', 'train_acc', 'val_acc', 'normal', 'obsolescent', 'solidified', 'disappearing', 'non_glom', 'f1' ]) elif args.dataset == 'ham': df = pd.DataFrame(columns=[ 'model', 'lr', 'epoch_num', 'train_loss', 'val_loss', 'train_acc', 'val_acc', 'MEL', 'NV', 'BCC', 'AKIEC', 'BKL', 'DF', 'VASC', 'f1' ]) else: raise ValueError('no such dataset exists!') # start training for epoch in range(start_epoch, args.epochs): epoch += 1 if args.optimizer != 'ADAM': cur_lr = adjust_learning_rate(lr, optimizer, epoch) else: cur_lr = lr print('\nEpoch: [%d | %d] LR: %f' % (epoch, args.epochs, cur_lr)) train_loss, train_acc, train_f1, train_f1s = train( train_loader, model, optimizer, criterion, device, args) val_loss, val_acc, val_f1, val_f1s = validate(val_loader, model, criterion, epoch, device, args) if tune_hyperparam: tune.report(loss=val_loss, accuracy=val_f1) writer.add_scalars("loss/", { 'train': train_loss, 'val': val_loss }, epoch) writer.add_scalars("f1/", {'train': train_f1, 'val': val_f1}, epoch) # write to csv df.loc[epoch] = [ args.network, cur_lr, epoch, train_loss, val_loss, train_acc, val_acc ] + val_f1s + [val_f1] output_csv_file = os.path.join(args.output_csv_dir, 'output.csv') df.to_csv(output_csv_file, index=False) # save model is_best_f1 = val_f1 > best_f1 best_f1 = max(val_f1, best_f1) save_checkpoint( { 'epoch': epoch, 'state_dict': model.state_dict(), 'acc': val_acc, 'best_f1': best_f1, 'optimizer': optimizer.state_dict(), }, is_best_f1, epoch, args.save_model_dir) print('Best f1:') print(best_f1)
i = i + 1 except (IOError, OSError): print("failed to process {}".format(image_path)) X = np.array(images, dtype='float32') y = np.eye(num_classes, dtype='uint8')[labels] return X, y NUM_CLASSES = 43 IMG_SIZE = 48 TRAINING_DATA_PATH = "./GTSRB/Final_Training/Images/" model = create_model(NUM_CLASSES, IMG_SIZE) X, y = get_training_data(TRAINING_DATA_PATH, NUM_CLASSES, IMG_SIZE) learning_rate = 0.01 sgd = SGD(lr=learning_rate, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) batch_size = 32 epochs = 30 history = model.fit(X, y, batch_size=batch_size,
args.save_path = 'babi.p' if args.callback_args['save_path'] is None: args.callback_args['save_path'] = args.save_path # setup backend args.batch_size = 32 be = gen_backend(**extract_valid_args(args, gen_backend)) # load the bAbI dataset babi = babi_handler(args.data_dir, args.task) train_set = QA(*babi.train) valid_set = QA(*babi.test) # create model model = create_model(babi.vocab_size, args.rlayer_type) # setup callbacks callbacks = Callbacks(model, eval_set=valid_set, **args.callback_args) # train model if args.test_only: model.fit(train_set, optimizer=Adam(), num_epochs=args.epochs, cost=GeneralizedCost(costfunc=CrossEntropyMulti()), callbacks=callbacks) # output accuracies print('Train Accuracy = %.1f%%' % (model.eval(train_set, metric=Accuracy()) * 100))
def __init__(self, docs, labels): threading.Thread.__init__(self) self.docs = docs self.labels = labels self.model = util.create_model()