def main(opts): model = Model(66, opts.size) model.model.summary() model.load(opts.weights) train_list, val_list = split(opts.data) val_dataset = AFLW2000(val_list, batch_size=1, input_size=opts.size) err, times = [], [] for idx, (x, y) in enumerate(val_dataset.data_generator()): print(f'{idx}/{val_dataset.epoch_steps}') t1 = time() res = model.test_online(x) times.append(time() - t1) ypr = np.array(y)[:, 0, 1] err.append(abs(ypr - res)) print(f'YPR: {np.mean(np.array(err), axis=0)}') print(f'TIME: {np.mean(times)}') if idx == val_dataset.epoch_steps: break
def __init__(self, data_processor, bottleneck_dim=128, num_codebooks=16, hidden_dim=512, decoder_layers=2, encoder_layers=2, **kwargs): super().__init__() self.data_processor = data_processor self.encoder1 = nn.Sequential( Feedforward(self.data_processor.input_dim, hidden_dim, num_layers=encoder_layers, **kwargs), nn.Linear(hidden_dim, bottleneck_dim)) self.quntizer = Model(input_dim=bottleneck_dim, hidden_dim=1024, bottleneck_dim=256, encoder_layers=2, decoder_layers=2, Activation=nn.ReLU, num_codebooks=8, codebook_size=256, initial_entropy=3.0, share_codewords=True).cuda() self.distance = DISTANCES['euclidian_squared'] self.triplet_delta = 5 all_parameters = list(self.encoder1.parameters()) + list( self.quntizer.parameters()) self.optimizer = OneCycleSchedule(QHAdam(all_parameters, nus=(0.8, 0.7), betas=(0.95, 0.998)), learning_rate_base=1e-3, warmup_steps=10000, decay_rate=0.2) self.experiment_path = 'logs' self.writer = SummaryWriter(self.experiment_path, comment='Cora')
def cronjob(sched_id, cmd_id): db_model = Model() devices = db_model.get_schedule_active_devices(sched_id, 'iled', 'iled') if not devices: return with open(cache_json_path, 'r') as fin: json = json_load(fin) cmd_name = json['command_names'][str(cmd_id)] errors = [] succeeded = [] dev_ids_to_update = [] for dev_id, dev_name, password in devices: # call worker.py res = handle_request( vendor='iled', model='iled', dev_name=dev_name, password=password, command=cmd_id ) if res['status'] != 'OK': errors.append({ 'dev_name': dev_name, 'status': res['status'], 'reason': res['message'] }) else: dev_ids_to_update.append(dev_id) succeeded.append(dev_name) if errors: json = json_dumps({'schedule_id' : sched_id, 'vendor' : 'iled', 'model' : 'iled', 'command': cmd_name, 'errors' : errors}) db_model.insert_log('schedule_error', json, 1) if dev_ids_to_update: db_model.update_device_status(dev_ids_to_update, cmd_id=cmd_id, dstat_name='active') db_model.cleanup()
def _cronjob(sched_id): db_model = Model() devices = db_model.get_schedule_active_devices(sched_id, 'nasys', 'ul2011') sched_cmd_data = generate_schedule(sched_id) cmd = 'send_custom_command' msg = json_dumps(sched_cmd_data) errors = [] succeeded = [] for _dev_id, dev_name, password in devices: res = handle_request(vendor='nasys', model='ul2011', dev_name=dev_name, password=password, command=cmd, parameter=msg) if res['status'] != 'OK': errors.append({ 'dev_name': dev_name, 'status': res['status'], 'reason': res['message'] }) else: succeeded.append(dev_name) if errors: json = json_dumps({ 'schedule_id': sched_id, 'vendor': 'nasys', 'model': 'ul2011', 'errors': errors }) db_model.insert_log('schedule_error', json, 1) db_model.cleanup()
def update_cron(): times = get_sunrise_sunset_times() db_model = Model() schedule_items = db_model.get_all_schedule_items(times) schedule_models = db_model.get_all_schedule_models() db_model.cleanup() # print(schedule_items) # print(schedule_models) with open(cron_txt_path, 'w') as fout: write_crons_header(fout) for sched_id in schedule_items: items = schedule_items[sched_id] models = schedule_models[sched_id] # FIXME: the lines probably belond in a separate function for each model in drivers/model/funcs.py # NASys controllers if ('nasys', 'ul2011') in models: fout.write( f'{crontimes["nasys_update_schedule"]} /usr/bin/python3 {basepath}/drivers/nasys/ul2011/cron.py {sched_id}\n' ) # iLED controllers if ('iled', 'iled') in models: for time, cmd_id, _cmd_name in items: fout.write( f'{time[3:5]} {time[0:2]} * * * /usr/bin/python3 {basepath}/drivers/iled/iled/cron.py {sched_id} {cmd_id}\n' ) fout.flush() username = getenv('USER') run([f'crontab -u {username} {cron_txt_path} <<EOL'], shell=True)
def setUp(self): self.exec_key = registry.create_exec_env(save_on_register=True) self.already_cleared = False self.test_agent = Agent(TEST_AGENT_NM, exec_key=self.exec_key, action=self.agent_action) self.model = Model(exec_key=self.exec_key)
def test_parse_should_call_given_parser_strategy(self): path = 'dummy/path' model = Model([], []) self._writer.write(path, model) self._writing_strategy_mock.write.assert_called_once_with(path, model)
def die(msg): print(msg) exit(-1) #################################################################################################### # main if __name__ == '__main__': # Ensure the script is not already running res = try_lock_pidfile(pidfile) if res['status'] != 'OK': die(json_dumps(res)) db_model = Model() # for each command for log in db_model.get_logs_with_type('command'): json = json_loads(log['log_message']) cmd_id = json['cmd_id'] succeeded, errors = [], [] try: # get command name from id try: with open(cache_json_path, 'r') as fin: all_commands = json_load(fin)['command_names'] cmd_name = all_commands[str(cmd_id)] except:
# -*- coding: utf-8 -*- import os from sklearn.preprocessing import RobustScaler from sklearn.ensemble import ExtraTreesRegressor from sklearn.svm import SVR from sklearn.pipeline import Pipeline from lib.training import routine from lib.model import Model if __name__ == '__main__': estimators = [ ExtraTreesRegressor(n_estimators=15, criterion='mae', min_samples_split=.01, bootstrap=True, n_jobs=-1, random_state=0), Pipeline([('scaler', RobustScaler()), ('svr', SVR(C=20, gamma=.3, epsilon=.1))]) ] model = Model(estimators, [.45, .55]) routine(dir_path=os.path.dirname(__file__), task_id=1, model=model, mode=os.environ.get('KDD_MODE', 'train')) print('task1 train done')
def test_intermediate_model(): efficientnet_model = Model() layer_details = efficientnet_model.get_layers() layer_name = layer_details[len(layer_details) - 2]["name"] intermediate_model = efficientnet_model.get_intermediate_model(layer_name) assert (intermediate_model.__class__.__name__ == "Functional")
def run_training(train, valid, feature_cols, target_cols, model_path, seed, param_provided=None): seed_everything(seed) x_train, y_train = train[feature_cols].values, train[ target_cols].values # create the dataset loader train_dataset = MoADataset(x_train, y_train) trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True) if valid is not None: x_valid, y_valid = valid[feature_cols].values, valid[ target_cols].values valid_dataset = MoADataset(x_valid, y_valid) validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False) # create an model instance if param_provided is not None: EPOCHS = param_provided['epoch'] hidden_size = param_provided['hidden_size'] LEARNING_RATE = param_provided['lr'] print("hidden_size: ", hidden_size, ", learning_rate: ", LEARNING_RATE) # create an model instance model = Model( num_features=num_features, num_targets=num_targets, hidden_size=hidden_size, ) model.to(DEVICE) optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY) # scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.05, div_factor=1.5e3, # max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader)) # lmbda = lambda epoch: 0.5 # scheduler = torch.optim.lr_scheduler.MultiplicativeLR(optimizer, lr_lambda=lmbda) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.5) loss_fn = nn.BCEWithLogitsLoss() early_stopping_steps = EARLY_STOPPING_STEPS early_step = 0 best_loss = np.inf # train_losses = [] valid_losses = [] for epoch in range(EPOCHS): print('Epoch {}, lr {}'.format(epoch, optimizer.param_groups[0]['lr'])) train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE) train_losses.append(train_loss) # print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}") if valid is not None: # only run the valid if valid set is provided valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE) valid_losses.append(valid_loss) if epoch % 5 == 0: print( f"EPOCH: {epoch}, train_loss: {train_loss}, valid_loss: {valid_loss}" ) if valid_loss < best_loss: best_loss = valid_loss # oof[val_idx] = valid_preds torch.save(model.state_dict(), model_path) elif EARLY_STOP == True: early_step += 1 if early_step >= early_stopping_steps: break else: if epoch % 10 == 0: print(f"EPOCH: {epoch}, train_loss: {train_loss}") print("early stop with epoch: ", epoch) print(f"LAST EPOCH: {epoch}, train_loss: {train_loss}") if valid is None: # when there is not valid set, save the model torch.save(model.state_dict(), model_path) return {"train_losses": train_losses, "valid_losses": valid_losses}
def load(self, path): data = torch.load(path) return Model(conf=data['conf'], idx_to_class=data['idx_to_class'], state=data['state'])
app = Flask(__name__, static_url_path='') from threading import Thread, current_thread logger.debug('子进程:%s,父进程:%s,线程:%r', os.getpid(), os.getppid(), current_thread()) config = Config('../config.json') try: confs = config.all() except Exception as e: logger.error("配置文件解析出错:%r", e) exit() model = Model("../model/") try: model.load_all(confs) except Exception as e: logger.error("加载Model出错:%r", e) exit() #注意!一定要放到加载了模型之后 graph = tf.get_default_graph() logger.debug("全局获得Graph:%r", graph) #读入的buffer是个纯byte数据 def process(buffer, bank): logger.debug("从web读取数据len:%r", len(buffer))
def _handle_request(**kwargs): # Load json with open(cache_json_path, 'r') as fin: cache_json = json_load(fin) try: cmd_id = str(kwargs['command']) except KeyError as e: return {'status': 'NOTOK', 'message': f'Missing parameters: {e}'} # If cmd_id is not a known id, assume the command name has been given try: cmd_name = cache_json['command_names'][cmd_id] except KeyError: cmd_name = cmd_id ################################################################# # Set dim level command. All current models support it. ################################################################# if cmd_name.startswith('set_dim_level'): try: vendor = kwargs['vendor'] model = kwargs['model'] dev_name = kwargs['dev_name'] password = kwargs['password'] except KeyError as e: return {'status': 'NOTOK', 'message': f'Missing parameters: {e}'} model_info = cache_json[vendor][model] cmd_data = model_info['commands'][cmd_name] cnx = model_info['cnx'] return send_commands(dev_name, cmd_data, cnx, password) ################################################################# # Update the cron file. ################################################################# elif cmd_name == 'update_cron': # run updater update_cron() return {'status': 'OK', 'message': 'Success'} ################################################################# # Update the schedule. ################################################################# elif cmd_name == 'update_cache': # run updater update_cache() return {'status': 'OK', 'message': 'Success'} ################################################################# # Start notification daemon for a certain model ################################################################# elif cmd_name == 'start_notifications': try: vendor = kwargs['vendor'] model = kwargs['model'] except KeyError as e: return {'status': 'NOTOK', 'message': f'Missing parameters: {e}'} pidfile = f'{basepath}/pids/__{vendor}__{model}__notifier__.pid' daemon = f'{basepath}/scripts/notification_daemon.py' if is_locked_pidfile(pidfile): return {'status': 'OK', 'message': 'Daemon is running'} Popen(['/usr/bin/python3', daemon, '-v', vendor, '-m', model]) return {'status': 'OK', 'message': 'Notification daemon started'} ################################################################# # Stop notification daemon for a certain model ################################################################# elif cmd_name == 'stop_notifications': try: vendor = kwargs['vendor'] model = kwargs['model'] except KeyError as e: return {'status': 'NOTOK', 'message': f'Missing parameters: {e}'} pidfile = f'{basepath}/pids/__{vendor}__{model}__notifier__.pid' try: with open(pidfile, 'r') as fin: pid = int(fin.readline()) # ask daemon to terminate kill(pid, SIGTERM) return {'status': 'OK', 'message': 'Notification daemon stopped'} except FileNotFoundError as e: return { 'status': 'OK', 'message': 'Notification daemon is not running' } except OSError as e: if e.errno == ESRCH: return { 'status': 'OK', 'message': 'Notification daemon is not running' } else: return { 'status': 'NOTOK', 'message': 'Stopping the daemon failed' } except Exception as e: return { 'status': 'OK', 'message': f'An unknown error occured: {e}' } ################################################################# # Custom command to update a device schedule. (only for devices # that support it, e.g. Nasys models) ################################################################# elif cmd_name.startswith('set_device_schedule'): try: vendor = kwargs['vendor'] model = kwargs['model'] dev_name = kwargs['dev_name'] password = kwargs['password'] sched_id = int(kwargs['parameter']) except KeyError as e: return {'status': 'NOTOK', 'message': f'Missing parameters: {e}'} except ValueError as e: return {'status': 'NOTOK', 'message': f'Invalid parameter: {e}'} model_info = cache_json[vendor][model] cnx = model_info['cnx'] # FIXME: pass the schedule as parameter, not the sched_id only, too slow db_model = Model() schedule = db_model.get_schedule_items(sched_id) db_model.cleanup() try: from importlib import import_module m = import_module(f'drivers.{vendor}.{model}.funcs') commands = m.generate_schedule_commands(schedule) except (ImportError, AttributeError): return { 'status': 'OK', 'message': f'{vendor}/{model} models do not store their schedule' } return send_commands(dev_name, commands, cnx, password) ################################################################# # Custom command to clear the schedule of a device (that supports it) ################################################################# elif cmd_name.startswith('clear_device_schedule'): try: vendor = kwargs['vendor'] model = kwargs['model'] dev_name = kwargs['dev_name'] password = kwargs['password'] except KeyError as e: return {'status': 'NOTOK', 'message': f'Missing parameters: {e}'} model_info = cache_json[vendor][model] cnx = model_info['cnx'] try: cmd_data = model_info['commands']['clear_device_schedule'] except KeyError: return { 'status': 'OK', 'message': f'{vendor}/{model} models do not store their schedule' } return send_commands(dev_name, cmd_data, cnx, password) ################################################################# # Send a custom message to a device ################################################################# elif cmd_name.startswith('send_custom_command'): try: vendor = kwargs['vendor'] model = kwargs['model'] dev_name = kwargs['dev_name'] password = kwargs['password'] parameter = kwargs['parameter'] except KeyError as e: return {'status': 'NOTOK', 'message': f'Missing parameters: {e}'} model_info = cache_json[vendor][model] cnx = model_info['cnx'] try: custom_cmd_data = json_loads(parameter) except: return {'status': 'NOTOK', 'message': f'Invalid message: {e}'} return send_commands(dev_name, custom_cmd_data, cnx, password) ################################################################# # Unknown command, error out. ################################################################# else: # Unknown command return { 'status': 'NOTOK', 'message': f'Command {cmd_name} is not implemented' }
batch_size=BATCH_SIZE, shuffle=False) for k_fold in np.arange(NFOLDS): # for k_fold in [0]: if best_param_with_epoch is not None: hidden_size = best_param_with_epoch["kfold_" + str(k_fold)]['hidden_size'] model_path = models_dir + prefix + "_model_fold_{0:d}.pth".format( k_fold) model = Model( num_features=num_features, num_targets=num_targets, hidden_size=hidden_size, ) model.load_state_dict(torch.load(model_path)) model.to(DEVICE) predictions = inference_fn(model, testloader, DEVICE) # save the prediction on test set pred_test = pd.DataFrame(data=predictions, columns=target_cols).fillna(0) pred_test = pd.concat([data_test_x[["sig_id"]], pred_test], axis=1) pred_result_f = models_dir + prefix + "_fold_{0:d}_prediction.csv".format( k_fold) pred_test.to_csv(pred_result_f, index=False) # merge all the results by average
def run_training_tune(trial, train, valid, feature_cols, target_cols, model_path, seed): seed_everything(seed) x_train, y_train = train[feature_cols].values, train[ target_cols].values x_valid, y_valid = valid[feature_cols].values, valid[ target_cols].values # create the dataset loader train_dataset = MoADataset(x_train, y_train) valid_dataset = MoADataset(x_valid, y_valid) trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True) validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False) hidden_size = trial.suggest_int("hidden_size", 30, 60, step=10) # create an model instance model = Model( num_features=num_features, num_targets=num_targets, hidden_size=hidden_size, ) model.to(DEVICE) lr = trial.suggest_float("lr", 1e-5, 1e-1, log=True) # lr = 0.0084 optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=WEIGHT_DECAY) # this is used to change the learning rate when training model # scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.05, # max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader)) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.5) loss_fn = nn.BCEWithLogitsLoss() early_stopping_steps = EARLY_STOPPING_STEPS early_step = 0 best_loss = np.inf # train_losses = [] valid_losses = [] for epoch in range(EPOCHS): train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE) train_losses.append(train_loss) # print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}") valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE) valid_losses.append(valid_loss) if epoch % 5 == 0: print( f"EPOCH: {epoch}, train_loss: {train_loss}, valid_loss: {valid_loss}" ) if valid_loss < best_loss: best_loss = valid_loss # now don't save the model to adapt to the use of optuna # torch.save(model.state_dict(), model_path) elif EARLY_STOP == True: early_step += 1 if early_step >= early_stopping_steps: break print("early stopping with epoch: ", epoch) if trial.should_prune(): raise optuna.exceptions.TrialPruned() return valid_losses[-1]
def test_get_env(self): """ See if we get an env we have registered back as the env. """ self.model = Model(exec_key=self.exec_key) self.assertEqual(self.model.env, get_env(exec_key=self.exec_key))
for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])): print("Step {:4d}".format(i)) print(" input: {} ({:s})".format(input_idx, repr(idx2char[input_idx]))) print(" expected output: {} ({:s})".format(target_idx, repr(idx2char[target_idx]))) # Buffer size to shuffle the dataset # (TF data is designed to work with possibly infinite sequences, # so it doesn't attempt to shuffle the entire sequence in memory. Instead, # it maintains a buffer in which it shuffles elements). BUFFER_SIZE = 10000 dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True) model = Model(len(char2idx), EMBEDDING_DIM, UNITS) # Using adam optimizer with default arguments optimizer = tf.train.AdamOptimizer() model.build(tf.TensorShape([BATCH_SIZE, SEQ_LENGTH])) model.summary() # Training step EPOCHS = 30 # Directory where the checkpoints will be saved checkpoint_dir = os.getenv('TRAINING_CHECKPOINT_DIR', './data/training_checkpoints') # Name of the checkpoint files
def main(): """ The main routine. """ # Fix random seeds for reproducibility - these are themselves generated from random.org # From https://keras.io/getting-started/faq/#how-can-i-obtain-reproducible-results-using-keras-during-development os.environ['PYTHONHASHSEED'] = '0' np.random.seed(91) rn.seed(95) session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1) tf.set_random_seed(47) sess = tf.Session(graph=tf.get_default_graph(), config=session_conf) k.set_session(sess) # Enable simple logging logging.basicConfig(level=logging.INFO, format='%(message)s') # Parse command line arguments args = parseargs() # Create run folder output_directory = create_output_folder(args.output) # Write arguments to file with open(output_directory + 'arguments.txt', 'a') as arguments_file: for arg in vars(args): arguments_file.write( str(arg) + ': ' + str(getattr(args, arg)) + '\n') ############## # Prepare data print('') data = Data(incidences_file=args.incidences, specifications_file=args.specifications, plot_data=args.plotData, output_directory=output_directory) data.state(message='Raw data') data.filter_cases(cases_file=args.cases) data.state(message='Filtered SEER*Stat cases from ASCII') # Determine inputs, filter, and pre process them data.apply_data_pipeline(pipelines.data_pipeline_full, args.oneHotEncoding) data.state( message= 'Remove irrelevant, combined, post-diagnosis, and treatment attributes' ) data.create_target(args.task) data.state(message='Create target label indicating cancer survival for ' + args.task) encodings = data.finalize() data.state(message='Remove inputs with constant values') ############### # Prepare model model = Model(model_type=args.model, task=args.task, input_dim=(len(data.frame.columns) - 1), encodings=encodings, mlp_layers=args.mlpLayers, mlp_width=args.mlpWidth, mlp_dropout=args.mlpDropout, mlp_emb_neurons=args.mlpEmbNeurons, svm_gamma=args.svmGamma, svm_c=args.svmC, logr_c=args.logrC) if args.plotData: model.plot_model(output_directory) ################ # Carry out task experiment = Experiment(model=model, data=data, task=args.task, valid_ratio=0.1, test_ratio=0.1, model_type=args.model, encodings=encodings, encode_categorical_inputs=args.oneHotEncoding, plot_results=args.plotResults, output_directory=output_directory) experiment.train(mlp_epochs=args.mlpEpochs) results_validate = experiment.validate() # Write validation results to file with open(output_directory + 'results_validate.txt', 'a') as results_file: for res in results_validate: results_file.write(res + '\n') # Only test final model, do not use for tuning if args.test: results_test = experiment.test() # Write validation results to file with open(output_directory + 'results_test.txt', 'a') as results_file: for res in results_test: results_file.write(res + '\n') ################### # Input importance if args.importance: importance = experiment.importance(encodings=encodings) # Write importance results to file with open(output_directory + 'results_importance.txt', 'a') as results_file: for (column, rel) in importance: results_file.write(column + '=' + str(rel) + '\n')
Pipeline([ ('scaler', RobustScaler()), ('knn', KNeighborsRegressor(5, weights='distance', n_jobs=-1)) ]), Pipeline([ ('scaler', RobustScaler()), ('svr', SVR( C=160, gamma=0.1, epsilon=0.1 )) ]), Pipeline([ ('scaler', RobustScaler()), ('lasso', LassoCV( cv=KFold(4, True, 0), n_jobs=-1 )) ]) ] model = Model(estimators, [.45, .05, .45, .05]) routine( dir_path=os.path.dirname(__file__), task_id=2, model=model, mode=os.environ.get('KDD_MODE', 'train') ) print('task2 train done')
def __init__(self, protector): self.protector = protector self.client = mqtt.Client() self.db_model = Model() self.initialize_notifier()
def __init__(self, version): self.view = {} self.model = Model(version) wx.App.__init__(self, False)
from lib.optimizers.adam import Adam from lib.data.data import Data from lib.hyperparameter_search import HyperparameterSearch from lib.model import Model data = Data().load() for hyperparameters in HyperparameterSearch().hyperparameters(): Model(data, Adam, hyperparameters['learning_rate'], hyperparameters['regularization_strength'], hyperparameters['number_hidden_layers'], hyperparameters['min_number_hidden_nodes']).train()
from data import load_data from lib.model import Model from util import Logger, train, validation, AdamOptimizer # os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" # os.environ["CUDA_VISIBLE_DEVICES"]=str(exp_config['device']) # torch.cuda.set_device(0) # data train_data, val_data = load_data(data_config, exp_config['batch_size']) eval_length = data_config['eval_length'] # logger # model model = Model(**model_config).to(0) # optimizer optimizer = AdamOptimizer(params=model.parameters(), lr=exp_config['lr'], grad_clip_value=exp_config['grad_clip_value'], grad_clip_norm=exp_config['grad_clip_norm']) logger_on = True if logger_on: logger = Logger(exp_config, model_config, data_config) # train / val loop for epoch in range(exp_config['n_epochs']): print('Epoch:', epoch)
test_dataset = SegDataset(ts.VALIDATION_LMDB) test_align_collate = AlignCollate('test', ts.LABELS, ts.MEAN, ts.STD, ts.IMAGE_SIZE_HEIGHT, ts.IMAGE_SIZE_WIDTH, ts.ANNOTATION_SIZE_HEIGHT, ts.ANNOTATION_SIZE_WIDTH, ts.CROP_SCALE, ts.CROP_AR, random_cropping=ts.RANDOM_CROPPING, horizontal_flipping=ts.HORIZONTAL_FLIPPING) assert test_dataset test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=opt.batchsize, shuffle=False, num_workers=opt.nworkers, pin_memory=pin_memory, collate_fn=test_align_collate) # Define Model # opt.batchsize # opt.nepochs model = Model(ts.LABELS, load_model_path=opt.model, usegpu=opt.usegpu) model.fit(ts.CRITERION, ts.LEARNING_RATE, ts.WEIGHT_DECAY, ts.CLIP_GRAD_NORM, ts.LR_DROP_FACTOR, ts.LR_DROP_PATIENCE, ts.OPTIMIZE_BG, ts.OPTIMIZER, ts.TRAIN_CNN, opt.nepochs, ts.CLASS_WEIGHTS, train_loader, test_loader, model_save_path)
from flask import Flask, send_from_directory, jsonify from lib.model import Model from lib.prezi_indexer import PerziIndexer app = Flask(__name__, static_folder="prezi-app/dist", static_url_path="") model = Model( PerziIndexer.index_json(open("data/prezis.json").read(), index_by='id')) @app.route('/') def index(): return app.send_static_file('index.html') @app.route('/<path:path>') def static_proxy(path): return app.send_static_file(path) @app.route('/prezi/<id>') def get_prezi_data(id): return jsonify(model[id]) @app.route('/search/<phrase>') def search_name(phrase): return jsonify(model.search(phrase, field_key="title")) if __name__ == '__main__':
from skimage import color from keras.models import Sequential from keras.layers.core import Dense, Dropout, Activation, Flatten from keras.layers.convolutional import Convolution2D, MaxPooling2D from keras.regularizers import l2 from keras import backend as K from keras.callbacks import TensorBoard from keras.callbacks import ModelCheckpoint from lib import label,image_process,log,crash_debug from lib.cnn import CNN from lib.model import Model from lib.config import Config __model = Model() # batch_size 太小会导致训练慢,过拟合等问题,太大会导致欠拟合。所以要适当选择 def train(data_dir,conf,epochs=2,batch_size=100): image_width = conf.width image_height = conf.height _letters = conf.charset#字符集合 _num_symbol = conf.number #识别字符的个数 model = __model.load4train(conf) model_path = __model.model_path(conf) # 训练集 x_train,y_train = image_process.load_all_image_by_dir(data_dir,conf) #训练期间保存checkpoint,防止整个crash