def train(argv):
import argparse
parser = argparse.ArgumentParser(description='train nn from history file')
parser.add_argument('game', help='a game name i.e. checkers')
parser.add_argument('history', help='history file')
parser.add_argument('input', help='input model file name')
parser.add_argument('output', help='output model file name')
parser.add_argument('--progress',
action='store_true',
help='show progress bar')
parser.add_argument('--epoch', default=1, type=int, help='training epochs')
parser.add_argument('--batch', default=256, type=int, help='batch size')
parser.add_argument('--block', default=100000, type=int, help='block size')
parser.add_argument('--gpu', type=float, help='gpu memory fraction')
args = parser.parse_args(argv)
# set gpu memory
if args.gpu != None:
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = args.gpu
set_session(tf.Session(config=config))
from util import game
State = game.importState(args.game).State
NN = game.importNn(args.game).NN
state = State()
nn = NN(args.input)
for epoch in range(args.epoch):
from util.trainer import train
train(state, nn, args.history, {
'batch': args.batch,
'block': args.block,
'progress': args.progress,
})
nn.save(args.output)
def loop(argv):
import argparse
parser = argparse.ArgumentParser(description='loop generate and train')
parser.add_argument('game', help='a game name i.e. checkers')
parser.add_argument('begin', default=1, type=int, help='begin with generation number')
parser.add_argument('count', nargs='?', default=100, type=int, help='number of generation')
parser.add_argument('-n', '--number', default=10000, type=int, help='number of generated states')
parser.add_argument('-s', '--simulation', default=100, type=int, help='number of simulations per move')
parser.add_argument('--hard', default=0, type=int, help='number of random moves')
parser.add_argument('--soft', default=1000, type=int, help='number of random moves that depends on visited node count')
parser.add_argument('--epoch', default=1, type=int, help='training epochs')
parser.add_argument('--batch', default=256, type=int, help='batch size')
parser.add_argument('--block', default=100000, type=int, help='block size')
parser.add_argument('--gpu', type=float, help='gpu memory fraction')
args = parser.parse_args(argv)
# set gpu memory
if args.gpu != None:
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = args.gpu
set_session(tf.Session(config=config))
from util import game
State = game.importState(args.game).State
NN = game.importNn(args.game).NN
state = State()
state_no = args.begin
nn = NN('{:06}.h5'.format(state_no))
for i in range(args.count):
history_filename = '{:06}.txt'.format(state_no)
# generate
with open(history_filename, 'a') as file:
def save_to_file(result):
file.write(result)
file.write('\n')
file.flush()
callback = save_to_file
from util.generator import generate
generate(state, nn, callback, {
'selfplay': args.number,
'simulation': args.simulation,
'hard_random': args.hard,
'soft_random': args.soft,
'progress': True,
})
# train
for epoch in range(args.epoch):
from util.trainer import train
train(state, nn, history_filename, {
'batch': args.batch,
'block': args.block,
'progress': True,
})
# next generation
state_no += 1
nn.save('{:06}.h5'.format(state_no))
collate_fn=tr.collate)
val_data_loader = DataLoader(val_data,
batch_size=batch_size,
collate_fn=tr.collate)
test_data_loader = DataLoader(test_data,
batch_size=batch_size,
collate_fn=tr.collate)
model = TGNN(cc.num_atom_feats, cc.num_bond_feats, 1).cuda()
# optimizer = AdaBound(model.parameters(), lr=1e-6, weight_decay=1e-8)
optimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=1e-8)
criterion = torch.nn.L1Loss()
best_val_loss = 1e+8
for epoch in range(0, num_epochs):
train_loss = tr.train(model, optimizer, train_data_loader, criterion)
val_loss, val_rmse, _, _, _ = tr.test(model, val_data_loader,
criterion)
test_loss, test_rmse, idxs, targets, preds = tr.test(
model, test_data_loader, criterion)
print(
'Epoch [{}/{}]\tTrain loss: {:.4f}\tVal loss: {:.4f} ({:.4f})\tTest loss: {:.4f} ({:.4f})'
.format(epoch + 1, num_epochs, train_loss, val_loss, val_rmse,
test_loss, test_rmse))
if val_loss < best_val_loss:
best_val_loss = val_loss
best_test_loss = test_loss
best_test_rmse = test_rmse
best_idxs = idxs
best_targets = targets
def classify(dev_id=None):
# Prepare model related meta data
mdl_name = args.model.lower().replace(' ', '_')
common_cfg = cfgr('validate', 'common')
pr = io.param_reader(os.path.join(PAR_DIR, 'etc', '%s.yaml' % common_cfg.setdefault('mdl_cfg', 'mdlcfg')))
config_kwargs = dict([(k, v) for k, v in args.__dict__.items() if not k.startswith('_') and k not in set(['dataset', 'model', 'template']) and v is not None and not callable(v)])
config = Configurable(args.task, mdl_name, common_cfg=common_cfg, wsdir=PAR_DIR, **config_kwargs)
params = pr('LM', config.lm_params) if mdl_name != 'none' else {}
use_gpu = dev_id is not None
tokenizer = config.tknzr.from_pretrained(params['pretrained_vocab_path'] if 'pretrained_vocab_path' in params else config.lm_mdl_name) if config.tknzr else {}
_adjust_encoder(tokenizer, config)
# Prepare task related meta data.
task_path, task_type, task_dstype, task_cols, task_trsfm, task_extparms = config.input if config.input and os.path.isdir(os.path.join(DATA_PATH, config.input)) else config.task_path, config.task_type, config.task_ds, config.task_col, config.task_trsfm, config.task_ext_params
ds_kwargs = config.ds_kwargs
# Prepare data
if (not config.distrb or config.distrb and hvd.rank() == 0): logging.info('Dataset path: %s' % os.path.join(DATA_PATH, task_path))
train_ds = task_dstype(os.path.join(DATA_PATH, task_path, 'train.%s' % config.fmt), tokenizer, config, **ds_kwargs)
# Calculate the class weights if needed
lb_trsfm = [x['get_lb'] for x in task_trsfm[1] if 'get_lb' in x]
if (not config.weight_class or task_type == 'sentsim'):
class_count = None
elif len(lb_trsfm) > 0:
lb_df = train_ds.df[task_cols['y']].apply(lb_trsfm[0])
class_count = np.array([[1 if lb in y else 0 for lb in train_ds.binlb.keys()] for y in lb_df]).sum(axis=0)
else:
lb_df = train_ds.df[task_cols['y']]
binlb = task_extparms['binlb'] if 'binlb' in task_extparms and type(task_extparms['binlb']) is not str else train_ds.binlb
class_count = lb_df.value_counts()[binlb.keys()].values
if (class_count is None):
class_weights = None
sampler = None
else:
class_weights = torch.Tensor(1.0 / class_count)
class_weights /= class_weights.sum()
sampler = None # WeightedRandomSampler does not work in new version
# sampler = WeightedRandomSampler(weights=class_weights, num_samples=config.bsize, replacement=True)
if not config.distrb and type(dev_id) is list: class_weights = class_weights.repeat(len(dev_id))
# Partition dataset among workers using DistributedSampler
if config.distrb: sampler = torch.utils.data.distributed.DistributedSampler(train_ds, num_replicas=hvd.size(), rank=hvd.rank())
train_loader = DataLoader(train_ds, batch_size=config.bsize, shuffle=sampler is None and config.droplast, sampler=sampler, num_workers=config.np, drop_last=config.droplast)
# Classifier
if (not config.distrb or config.distrb and hvd.rank() == 0):
logging.info('Language model input fields: %s' % config.input_keys)
logging.info('Classifier hyper-parameters: %s' % config.clf_ext_params)
logging.info('Classifier task-related parameters: %s' % task_extparms['mdlaware'])
if (config.resume):
# Load model
clf, prv_optimizer, resume, chckpnt = load_model(config.resume)
if config.refresh:
logging.info('Refreshing and saving the model with newest code...')
try:
if (not distrb or distrb and hvd.rank() == 0):
save_model(clf, prv_optimizer, '%s_%s.pth' % (config.task, config.model))
except Exception as e:
logging.warning(e)
# Update parameters
clf.update_params(task_params=task_extparms['mdlaware'], **config.clf_ext_params)
if (use_gpu): clf = _handle_model(clf, dev_id=dev_id, distrb=config.distrb)
# Construct optimizer
optmzr_cls = config.optmzr if config.optmzr else (torch.optim.Adam, {}, None)
optimizer = optmzr_cls[0](clf.parameters(), lr=config.lr, weight_decay=config.wdecay, **optmzr_cls[1]) if config.optim == 'adam' else torch.optim.SGD(clf.parameters(), lr=config.lr, momentum=0.9)
if prv_optimizer: optimizer.load_state_dict(prv_optimizer.state_dict())
training_steps = int(len(train_ds) / config.bsize) if hasattr(train_ds, '__len__') else config.trainsteps
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=int(config.wrmprop*training_steps), num_training_steps=training_steps) if not config.noschdlr and len(optmzr_cls) > 2 and optmzr_cls[2] and optmzr_cls[2] == 'linwarm' else None
if (not config.distrb or config.distrb and hvd.rank() == 0): logging.info((optimizer, scheduler))
else:
# Build model
lm_model, lm_config = gen_mdl(config, use_gpu=use_gpu, distrb=config.distrb, dev_id=dev_id)
clf = gen_clf(config, lm_model, lm_config, num_lbs=len(train_ds.binlb) if train_ds.binlb else 1, mlt_trnsfmr=True if task_type in ['entlmnt', 'sentsim'] and task_extparms['mdlaware'].setdefault('sentsim_func', None) is not None else False, task_params=task_extparms['mdlaware'], binlb=train_ds.binlb, binlbr=train_ds.binlbr, use_gpu=use_gpu, distrb=config.distrb, dev_id=dev_id, **config.clf_ext_params)
optmzr_cls = config.optmzr if config.optmzr else (torch.optim.Adam, {}, None)
optimizer = optmzr_cls[0](clf.parameters(), lr=config.lr, weight_decay=config.wdecay, **optmzr_cls[1]) if config.optim == 'adam' else torch.optim.SGD(clf.parameters(), lr=config.lr, momentum=0.9)
training_steps = int(len(train_ds) / config.bsize) if hasattr(train_ds, '__len__') else config.trainsteps
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=config.wrmprop, num_training_steps=training_steps) if not config.noschdlr and len(optmzr_cls) > 2 and optmzr_cls[2] and optmzr_cls[2] == 'linwarm' else None
if (not config.distrb or config.distrb and hvd.rank() == 0): logging.info((optimizer, scheduler))
config.execute_all_callback()
if config.verbose:
logging.debug(config.__dict__)
torch.autograd.set_detect_anomaly(True)
if config.configfmt == 'yaml':
config.to_yaml()
else:
config.to_json()
if config.distrb:
# Add Horovod Distributed Optimizer
optimizer = hvd.DistributedOptimizer(optimizer, named_parameters=clf.named_parameters())
# Broadcast parameters from rank 0 to all other processes.
hvd.broadcast_parameters(clf.state_dict(), root_rank=0)
# Training
train(clf, optimizer, train_loader, config, scheduler, weights=class_weights, lmcoef=config.lmcoef, clipmaxn=config.clipmaxn, epochs=config.epochs, earlystop=config.earlystop, earlystop_delta=config.es_delta, earlystop_patience=config.es_patience, use_gpu=use_gpu, devq=dev_id, distrb=config.distrb, resume=resume if config.resume else {})
if config.distrb:
if hvd.rank() == 0:
clf = _handle_model(clf, dev_id=dev_id, distrb=False)
else:
return
if config.noeval: return
dev_ds = task_dstype(os.path.join(DATA_PATH, task_path, 'dev.%s' % config.fmt), tokenizer, config, binlb=task_extparms['binlb'] if 'binlb' in task_extparms and type(task_extparms['binlb']) is not str else train_ds.binlb, **ds_kwargs)
dev_loader = DataLoader(dev_ds, batch_size=config.bsize, shuffle=False, num_workers=config.np)
test_ds = task_dstype(os.path.join(DATA_PATH, task_path, 'test.%s' % config.fmt), tokenizer, config, binlb=task_extparms['binlb'] if 'binlb' in task_extparms and type(task_extparms['binlb']) is not str else train_ds.binlb, **ds_kwargs)
test_loader = DataLoader(test_ds, batch_size=config.bsize, shuffle=False, num_workers=config.np)
logging.debug(('binlb', train_ds.binlb, dev_ds.binlb, test_ds.binlb))
# Evaluation
eval(clf, dev_loader, config, ds_name='dev', use_gpu=use_gpu, devq=dev_id, distrb=config.distrb, ignored_label=task_extparms.setdefault('ignored_label', None))
if config.traindev: train(clf, optimizer, dev_loader, config, scheduler=scheduler, weights=class_weights, lmcoef=config.lmcoef, clipmaxn=config.clipmaxn, epochs=config.epochs, earlystop=config.earlystop, earlystop_delta=config.es_delta, earlystop_patience=config.es_patience, use_gpu=use_gpu, devq=dev_id, distrb=config.distrb)
eval(clf, test_loader, config, ds_name='test', use_gpu=use_gpu, devq=dev_id, distrb=config.distrb, ignored_label=task_extparms.setdefault('ignored_label', None))
def multi_clf(dev_id=None):
'''Train multiple classifiers and use them to predict multiple set of labels'''
import inflect
from bionlp.util import fs
iflteng = inflect.engine()
logging.info('### Multi Classifier Head Mode ###')
# Prepare model related meta data
mdl_name = args.model.lower().replace(' ', '_')
common_cfg = cfgr('validate', 'common')
pr = io.param_reader(os.path.join(PAR_DIR, 'etc', '%s.yaml' % common_cfg.setdefault('mdl_cfg', 'mdlcfg')))
config_kwargs = dict([(k, v) for k, v in args.__dict__.items() if not k.startswith('_') and k not in set(['dataset', 'model', 'template']) and v is not None and type(v) is not function])
config = Configurable(args.task, mdl_name, common_cfg=common_cfg, wsdir=PAR_DIR, **config_kwargs)
params = pr('LM', config.lm_params) if mdl_name != 'none' else {}
use_gpu = dev_id is not None
tokenizer = config.tknzr.from_pretrained(params['pretrained_vocab_path'] if 'pretrained_vocab_path' in params else config.lm_mdl_name) if config.tknzr else None
task_type = config.task_type
_adjust_encoder(tokenizer, config)
special_tknids_args = dict(zip(special_tkns[0], special_tknids))
task_trsfm_kwargs = dict(list(zip(special_tkns[0], special_tknids))+[('model',args.model), ('sentsim_func', args.sentsim_func), ('seqlen',args.maxlen)])
# Prepare task related meta data.
task_path, task_dstype, task_cols, task_trsfm, task_extparms = args.input if args.input and os.path.isdir(os.path.join(DATA_PATH, args.input)) else config.task_path, config.task_ds, config.task_col, config.task_trsfm, config.task_ext_params
trsfms = (task_trsfm[0] if len(task_trsfm) > 0 else [])
# trsfms_kwargs = ([] if args.model in LM_EMBED_MDL_MAP else ([{'seqlen':args.maxlen, 'xpad_val':task_extparms.setdefault('xpad_val', 0), 'ypad_val':task_extparms.setdefault('ypad_val', None)}] if TASK_TYPE_MAP[args.task]=='nmt' else [{'seqlen':args.maxlen, 'trimlbs':task_extparms.setdefault('trimlbs', False), 'special_tkns':special_tknids_args}, task_trsfm_kwargs, {'seqlen':args.maxlen, 'xpad_val':task_extparms.setdefault('xpad_val', 0), 'ypad_val':task_extparms.setdefault('ypad_val', None)}])) + (task_trsfm[1] if len(task_trsfm) >= 2 else [{}] * len(task_trsfm[0]))
trsfms_kwargs = ([] if hasattr(config, 'embed_type') and config.embed_type else ([{'seqlen':args.maxlen, 'xpad_val':task_extparms.setdefault('xpad_val', 0), 'ypad_val':task_extparms.setdefault('ypad_val', None)}] if config.task_type=='nmt' else [{'seqlen':args.maxlen, 'trimlbs':task_extparms.setdefault('trimlbs', False), 'required_special_tkns':['start_tknids', 'clf_tknids', 'delim_tknids'] if task_type in ['entlmnt', 'sentsim'] and (task_extparms.setdefault('sentsim_func', None) is None or not mdl_name.startswith('bert')) else ['start_tknids', 'clf_tknids'], 'special_tkns':special_tknids_args}, task_trsfm_kwargs, {'seqlen':args.maxlen, 'xpad_val':task_extparms.setdefault('xpad_val', 0), 'ypad_val':task_extparms.setdefault('ypad_val', None)}])) + (task_trsfm[1] if len(task_trsfm) >= 2 else [{}] * len(task_trsfm[0]))
ds_kwargs = {'sampw':args.sample_weights, 'sampfrac':args.sampfrac}
if task_type == 'nmt':
ds_kwargs.update({'lb_coding':task_extparms.setdefault('lb_coding', 'IOB')})
elif task_type == 'entlmnt':
ds_kwargs.update(dict((k, task_extparms[k]) for k in ['origlb', 'lbtxt', 'neglbs', 'reflb'] if k in task_extparms))
elif task_type == 'sentsim':
ds_kwargs.update({'ynormfunc':task_extparms.setdefault('ynormfunc', None)})
global_all_binlb = {}
ext_params = dict([(k, getattr(args, k)) if hasattr(args, k) else (k, v) for k, v in config.clf_ext_params.items()])
if hasattr(config, 'embed_type') and config.embed_type: ext_params['embed_type'] = config.embed_type
task_params = dict([(k, getattr(args, k)) if hasattr(args, k) and getattr(args, k) is not None else (k, v) for k, v in task_extparms.setdefault('mdlcfg', {}).items()])
logging.info('Classifier hyper-parameters: %s' % ext_params)
logging.info('Classifier task-related parameters: %s' % task_params)
orig_epochs = mltclf_epochs = args.epochs
elapsed_mltclf_epochs, args.epochs = 0, 1
if (args.resume):
# Load model
clf, prv_optimizer, resume, chckpnt = load_model(args.resume)
if args.refresh:
logging.info('Refreshing and saving the model with newest code...')
try:
save_model(clf, prv_optimizer, '%s_%s.pth' % (args.task, args.model))
except Exception as e:
logging.warning(e)
elapsed_mltclf_epochs, all_binlb = chckpnt.setdefault('mltclf_epochs', 0), clf.binlb
# Update parameters
clf.update_params(task_params=task_params, **ext_params)
if (use_gpu): clf = _handle_model(clf, dev_id=dev_id, distrb=args.distrb)
# optmzr_cls = OPTMZR_MAP.setdefault(args.model.split('_')[0], (torch.optim.Adam, {}, None))
optmzr_cls = config.optmzr if config.optmzr else (torch.optim.Adam, {}, None)
optimizer = optmzr_cls[0](clf.parameters(), lr=args.lr, weight_decay=args.wdecay, **optmzr_cls[1]) if args.optim == 'adam' else torch.optim.SGD(clf.parameters(), lr=args.lr, momentum=0.9)
if prv_optimizer: optimizer.load_state_dict(prv_optimizer.state_dict())
training_steps = int(len(train_ds) / args.bsize) if hasattr(train_ds, '__len__') else args.trainsteps
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.wrmprop, num_training_steps=training_steps) if not args.noschdlr and len(optmzr_cls) > 2 and optmzr_cls[2] and optmzr_cls[2] == 'linwarm' else None
logging.info((optimizer, scheduler))
else:
# Build model
lm_model = gen_mdl(mdl_name, config, pretrained=True if type(args.pretrained) is str and args.pretrained.lower() == 'true' else args.pretrained, use_gpu=use_gpu, distrb=args.distrb, dev_id=dev_id) if mdl_name != 'none' else None
clf = gen_clf(args.model, config, args.encoder, lm_model=lm_model, mlt_trnsfmr=True if task_type in ['entlmnt', 'sentsim'] and task_params.setdefault('sentsim_func', None) is not None else False, task_params=task_params, use_gpu=use_gpu, distrb=args.distrb, dev_id=dev_id, **ext_params)
# optmzr_cls = OPTMZR_MAP.setdefault(args.model.split('_')[0], (torch.optim.Adam, {}, None))
optmzr_cls = config.optmzr if config.optmzr else (torch.optim.Adam, {}, None)
optimizer = optmzr_cls[0](clf.parameters(), lr=args.lr, weight_decay=args.wdecay, **optmzr_cls[1]) if args.optim == 'adam' else torch.optim.SGD(clf.parameters(), lr=args.lr, momentum=0.9)
training_steps = int(len(train_ds) / args.bsize) if hasattr(train_ds, '__len__') else args.trainsteps
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.wrmprop, num_training_steps=training_steps) if not args.noschdlr and len(optmzr_cls) > 2 and optmzr_cls[2] and optmzr_cls[2] == 'linwarm' else None
logging.info((optimizer, scheduler))
# Prepare data
logging.info('Dataset path: %s' % os.path.join(DATA_PATH, task_path))
num_clfs = min([len(fs.listf(os.path.join(DATA_PATH, task_path), pattern='%s_\d.csv' % x)) for x in ['train', 'dev', 'test']])
for epoch in range(elapsed_mltclf_epochs, mltclf_epochs):
logging.info('Global %i epoch(s)...' % epoch)
clf.reset_global_binlb()
all_binlb = {}
for i in range(num_clfs):
logging.info('Training on the %s sub-dataset...' % iflteng.ordinal(i+1))
train_ds = task_dstype(os.path.join(DATA_PATH, task_path, 'train_%i.%s' % (i, args.fmt)), task_cols['X'], task_cols['y'], config.encode_func, tokenizer, config, sep='\t', index_col=task_cols['index'], binlb=task_extparms['binlb'] if 'binlb' in task_extparms else None, transforms=trsfms, transforms_kwargs=trsfms_kwargs, mltl=task_extparms.setdefault('mltl', False), **ds_kwargs)
new_lbs = [k for k in train_ds.binlb.keys() if k not in all_binlb]
all_binlb.update(dict([(k, v) for k, v in zip(new_lbs, range(len(all_binlb), len(all_binlb)+len(new_lbs)))]))
if mdl_name.startswith('bert'): train_ds = MaskedLMIterDataset(train_ds) if isinstance(train_ds, BaseIterDataset) else MaskedLMDataset(train_ds)
lb_trsfm = [x['get_lb'] for x in task_trsfm[1] if 'get_lb' in x]
if (not args.weight_class or task_type == 'sentsim'):
class_count = None
elif len(lb_trsfm) > 0:
lb_df = train_ds.df[task_cols['y']].apply(lb_trsfm[0])
class_count = np.array([[1 if lb in y else 0 for lb in train_ds.binlb.keys()] for y in lb_df]).sum(axis=0)
else:
lb_df = train_ds.df[task_cols['y']]
binlb = task_extparms['binlb'] if 'binlb' in task_extparms and type(task_extparms['binlb']) is not str else train_ds.binlb
class_count = lb_df.value_counts()[binlb.keys()].values
if (class_count is None):
class_weights = None
sampler = None
else:
class_weights = torch.Tensor(1.0 / class_count)
class_weights /= class_weights.sum()
class_weights *= (args.clswfac[min(len(args.clswfac)-1, i)] if type(args.clswfac) is list else args.clswfac)
sampler = WeightedRandomSampler(weights=class_weights, num_samples=args.bsize, replacement=True)
if type(dev_id) is list: class_weights = class_weights.repeat(len(dev_id))
train_loader = DataLoader(train_ds, batch_size=args.bsize, shuffle=False, sampler=None, num_workers=args.np, drop_last=args.droplast)
dev_ds = task_dstype(os.path.join(DATA_PATH, task_path, 'dev_%i.%s' % (i, args.fmt)), task_cols['X'], task_cols['y'], config.encode_func, tokenizer, config, sep='\t', index_col=task_cols['index'], binlb=task_extparms['binlb'] if 'binlb' in task_extparms and type(task_extparms['binlb']) is not str else all_binlb, transforms=trsfms, transforms_kwargs=trsfms_kwargs, mltl=task_extparms.setdefault('mltl', False), **ds_kwargs)
if mdl_name.startswith('bert'): dev_ds = MaskedLMIterDataset(train_ds) if isinstance(train_ds, BaseIterDataset) else MaskedLMDataset(dev_ds)
dev_loader = DataLoader(dev_ds, batch_size=args.bsize, shuffle=False, num_workers=args.np)
test_ds = task_dstype(os.path.join(DATA_PATH, task_path, 'test_%i.%s' % (i, args.fmt)), task_cols['X'], task_cols['y'], config.encode_func, tokenizer, config, sep='\t', index_col=task_cols['index'], binlb=task_extparms['binlb'] if 'binlb' in task_extparms and type(task_extparms['binlb']) is not str else all_binlb, transforms=trsfms, transforms_kwargs=trsfms_kwargs, mltl=task_extparms.setdefault('mltl', False), **ds_kwargs)
if mdl_name.startswith('bert'): test_ds = MaskedLMIterDataset(train_ds) if isinstance(train_ds, BaseIterDataset) else MaskedLMDataset(test_ds)
test_loader = DataLoader(test_ds, batch_size=args.bsize, shuffle=False, num_workers=args.np)
logging.debug(('binlb', train_ds.binlb, dev_ds.binlb, test_ds.binlb))
# Adjust the model
clf.get_linear(binlb=train_ds.binlb, idx=i)
# Training on splitted datasets
train(clf, optimizer, train_loader, config, special_tknids_args, scheduler=scheduler, pad_val=(task_extparms.setdefault('xpad_val', 0), train_ds.binlb[task_extparms.setdefault('ypad_val', 0)]) if task_type=='nmt' else task_extparms.setdefault('xpad_val', 0), weights=class_weights, lmcoef=args.lmcoef, clipmaxn=args.clipmaxn, epochs=args.epochs, earlystop=args.earlystop, earlystop_delta=args.es_delta, earlystop_patience=args.es_patience, task_type=task_type, task_name=args.task, mdl_name=args.model, use_gpu=use_gpu, devq=dev_id, resume=resume if args.resume else {}, chckpnt_kwargs=dict(mltclf_epochs=epoch))
# Adjust the model
clf_trnsfmr = MultiClfTransformer(clf)
clf_trnsfmr.merge_linear(num_linear=i+1)
clf.linear = _handle_model(clf.linear, dev_id=dev_id, distrb=args.distrb)
# Evaluating on the accumulated dev and test sets
eval(clf, dev_loader, config, dev_ds.binlbr, special_tknids_args, pad_val=(task_extparms.setdefault('xpad_val', 0), train_ds.binlb[task_extparms.setdefault('ypad_val', 0)]) if task_type=='nmt' else task_extparms.setdefault('xpad_val', 0), task_type=task_type, task_name=args.task, ds_name='dev', mdl_name=args.model, use_gpu=use_gpu, ignored_label=task_extparms.setdefault('ignored_label', None))
eval(clf, test_loader, config, test_ds.binlbr, special_tknids_args, pad_val=(task_extparms.setdefault('xpad_val', 0), train_ds.binlb[task_extparms.setdefault('ypad_val', 0)]) if task_type=='nmt' else task_extparms.setdefault('xpad_val', 0), task_type=task_type, task_name=args.task, ds_name='test', mdl_name=args.model, use_gpu=use_gpu, ignored_label=task_extparms.setdefault('ignored_label', None))
global_all_binlb.update(all_binlb)
# clf.binlb = all_binlb
# clf.binlbr = dict([(v, k) for k, v in all_binlb.items()])
else:
if orig_epochs > 0:
try:
save_model(clf, optimizer, '%s_%s.pth' % (args.task, args.model), devq=dev_id, distrb=args.distrb)
except Exception as e:
logging.warning(e)
args.epochs = orig_epochs
if args.noeval: return
dev_ds = task_dstype(os.path.join(DATA_PATH, task_path, 'dev.%s' % args.fmt), task_cols['X'], task_cols['y'], config.encode_func, tokenizer, config, sep='\t', index_col=task_cols['index'], binlb=task_extparms['binlb'] if 'binlb' in task_extparms and type(task_extparms['binlb']) is not str else all_binlb, transforms=trsfms, transforms_kwargs=trsfms_kwargs, mltl=task_extparms.setdefault('mltl', False), **ds_kwargs)
if mdl_name.startswith('bert'): dev_ds = MaskedLMIterDataset(train_ds) if isinstance(train_ds, BaseIterDataset) else MaskedLMDataset(dev_ds)
dev_loader = DataLoader(dev_ds, batch_size=args.bsize, shuffle=False, num_workers=args.np)
test_ds = task_dstype(os.path.join(DATA_PATH, task_path, 'test.%s' % args.fmt), task_cols['X'], task_cols['y'], config.encode_func, tokenizer, config, sep='\t', index_col=task_cols['index'], binlb=task_extparms['binlb'] if 'binlb' in task_extparms and type(task_extparms['binlb']) is not str else all_binlb, transforms=trsfms, transforms_kwargs=trsfms_kwargs, mltl=task_extparms.setdefault('mltl', False), **ds_kwargs)
if mdl_name.startswith('bert'): test_ds = MaskedLMIterDataset(train_ds) if isinstance(train_ds, BaseIterDataset) else MaskedLMDataset(test_ds)
test_loader = DataLoader(test_ds, batch_size=args.bsize, shuffle=False, num_workers=args.np)
# Evaluation
eval(clf, dev_loader, config, dev_ds.binlbr, special_tknids_args, pad_val=(task_extparms.setdefault('xpad_val', 0), train_ds.binlb[task_extparms.setdefault('ypad_val', 0)]) if task_type=='nmt' else task_extparms.setdefault('xpad_val', 0), task_type=task_type, task_name=args.task, ds_name='dev', mdl_name=args.model, use_gpu=use_gpu, ignored_label=task_extparms.setdefault('ignored_label', None))
if args.traindev: train(clf, optimizer, dev_loader, config, special_tknids_args, scheduler=scheduler, pad_val=(task_extparms.setdefault('xpad_val', 0), train_ds.binlb[task_extparms.setdefault('ypad_val', 0)]) if task_type=='nmt' else task_extparms.setdefault('xpad_val', 0), weights=class_weights, lmcoef=args.lmcoef, clipmaxn=args.clipmaxn, epochs=orig_epochs, earlystop=args.earlystop, earlystop_delta=args.es_delta, earlystop_patience=args.es_patience, task_type=task_type, task_name=args.task, mdl_name=args.model, use_gpu=use_gpu, devq=dev_id)
eval(clf, test_loader, config, test_ds.binlbr, special_tknids_args, pad_val=(task_extparms.setdefault('xpad_val', 0), train_ds.binlb[task_extparms.setdefault('ypad_val', 0)]) if task_type=='nmt' else task_extparms.setdefault('xpad_val', 0), task_type=task_type, task_name=args.task, ds_name='test', mdl_name=args.model, use_gpu=use_gpu, ignored_label=task_extparms.setdefault('ignored_label', None))
from util import trainer
from data.train_data_provider import ImageDataProvider
from model.unet import Unet
output_path = "/data/Cell/unet/model3/"
data_provider = ImageDataProvider("/data/Cell/unet/*.jpg")
net = Unet(layers=3, features_root=32, channels=3, n_class=2)
trainer = trainer.Trainer(net, optimizer="adam")
path = trainer.train(data_provider, output_path, training_iters=32, epochs=100)