def main():
usage = 'usage: %prog [options] <params_file> <data1_dir> <data2_dir> ...'
parser = OptionParser(usage)
parser.add_option(
'-o',
dest='out_dir',
default='train2_out',
help='Output directory for test statistics [Default: %default]')
parser.add_option(
'--restore',
dest='restore',
help='Restore model and continue training [Default: %default]')
parser.add_option('--trunk',
dest='trunk',
default=False,
action='store_true',
help='Restore only model trunk [Default: %default]')
parser.add_option(
'--tfr_train',
dest='tfr_train_pattern',
default=None,
help=
'Training TFR pattern string appended to data_dir/tfrecords for subsetting [Default: %default]'
)
parser.add_option(
'--tfr_eval',
dest='tfr_eval_pattern',
default=None,
help=
'Evaluation TFR pattern string appended to data_dir/tfrecords for subsetting [Default: %default]'
)
(options, args) = parser.parse_args()
if len(args) < 2:
parser.error('Must provide parameters and data directory.')
else:
params_file = args[0]
data_dirs = args[1:]
if not os.path.isdir(options.out_dir):
os.mkdir(options.out_dir)
if params_file != '%s/params.json' % options.out_dir:
shutil.copy(params_file, '%s/params.json' % options.out_dir)
# read model parameters
with open(params_file) as params_open:
params = json.load(params_open)
params_model = params['model']
params_train = params['train']
# read datasets
train_data = []
eval_data = []
for data_dir in data_dirs:
# load train data
train_data.append(
dataset.SeqDataset(data_dir,
split_label='train',
batch_size=params_train['batch_size'],
mode=tf.estimator.ModeKeys.TRAIN,
tfr_pattern=options.tfr_train_pattern))
# load eval data
eval_data.append(
dataset.SeqDataset(data_dir,
split_label='valid',
batch_size=params_train['batch_size'],
mode=tf.estimator.ModeKeys.EVAL,
tfr_pattern=options.tfr_eval_pattern))
if params_train.get('num_gpu', 1) == 1:
########################################
# one GPU
# initialize model
seqnn_model = seqnn.SeqNN(params_model)
# restore
if options.restore:
seqnn_model.restore(options.restore, options.trunk)
# initialize trainer
seqnn_trainer = trainer.Trainer(params_train, train_data, eval_data,
options.out_dir)
# compile model
seqnn_trainer.compile(seqnn_model)
# train model
seqnn_trainer.fit2(seqnn_model)
else:
########################################
# two GPU
strategy = tf.distribute.MirroredStrategy()
with strategy.scope():
# distribute data
for di in range(len(data_dirs)):
train_data[di].distribute(strategy)
eval_data[di].distribute(strategy)
# initialize model
seqnn_model = seqnn.SeqNN(params_model)
# restore
if options.restore:
seqnn_model.restore(options.restore, options.trunk)
# initialize trainer
seqnn_trainer = trainer.Trainer(params_train, train_data,
eval_data, options.out_dir,
strategy, params_train['num_gpu'])
# compile model
seqnn_trainer.compile(seqnn_model)
# train model
seqnn_trainer.fit2(seqnn_model)
def main(_):
# I could write some additional code around this to check for common
# problems, such as with num_targets.
with open(FLAGS.params) as params_open:
params = json.load(params_open)
params_model = params['model']
params_train = params['train']
if params_train.get('use_gpu', 1) == False:
os.environ["CUDA_VISIBLE_DEVICES"] = '-1'
print(" ")
print(" training on CPU ")
print(" ")
#need to blind to CPUs before tf is imported
import shutil
if not os.path.isdir(FLAGS.log_dir):
os.mkdir(FLAGS.log_dir)
if not os.path.isfile(FLAGS.log_dir + '/params.json'):
shutil.copy(FLAGS.params, FLAGS.log_dir + '/params.json')
import tensorflow as tf
if tf.__version__[0] == '1':
tf.compat.v1.enable_eager_execution()
print('tf version:', tf.__version__)
from basenji import dataset
from basenji import seqnn
from basenji import trainer
# load data
diagonal_offset = params_model.get('diagonal_offset', 2)
target_crop = params_model.get('target_crop', 0)
target_length_crop = params_model[
'target_length'] - diagonal_offset - 2 * target_crop
tlen = target_length_crop * (target_length_crop + 1) // 2
train_data = dataset.SeqDataset(FLAGS.train_data,
params_train['batch_size'],
params_model['seq_length'], tlen,
tf.estimator.ModeKeys.TRAIN)
eval_data = dataset.SeqDataset(FLAGS.eval_data, params_train['batch_size'],
params_model['seq_length'], tlen,
tf.estimator.ModeKeys.EVAL)
if params_train.get('num_gpu', 1) == 1:
########################################
# one GPU
# initialize model
seqnn_model = seqnn.SeqNN(params_model)
# restore
if FLAGS.restore:
seqnn_model.restore(FLAGS.restore, FLAGS.trunk)
print('restored weights')
if FLAGS.freeze_trunk:
seqnn_model.model_trunk.trainable = False
print('frozen trunk')
# initialize trainer
seqnn_trainer = trainer.Trainer(params_train, train_data, eval_data)
# compile model
seqnn_trainer.compile(seqnn_model.model)
# train model
seqnn_trainer.fit(seqnn_model.model)
else:
########################################
# two GPU
print('need to update multigpu')
'''
def main():
usage = 'usage: %prog [options] <params_file> <data1_dir> <data2_dir> ...'
parser = OptionParser(usage)
parser.add_option('-o', dest='out_dir',
default='train2_out',
help='Output directory for test statistics [Default: %default]')
parser.add_option('--restore', dest='restore',
help='Restore model and continue training [Default: %default]')
parser.add_option('--trunk', dest='trunk',
default=False, action='store_true',
help='Restore only model trunk [Default: %default]')
parser.add_option('--tfr_train', dest='tfr_train_pattern',
default='train-*.tfr',
help='Training TFRecord pattern string appended to data_dir [Default: %default]')
parser.add_option('--tfr_eval', dest='tfr_eval_pattern',
default='valid-*.tfr',
help='Evaluation TFRecord pattern string appended to data_dir [Default: %default]')
(options, args) = parser.parse_args()
if len(args) < 2:
parser.error('Must provide parameters and data directory.')
else:
params_file = args[0]
data_dirs = args[1:]
if not os.path.isdir(options.out_dir):
os.mkdir(options.out_dir)
if params_file != '%s/params.json' % options.out_dir:
shutil.copy(params_file, '%s/params.json' % options.out_dir)
# read model parameters
with open(params_file) as params_open:
params = json.load(params_open)
params_model = params['model']
params_train = params['train']
# read datasets
data_stats = []
train_data = []
eval_data = []
for data_dir in data_dirs:
# read data parameters
data_stats_file = '%s/statistics.json' % data_dir
with open(data_stats_file) as data_stats_open:
data_stats.append(json.load(data_stats_open))
# load train data
tfr_train_full = '%s/tfrecords/%s' % (data_dir, options.tfr_train_pattern)
train_data.append(dataset.SeqDataset(tfr_train_full,
seq_length=data_stats[0]['seq_length'],
target_length=data_stats[0]['target_length'],
batch_size=params_train['batch_size'],
mode=tf.estimator.ModeKeys.TRAIN))
# load eval data
tfr_eval_full = '%s/tfrecords/%s' % (data_dir, options.tfr_eval_pattern)
eval_data.append(dataset.SeqDataset(tfr_eval_full,
seq_length=data_stats[0]['seq_length'],
target_length=data_stats[0]['target_length'],
batch_size=params_train['batch_size'],
mode=tf.estimator.ModeKeys.EVAL))
if params_train.get('num_gpu', 1) == 1:
########################################
# one GPU
# initialize model
seqnn_model = seqnn.SeqNN(params_model)
# restore
if options.restore:
seqnn_model.restore(options.restore, options.trunk)
# initialize trainer
seqnn_trainer = trainer.Trainer(params_train, train_data,
eval_data, options.out_dir)
# compile model
seqnn_trainer.compile(seqnn_model)
# train model
seqnn_trainer.fit2(seqnn_model)
else:
########################################
# two GPU
print('Multiple GPUs untested for joint genome training.', file=sys.stderr)
exit(1)
mirrored_strategy = tf.distribute.MirroredStrategy()
with mirrored_strategy.scope():
# initialize model
seqnn_model = seqnn.SeqNN(params_model)
# restore
if options.restore:
seqnn_model.restore(options.restore, options.trunk)
# initialize trainer
seqnn_trainer = trainer.Trainer(params_train, train_data,
eval_data, options.out_dir)
# compile model
seqnn_trainer.compile(seqnn_model)
# train model
seqnn_trainer.fit2(seqnn_model)
def main(_):
# I could write some additional code around this to check for common
# problems, such as with num_targets.
with open(FLAGS.params) as params_open:
params = json.load(params_open)
params_model = params['model']
params_train = params['train']
if params_train.get('use_gpu', 1) == False:
os.environ["CUDA_VISIBLE_DEVICES"] = '-1'
print(" ")
print(" training on CPU ")
print(" ")
# load data
train_data = dataset.SeqDataset(FLAGS.train_data,
params_train['batch_size'],
params_model['seq_length'],
params_model['target_length'],
tf.estimator.ModeKeys.TRAIN)
eval_data = dataset.SeqDataset(FLAGS.eval_data, params_train['batch_size'],
params_model['seq_length'],
params_model['target_length'],
tf.estimator.ModeKeys.EVAL)
if params_train.get('num_gpu', 1) == 1:
########################################
# one GPU
# initialize model
seqnn_model = seqnn.SeqNN(params_model)
# restore
if FLAGS.restore:
seqnn_model.restore(FLAGS.restore, FLAGS.trunk)
# initialize trainer
seqnn_trainer = trainer.Trainer(params_train, train_data, eval_data)
# compile model
seqnn_trainer.compile(seqnn_model.model)
# train model
seqnn_trainer.fit(seqnn_model.model)
else:
########################################
# two GPU
mirrored_strategy = tf.distribute.MirroredStrategy()
with mirrored_strategy.scope():
# initialize model
seqnn_model = seqnn.SeqNN(params_model)
# restore
if FLAGS.restore:
seqnn_model.restore(FLAGS.restore, FLAGS.trunk)
# initialize trainer
seqnn_trainer = trainer.Trainer(params_train, train_data,
eval_data)
# compile model
seqnn_trainer.compile(seqnn_model.model, None)
# train model
seqnn_trainer.fit(seqnn_model.model)