def main(self, name, opts): logging.basicConfig(filename=opts.log_file, format='%(levelname)s (%(asctime)s): %(message)s') log = logging.getLogger(name) if opts.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) if not opts.model_files: raise ValueError('No model files provided!') log.info('Loading model ...') model = mod.load_model(opts.model_files) log.info('Loading data ...') nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample) replicate_names = dat.get_replicate_names(opts.data_files[0], regex=opts.replicate_names, nb_key=opts.nb_replicate) data_reader = mod.data_reader_from_model( model, replicate_names, replicate_names=replicate_names) # Seed used since unobserved input CpG states are randomly sampled if opts.seed is not None: np.random.seed(opts.seed) random.seed(opts.seed) data_reader = data_reader(opts.data_files, nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'], nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) writer = None if opts.out_data: writer = H5Writer(opts.out_data, nb_sample) log.info('Predicting ...') nb_tot = 0 nb_eval = 0 data_eval = dict() perf_eval = [] progbar = ProgressBar(nb_sample, log.info) for inputs, outputs, weights in data_reader: batch_size = len(list(inputs.values())[0]) nb_tot += batch_size progbar.update(batch_size) preds = to_list(model.predict(inputs)) data_batch = dict() data_batch['preds'] = dict() data_batch['outputs'] = dict() for i, name in enumerate(model.output_names): data_batch['preds'][name] = preds[i].squeeze() data_batch['outputs'][name] = outputs[name].squeeze() for name, value in six.iteritems(next(meta_reader)): data_batch[name] = value if writer: writer.write_dict(data_batch) nb_eval += batch_size dat.add_to_dict(data_batch, data_eval) if nb_tot >= nb_sample or \ (opts.eval_size and nb_eval >= opts.eval_size): data_eval = dat.stack_dict(data_eval) perf_eval.append( ev.evaluate_outputs(data_eval['outputs'], data_eval['preds'])) data_eval = dict() nb_eval = 0 progbar.close() if writer: writer.close() report = pd.concat(perf_eval) report = report.groupby(['metric', 'output']).mean().reset_index() if opts.out_report: report.to_csv(opts.out_report, sep='\t', index=False) report = ev.unstack_report(report) print(report.to_string()) log.info('Done!') return 0
def main(self, name, opts): logging.basicConfig(filename=opts.log_file, format='%(levelname)s (%(asctime)s): %(message)s') log = logging.getLogger(name) if opts.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) log.debug(opts) if opts.seed is not None: np.random.seed(opts.seed) if not opts.model_files: raise ValueError('No model files provided!') log.info('Loading model ...') K.set_learning_phase(0) model = mod.load_model(opts.model_files, log=log.info) weight_layer, act_layer = mod.get_first_conv_layer(model.layers, True) log.info('Using activation layer "%s"' % act_layer.name) log.info('Using weight layer "%s"' % weight_layer.name) try: dna_idx = model.input_names.index('dna') except BaseException: raise IOError('Model is not a valid DNA model!') fun_outputs = to_list(act_layer.output) if opts.store_preds: fun_outputs += to_list(model.output) fun = K.function([to_list(model.input)[dna_idx]], fun_outputs) log.info('Reading data ...') if opts.store_outputs or opts.store_preds: output_names = model.output_names else: output_names = None data_reader = mod.DataReader( output_names=output_names, use_dna=True, dna_wlen=to_list(model.input_shape)[dna_idx][1] ) nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample) data_reader = data_reader(opts.data_files, nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=opts.shuffle) meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'], nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) out_file = h5.File(opts.out_file, 'w') out_group = out_file weights = weight_layer.get_weights() out_group['weights/weights'] = weights[0] out_group['weights/bias'] = weights[1] def h5_dump(path, data, idx, dtype=None, compression='gzip'): if path not in out_group: if dtype is None: dtype = data.dtype out_group.create_dataset( name=path, shape=[nb_sample] + list(data.shape[1:]), dtype=dtype, compression=compression ) out_group[path][idx:idx+len(data)] = data log.info('Computing activations') progbar = ProgressBar(nb_sample, log.info) idx = 0 for data in data_reader: if isinstance(data, tuple): inputs, outputs, weights = data else: inputs = data if isinstance(inputs, dict): inputs = list(inputs.values()) batch_size = len(inputs[0]) progbar.update(batch_size) if opts.store_inputs: for i, name in enumerate(model.input_names): h5_dump('inputs/%s' % name, dna.onehot_to_int(inputs[i]), idx) if opts.store_outputs: for name, output in six.iteritems(outputs): h5_dump('outputs/%s' % name, output, idx) fun_eval = fun(inputs) act = fun_eval[0] if opts.act_wlen: delta = opts.act_wlen // 2 ctr = act.shape[1] // 2 act = act[:, (ctr-delta):(ctr+delta+1)] if opts.act_fun: if opts.act_fun == 'mean': act = act.mean(axis=1) elif opts.act_fun == 'wmean': weights = linear_weights(act.shape[1]) act = np.average(act, axis=1, weights=weights) elif opts.act_fun == 'max': act = act.max(axis=1) else: raise ValueError('Invalid function "%s"!' % (opts.act_fun)) h5_dump('act', act, idx) if opts.store_preds: preds = fun_eval[1:] for i, name in enumerate(model.output_names): h5_dump('preds/%s' % name, preds[i].squeeze(), idx) for name, value in six.iteritems(next(meta_reader)): h5_dump(name, value, idx) idx += batch_size progbar.close() out_file.close() log.info('Done!') return 0
def main(self, name, opts): logging.basicConfig(filename=opts.log_file, format='%(levelname)s (%(asctime)s): %(message)s') log = logging.getLogger(name) if opts.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) log.debug(opts) if opts.seed is not None: np.random.seed(opts.seed) if not opts.model_files: raise ValueError('No model files provided!') log.info('Loading model ...') K.set_learning_phase(0) model = mod.load_model(opts.model_files) # Get DNA layer. dna_layer = None for layer in model.layers: if layer.name == 'dna': dna_layer = layer break if not dna_layer: raise ValueError('The provided model is not a DNA model!') # Create output vector. outputs = [] for output in model.outputs: outputs.append(K.reshape(output, (-1, 1))) outputs = K.concatenate(outputs, axis=1) # Compute gradient of outputs wrt. DNA layer. grads = [] for name in opts.targets: if name == 'mean': target = K.mean(outputs, axis=1) elif name == 'var': target = K.var(outputs, axis=1) else: raise ValueError('Invalid effect size "%s"!' % name) grad = K.gradients(target, dna_layer.output) grads.extend(grad) grad_fun = K.function(model.inputs, grads) log.info('Reading data ...') nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample) replicate_names = dat.get_replicate_names(opts.data_files[0], regex=opts.replicate_names, nb_key=opts.nb_replicate) data_reader = mod.data_reader_from_model( model, outputs=False, replicate_names=replicate_names) data_reader = data_reader(opts.data_files, nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'], nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) out_file = h5.File(opts.out_file, 'w') out_group = out_file def h5_dump(path, data, idx, dtype=None, compression='gzip'): if path not in out_group: if dtype is None: dtype = data.dtype out_group.create_dataset(name=path, shape=[nb_sample] + list(data.shape[1:]), dtype=dtype, compression=compression) out_group[path][idx:idx + len(data)] = data log.info('Computing effects ...') progbar = ProgressBar(nb_sample, log.info) idx = 0 for inputs in data_reader: if isinstance(inputs, dict): inputs = list(inputs.values()) batch_size = len(inputs[0]) progbar.update(batch_size) # Compute gradients. grads = grad_fun(inputs) # Slice window at center. if opts.dna_wlen: for i, grad in enumerate(grads): delta = opts.dna_wlen // 2 ctr = grad.shape[1] // 2 grads[i] = grad[:, (ctr - delta):(ctr + delta + 1)] # Aggregate effects in window if opts.agg_effects: for i, grad in enumerate(grads): if opts.agg_effects == 'mean': grad = grad.mean(axis=1) elif opts.agg_effects == 'wmean': weights = linear_weights(grad.shape[1]) grad = np.average(grad, axis=1, weights=weights) elif opts.agg_effects == 'max': grad = grad.max(axis=1) else: tmp = 'Invalid function "%s"!' % (opts.agg_effects) raise ValueError(tmp) grads[i] = grad # Write computed effects for name, grad in zip(opts.targets, grads): h5_dump(name, grad, idx) # Store inputs if opts.store_inputs: for name, value in zip(model.input_names, inputs): h5_dump(name, value, idx) # Store positions for name, value in next(meta_reader).items(): h5_dump(name, value, idx) idx += batch_size progbar.close() out_file.close() log.info('Done!') return 0
def main(self, name, opts): logging.basicConfig(filename=opts.log_file, format='%(levelname)s (%(asctime)s): %(message)s') log = logging.getLogger(name) if opts.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) log.debug(opts) if opts.seed is not None: np.random.seed(opts.seed) if not opts.model_files: raise ValueError('No model files provided!') log.info('Loading model ...') K.set_learning_phase(0) model = mod.load_model(opts.model_files, log=log.info) weight_layer, act_layer = mod.get_first_conv_layer(model.layers, True) log.info('Using activation layer "%s"' % act_layer.name) log.info('Using weight layer "%s"' % weight_layer.name) try: dna_idx = model.input_names.index('dna') except BaseException: raise IOError('Model is not a valid DNA model!') fun_outputs = to_list(act_layer.output) if opts.store_preds: fun_outputs += to_list(model.output) fun = K.function([to_list(model.input)[dna_idx]], fun_outputs) log.info('Reading data ...') if opts.store_outputs or opts.store_preds: output_names = model.output_names else: output_names = None data_reader = mod.DataReader(output_names=output_names, use_dna=True, dna_wlen=to_list( model.input_shape)[dna_idx][1]) nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample) data_reader = data_reader(opts.data_files, nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'], nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) out_file = h5.File(opts.out_file, 'w') out_group = out_file weights = weight_layer.get_weights() out_group['weights/weights'] = weights[0] out_group['weights/bias'] = weights[1] def h5_dump(path, data, idx, dtype=None, compression='gzip'): if path not in out_group: if dtype is None: dtype = data.dtype out_group.create_dataset(name=path, shape=[nb_sample] + list(data.shape[1:]), dtype=dtype, compression=compression) out_group[path][idx:idx + len(data)] = data log.info('Computing activations') progbar = ProgressBar(nb_sample, log.info) idx = 0 for data in data_reader: if isinstance(data, tuple): inputs, outputs, weights = data else: inputs = data if isinstance(inputs, dict): inputs = list(inputs.values()) batch_size = len(inputs[0]) progbar.update(batch_size) if opts.store_inputs: for i, name in enumerate(model.input_names): h5_dump('inputs/%s' % name, dna.onehot_to_int(inputs[i]), idx) if opts.store_outputs: for name, output in six.iteritems(outputs): h5_dump('outputs/%s' % name, output, idx) fun_eval = fun(inputs) act = fun_eval[0] if opts.act_wlen: delta = opts.act_wlen // 2 ctr = act.shape[1] // 2 act = act[:, (ctr - delta):(ctr + delta + 1)] if opts.act_fun: if opts.act_fun == 'mean': act = act.mean(axis=1) elif opts.act_fun == 'wmean': weights = linear_weights(act.shape[1]) act = np.average(act, axis=1, weights=weights) elif opts.act_fun == 'max': act = act.max(axis=1) else: raise ValueError('Invalid function "%s"!' % (opts.act_fun)) h5_dump('act', act, idx) if opts.store_preds: preds = fun_eval[1:] for i, name in enumerate(model.output_names): h5_dump('preds/%s' % name, preds[i].squeeze(), idx) for name, value in six.iteritems(next(meta_reader)): h5_dump(name, value, idx) idx += batch_size progbar.close() out_file.close() log.info('Done!') return 0
def main(self, name, opts): logging.basicConfig(filename=opts.log_file, format='%(levelname)s (%(asctime)s): %(message)s') log = logging.getLogger(name) if opts.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) log.debug(opts) if opts.seed is not None: np.random.seed(opts.seed) if not opts.model_files: raise ValueError('No model files provided!') log.info('Loading model ...') K.set_learning_phase(0) model = mod.load_model(opts.model_files) # Get DNA layer. dna_layer = None for layer in model.layers: if layer.name == 'dna': dna_layer = layer break if not dna_layer: raise ValueError('The provided model is not a DNA model!') # Create output vector. outputs = [] for output in model.outputs: outputs.append(K.reshape(output, (-1, 1))) outputs = K.concatenate(outputs, axis=1) # Compute gradient of outputs wrt. DNA layer. grads = [] for name in opts.targets: if name == 'mean': target = K.mean(outputs, axis=1) elif name == 'var': target = K.var(outputs, axis=1) else: raise ValueError('Invalid effect size "%s"!' % name) grad = K.gradients(target, dna_layer.output) grads.extend(grad) grad_fun = K.function(model.inputs, grads) log.info('Reading data ...') nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample) replicate_names = dat.get_replicate_names( opts.data_files[0], regex=opts.replicate_names, nb_key=opts.nb_replicate) data_reader = mod.data_reader_from_model( model, outputs=False, replicate_names=replicate_names) data_reader = data_reader(opts.data_files, nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'], nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) out_file = h5.File(opts.out_file, 'w') out_group = out_file def h5_dump(path, data, idx, dtype=None, compression='gzip'): if path not in out_group: if dtype is None: dtype = data.dtype out_group.create_dataset( name=path, shape=[nb_sample] + list(data.shape[1:]), dtype=dtype, compression=compression ) out_group[path][idx:idx+len(data)] = data log.info('Computing effects ...') progbar = ProgressBar(nb_sample, log.info) idx = 0 for inputs in data_reader: if isinstance(inputs, dict): inputs = list(inputs.values()) batch_size = len(inputs[0]) progbar.update(batch_size) # Compute gradients. grads = grad_fun(inputs) # Slice window at center. if opts.dna_wlen: for i, grad in enumerate(grads): delta = opts.dna_wlen // 2 ctr = grad.shape[1] // 2 grads[i] = grad[:, (ctr-delta):(ctr+delta+1)] # Aggregate effects in window if opts.agg_effects: for i, grad in enumerate(grads): if opts.agg_effects == 'mean': grad = grad.mean(axis=1) elif opts.agg_effects == 'wmean': weights = linear_weights(grad.shape[1]) grad = np.average(grad, axis=1, weights=weights) elif opts.agg_effects == 'max': grad = grad.max(axis=1) else: tmp = 'Invalid function "%s"!' % (opts.agg_effects) raise ValueError(tmp) grads[i] = grad # Write computed effects for name, grad in zip(opts.targets, grads): h5_dump(name, grad, idx) # Store inputs if opts.store_inputs: for name, value in zip(model.input_names, inputs): h5_dump(name, value, idx) # Store positions for name, value in next(meta_reader).items(): h5_dump(name, value, idx) idx += batch_size progbar.close() out_file.close() log.info('Done!') return 0
def main(self, name, opts): logging.basicConfig(filename=opts.log_file, format='%(levelname)s (%(asctime)s): %(message)s') log = logging.getLogger(name) if opts.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) if not opts.model_files: raise ValueError('No model files provided!') log.info('Loading model ...') model = mod.load_model(opts.model_files) log.info('Loading data ...') nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample) replicate_names = dat.get_replicate_names( opts.data_files[0], regex=opts.replicate_names, nb_key=opts.nb_replicate) data_reader = mod.data_reader_from_model( model, replicate_names, replicate_names=replicate_names) data_reader = data_reader(opts.data_files, nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'], nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) log.info('Predicting ...') data = dict() progbar = ProgressBar(nb_sample, log.info) for inputs, outputs, weights in data_reader: batch_size = len(list(inputs.values())[0]) progbar.update(batch_size) preds = to_list(model.predict(inputs)) data_batch = dict() data_batch['preds'] = dict() data_batch['outputs'] = dict() for i, name in enumerate(model.output_names): data_batch['preds'][name] = preds[i].squeeze() data_batch['outputs'][name] = outputs[name].squeeze() for name, value in next(meta_reader).items(): data_batch[name] = value dat.add_to_dict(data_batch, data) progbar.close() data = dat.stack_dict(data) report = ev.evaluate_outputs(data['outputs'], data['preds']) if opts.out_report: report.to_csv(opts.out_report, sep='\t', index=False) report = ev.unstack_report(report) print(report.to_string()) if opts.out_data: hdf.write_data(data, opts.out_data) log.info('Done!') return 0
def main(self, name, opts): logging.basicConfig(filename=opts.log_file, format='%(levelname)s (%(asctime)s): %(message)s') log = logging.getLogger(name) if opts.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) if not opts.model_files: raise ValueError('No model files provided!') log.info('Loading model ...') model = mod.load_model(opts.model_files) log.info('Loading data ...') nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample) replicate_names = dat.get_replicate_names( opts.data_files[0], regex=opts.replicate_names, nb_key=opts.nb_replicate) data_reader = mod.data_reader_from_model( model, replicate_names, replicate_names=replicate_names) # Seed used since unobserved input CpG states are randomly sampled if opts.seed is not None: np.random.seed(opts.seed) random.seed(opts.seed) data_reader = data_reader(opts.data_files, nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'], nb_sample=nb_sample, batch_size=opts.batch_size, loop=False, shuffle=False) writer = None if opts.out_data: writer = H5Writer(opts.out_data, nb_sample) log.info('Predicting ...') nb_tot = 0 nb_eval = 0 data_eval = dict() perf_eval = [] progbar = ProgressBar(nb_sample, log.info) for inputs, outputs, weights in data_reader: batch_size = len(list(inputs.values())[0]) nb_tot += batch_size progbar.update(batch_size) preds = to_list(model.predict(inputs)) data_batch = dict() data_batch['preds'] = dict() data_batch['outputs'] = dict() for i, name in enumerate(model.output_names): data_batch['preds'][name] = preds[i].squeeze() data_batch['outputs'][name] = outputs[name].squeeze() for name, value in six.iteritems(next(meta_reader)): data_batch[name] = value if writer: writer.write_dict(data_batch) nb_eval += batch_size dat.add_to_dict(data_batch, data_eval) if nb_tot >= nb_sample or \ (opts.eval_size and nb_eval >= opts.eval_size): data_eval = dat.stack_dict(data_eval) perf_eval.append(ev.evaluate_outputs(data_eval['outputs'], data_eval['preds'])) data_eval = dict() nb_eval = 0 progbar.close() if writer: writer.close() report = pd.concat(perf_eval) report = report.groupby(['metric', 'output']).mean().reset_index() if opts.out_report: report.to_csv(opts.out_report, sep='\t', index=False) report = ev.unstack_report(report) print(report.to_string()) log.info('Done!') return 0