def asbatches(self, batchsize=64, reshuffle=False):
n = len(self)
assert n > 0
nbatch = (n + batchsize - 1) // batchsize
nseq = len(self.sequencenames)
padding = self.requirements.get('padding', 0)
batches = []
for i in range(nbatch):
# Slice a our data attributes row-wise, according to batch index
batch = self[np.arange(i * batchsize, min(n, (i + 1) * batchsize))]
batch._insert_reversecomplements()
# Convert each sequence attribute from a list of strings ("GATC") to a
# single contiguous numpy array X (0..3), along with a list of
# regions R that identify the batch-relative offsets to the start/end
# of each individual sequence
for i in range(nseq):
Xname, Rname = self._seqattrnames(i)
batchX = getattr(batch, Xname)
batchR = np.asarray(
np.cumsum([0] + [padding + len(x) for x in batchX]),
np.uint32).reshape((-1, 1))
batchR = np.hstack([batchR[:-1], batchR[1:]])
# Convert list of strings to giant contiguous array of integers 0..3,
# with padding values of 255 put between the individual sequences
batchX = acgt2ord(
("." * padding).join([""] + [x for x in batchX] +
[""])).reshape((-1, 1))
# Convert each batch from numpy array to sarray,
# and then quickly forget about the numpy batch
batchX = sm.asarray(batchX)
batchR = sm.asarray(batchR)
setattr(batch, Xname, batchX)
setattr(batch, Rname, batchR)
setattr(batch, "regions", batchR)
batch._data_attrs = batch._data_attrs + ("regions", )
if hasattr(batch, "F") and batch.F is not None:
batch.F = sm.asarray(batch.F, sm.get_default_dtype())
if hasattr(batch, "Y") and batch.Y is not None:
batch.Y = sm.asarray(batch.Y, sm.get_default_dtype())
if isinstance(batch.Ymask, np.ndarray):
batch.Ymask = sm.asarray(batch.Ymask)
batches.append(batch)
return deepity.shuffled_repeat_iter(batches, reshuffle)
def _fprop(self,X):
if X is None:
return None
if np.isscalar(self.rate):
if self.rate == 0:
return X
self.rate = sm.asarray([self.rate for i in range(self.ninst)], dtype=X.dtype)
elif not isinstance(self.rate, sm.sarray):
self.rate = sm.asarray(self.rate,dtype=X.dtype)
if "train_mode" in globals.flags:
Z,self.M = _ext.dropout_fp_train(X, self.rate, "reverse_complement" in globals.flags)
else:
Z = _ext.dropout_fp_test(X, self.rate)
return Z
def asbatches(self, batchsize=64, reshuffle=False):
n = len(self)
assert n > 0
nbatch = (n + batchsize - 1) // batchsize
nseq = len(self.sequencenames)
padding = self.requirements.get('padding',0)
batches = []
for i in range(nbatch):
# Slice a our data attributes row-wise, according to batch index
batch = self[np.arange(i*batchsize,min(n,(i+1)*batchsize))]
batch._insert_reversecomplements()
# Convert each sequence attribute from a list of strings ("GATC") to a
# single contiguous numpy array X (0..3), along with a list of
# regions R that identify the batch-relative offsets to the start/end
# of each individual sequence
for i in range(nseq):
Xname,Rname = self._seqattrnames(i)
batchX = getattr(batch, Xname)
batchR = np.asarray(np.cumsum([0]+[padding+len(x) for x in batchX]),np.uint32).reshape((-1,1))
batchR = np.hstack([batchR[:-1],batchR[1:]])
# Convert list of strings to giant contiguous array of integers 0..3,
# with padding values of 255 put between the individual sequences
batchX = acgt2ord(("."*padding).join([""]+[x for x in batchX]+[""])).reshape((-1,1))
# Convert each batch from numpy array to sarray,
# and then quickly forget about the numpy batch
batchX = sm.asarray(batchX)
batchR = sm.asarray(batchR)
setattr(batch, Xname, batchX)
setattr(batch, Rname, batchR)
setattr(batch, "regions", batchR)
batch._data_attrs = batch._data_attrs + ("regions",)
if hasattr(batch,"F") and batch.F is not None:
batch.F = sm.asarray(batch.F,sm.get_default_dtype())
if hasattr(batch,"Y") and batch.Y is not None:
batch.Y = sm.asarray(batch.Y,sm.get_default_dtype())
if isinstance(batch.Ymask,np.ndarray):
batch.Ymask = sm.asarray(batch.Ymask)
batches.append(batch)
return deepity.shuffled_repeat_iter(batches, reshuffle)
def _fprop(self, X):
if X is None:
return None
if np.isscalar(self.rate):
if self.rate == 0:
return X
self.rate = sm.asarray([self.rate for i in range(self.ninst)],
dtype=X.dtype)
elif not isinstance(self.rate, sm.sarray):
self.rate = sm.asarray(self.rate, dtype=X.dtype)
if "train_mode" in globals.flags:
Z, self.M = _ext.dropout_fp_train(
X, self.rate, "reverse_complement" in globals.flags)
else:
Z = _ext.dropout_fp_test(X, self.rate)
return Z
def _fprop(self,X,R):
if deepity.globals.flags.get("collect_featuremaps",False):
old = deepity.globals.flags.pop("collect_featuremaps")
fmaps = old if isinstance(old, list) else []
fmaps += self._collect_featuremaps(X,R)
deepity.globals.flags.push("collect_featuremaps",fmaps)
Z,self.I = kangaroo_smat.poolrgn(X,R,ptype="max",
want_argmax = "bprop_mode" in deepity.globals.flags)
if "collect_argmax" in deepity.globals.flags:
deepity.globals.flags.pop("collect_argmax")
deepity.globals.flags.push("collect_argmax", self.I.asnumpy())
elif "force_argmax" in deepity.globals.flags:
_I = deepity.globals.flags.get("force_argmax")
_X = X.asnumpy()
_Z = _X.ravel()[_I.ravel()].reshape(Z.shape)
Z = sm.asarray(_Z)
return Z
def gen_convnet_predictions(model, data, want_gmaps=False):
# We must feed each sequence through the model several times
# by applying the model repeatedly on sliding a window along the sequence.
# That generates a prediction map, from which we can take max, sum, etc.
predictions = []
gmaps = {}
batches = data.asbatches(batchsize=2048, reshuffle=False)
for batch in batches:
args = batch.input_data()
args["want_bprop_inputs"] = bool(want_gmaps)
if isinstance(model.Z.origin().node,deepity.std.softmaxnode):
args["bprop_inputs_loss"] = deepity.std.nll()
else:
args["bprop_inputs_loss"] = deepity.std.mse()
globals.flags.push("collect_argmax",None)
outputs = model.eval(**args)
I = globals.flags.pop("collect_argmax")
Z = outputs['Z'].asnumpy()
Z, Zmask = maskout_revcomp(Z)
if Zmask is not None:
if "collect_Zmask" in globals.flags:
global_Zmask = globals.flags.pop("collect_Zmask")
if not isinstance(global_Zmask,np.ndarray):
global_Zmask = Zmask
else:
global_Zmask = np.vstack([global_Zmask, Zmask])
globals.flags.push("collect_Zmask", global_Zmask)
predictions.append(Z)
# If user wants gradientmaps, then for every sequence we need one
if want_gmaps:
for key in args:
dkey = "d"+key
if outputs.get(dkey,None) is not None:
X = args[key].asnumpy()
dX = outputs[dkey].asnumpy()
if X.dtype == np.uint8: # Is it an sequence of ordinals (bytes)?
pad = data.requirements.get("padding",0)
R = args["R"+key[1:]].asnumpy() # regions associated with X
if want_gmaps == "finite_diff":
is_rc = "reverse_complement" in globals.flags
#globals.flags.push("force_argmax",I)
rcindex = [3,2,1,0]
oldF = args['F']
# If user specifically asked for finite differences, not instantaneous gradient,
# then we need to explicitly mutate every position, generate predictions, and
# subtract the result from Z to find the actual delta for each base
Xlen = R[:,1]-R[:,0]
nbase = dX.shape[1]
for i in range(Xlen.max()):
for j in range(nbase):
mtX = X.copy()
mtF = args['F'].asnumpy().copy()
for k in range(len(R)):
a,b = R[k]
if i < b-a:
if (k % 2 == 0) or not is_rc:
mtX[pad+a+i] = j # mutate position i in sequence k (which starts at byte index a) to base j
else:
mtX[b-i-1] = rcindex[j]
mtF[k] = data._generate_dinuc_featurevec(mtX[pad+a:b])
args[key] = sm.asarray(mtX) # This time use the mutated X instead of the original
args['F'] = sm.asarray(mtF)
mtoutputs = model.eval(**args)
mtZ = mtoutputs['Z'].asnumpy()
mtZ, mtZmask = maskout_revcomp(mtZ)
dZ = mtZ-Z # output
dZ *= np.maximum(0,np.sign(np.maximum(Z,mtZ)))
for k in range(len(R)):
if (k % 2 == 0) or not is_rc:
a,b = R[k]
if i < b-a:
dX[pad+a+i,j] = dZ[(k//2) if is_rc else k]
#globals.flags.pop("force_argmax")
args['F'] = oldF
# Only include forward strand in finite_diff results
if is_rc:
dX = [(util.ord2acgt(X[a+pad:b]), dX[a+pad:b]) for a,b in R[np.arange(0,len(R),2)]]
else:
dX = [(util.ord2acgt(X[a+pad:b]), dX[a+pad:b]) for a,b in R]
else:
dX = [(util.ord2acgt(X[a+pad:b]), dX[a+pad:b]) for a,b in R]
if Zmask is not None:
dX = [dX[i] for i in range(len(dX)) if Zmask[i]]
else:
if Zmask is not None:
X = X[Zmask.ravel()]
dX = dX[Zmask.ravel()]
dX *= np.maximum(0,Z)
dX = [(X[i], dX[i]) for i in range(len(dX))]
if dkey not in gmaps:
gmaps[dkey] = []
gmaps[dkey] += dX
# Concatenate all numpy arrays if they're the same size
predictions = np.vstack(predictions)
return (predictions, gmaps) if want_gmaps else (predictions, None)
def convert_to_sarray(self):
# Upload the data to a GPU device
for name in self._data_attrs:
oldattr = getattr(self, name)
setattr(self, name, sm.asarray(oldattr))
def gen_convnet_predictions(model, data, want_gmaps=False):
# We must feed each sequence through the model several times
# by applying the model repeatedly on sliding a window along the sequence.
# That generates a prediction map, from which we can take max, sum, etc.
predictions = []
gmaps = {}
batches = data.asbatches(batchsize=2048, reshuffle=False)
for batch in batches:
args = batch.input_data()
args["want_bprop_inputs"] = bool(want_gmaps)
if isinstance(model.Z.origin().node, deepity.std.softmaxnode):
args["bprop_inputs_loss"] = deepity.std.nll()
else:
args["bprop_inputs_loss"] = deepity.std.mse()
globals.flags.push("collect_argmax", None)
outputs = model.eval(**args)
I = globals.flags.pop("collect_argmax")
Z = outputs['Z'].asnumpy()
Z, Zmask = maskout_revcomp(Z)
if Zmask is not None:
if "collect_Zmask" in globals.flags:
global_Zmask = globals.flags.pop("collect_Zmask")
if not isinstance(global_Zmask, np.ndarray):
global_Zmask = Zmask
else:
global_Zmask = np.vstack([global_Zmask, Zmask])
globals.flags.push("collect_Zmask", global_Zmask)
predictions.append(Z)
# If user wants gradientmaps, then for every sequence we need one
if want_gmaps:
for key in args:
dkey = "d" + key
if outputs.get(dkey, None) is not None:
X = args[key].asnumpy()
dX = outputs[dkey].asnumpy()
if X.dtype == np.uint8: # Is it an sequence of ordinals (bytes)?
pad = data.requirements.get("padding", 0)
R = args[
"R" +
key[1:]].asnumpy() # regions associated with X
if want_gmaps == "finite_diff":
is_rc = "reverse_complement" in globals.flags
#globals.flags.push("force_argmax",I)
rcindex = [3, 2, 1, 0]
oldF = args['F']
# If user specifically asked for finite differences, not instantaneous gradient,
# then we need to explicitly mutate every position, generate predictions, and
# subtract the result from Z to find the actual delta for each base
Xlen = R[:, 1] - R[:, 0]
nbase = dX.shape[1]
for i in range(Xlen.max()):
for j in range(nbase):
mtX = X.copy()
mtF = args['F'].asnumpy().copy()
for k in range(len(R)):
a, b = R[k]
if i < b - a:
if (k % 2 == 0) or not is_rc:
mtX[pad + a +
i] = j # mutate position i in sequence k (which starts at byte index a) to base j
else:
mtX[b - i - 1] = rcindex[j]
mtF[k] = data._generate_dinuc_featurevec(
mtX[pad + a:b])
args[key] = sm.asarray(
mtX
) # This time use the mutated X instead of the original
args['F'] = sm.asarray(mtF)
mtoutputs = model.eval(**args)
mtZ = mtoutputs['Z'].asnumpy()
mtZ, mtZmask = maskout_revcomp(mtZ)
dZ = mtZ - Z # output
dZ *= np.maximum(
0, np.sign(np.maximum(Z, mtZ)))
for k in range(len(R)):
if (k % 2 == 0) or not is_rc:
a, b = R[k]
if i < b - a:
dX[pad + a + i,
j] = dZ[(k //
2) if is_rc else k]
#globals.flags.pop("force_argmax")
args['F'] = oldF
# Only include forward strand in finite_diff results
if is_rc:
dX = [(util.ord2acgt(X[a + pad:b]),
dX[a + pad:b])
for a, b in R[np.arange(0, len(R), 2)]]
else:
dX = [(util.ord2acgt(X[a + pad:b]),
dX[a + pad:b]) for a, b in R]
else:
dX = [(util.ord2acgt(X[a + pad:b]), dX[a + pad:b])
for a, b in R]
if Zmask is not None:
dX = [
dX[i] for i in range(len(dX)) if Zmask[i]
]
else:
if Zmask is not None:
X = X[Zmask.ravel()]
dX = dX[Zmask.ravel()]
dX *= np.maximum(0, Z)
dX = [(X[i], dX[i]) for i in range(len(dX))]
if dkey not in gmaps:
gmaps[dkey] = []
gmaps[dkey] += dX
# Concatenate all numpy arrays if they're the same size
predictions = np.vstack(predictions)
return (predictions, gmaps) if want_gmaps else (predictions, None)
def convert_to_sarray(self):
# Upload the data to a GPU device
for name in self._data_attrs:
oldattr = getattr(self,name)
setattr(self,name,sm.asarray(oldattr))
def _train_setup(self, trainable_plugs, cost):
# For each trainable plug, figure out how many weights it needs.
sizes = [ np.prod(p.shape)*cost.ninst for p in trainable_plugs ]
offsets = np.asarray(np.cumsum([0] + [ size for size in sizes ]),np.uint32)
# Allocate giant contiguous arrays for P, dP, and mP
P = sm.zeros((offsets[-1],1))
dP = sm.zeros_like(P)
mP = sm.zeros_like(P)
# Per-inst learn rates / momentum rates go here.
# Giant contiguous array maps to same indicies as in P, dP, mP
drate = sm.zeros_like(P)
mrate = sm.zeros_like(P)
trnodes = []
# For each plug, create a trainable node that is bound to
# a chunk of our P (parameter) and dP (gradient) vectors, where the node can
for i,tplug in enumerate(trainable_plugs):
# Grow the actual shape of the trainable parameters, using the
# axis specified by the trainable plug.
shape = list(tplug.shape)
shape[tplug.inst_axis] *= tplug.node.ninst
# Allocate a new trainable node, and connect it to the plug
trnode = trainable(P[offsets[i]:offsets[i+1]].reshape(tuple(shape)),
dP[offsets[i]:offsets[i+1]].reshape(tuple(shape)))
trnode >> tplug
trnodes.append(trnode)
# Assign instance-specific learning rates and momentum rates
# to each corresponding element in the giant drate/mrate vectors
if tplug.inst_axis == 0:
k = np.prod(tplug.shape)
else:
k = tplug.shape[1]
dratevec = drate[offsets[i]:offsets[i+1]]
mratevec = mrate[offsets[i]:offsets[i+1]]
_ext.madd_bcast(sm.ones_like(dratevec),self.rate,k,dratevec)
_ext.madd_bcast(sm.ones_like(mratevec),self.momentum,k,mratevec)
# Also initialize elements of P based on the trainable plug's initialization scale,
# which can be different for each individual instance
Pvec = P[offsets[i]:offsets[i+1]]
initval = tplug.origin().node.init
if isinstance(initval, np.ndarray) and initval.ndim == 3:
# Specific initialization of individual filters
Pvec[:] = sm.asarray(np.require(np.rollaxis(initval,1),requirements="C").reshape((-1,1)))
else:
# Random initialization
_ext.madd_bcast(sm.randn(Pvec.shape[0],Pvec.shape[1]),
initval,k,Pvec)
if hasattr(tplug.origin().node,'init_mu'):
initmu_val = tplug.origin().node.init_mu
if isinstance(initmu_val, list):
# Specific initialization of individual bias elements
initmu_val = np.tile(initmu_val,tplug.origin().node.ninst)
Pvec[:] = sm.asarray(initmu_val).reshape(Pvec.shape)
else:
_ext.madd_bcast(sm.ones_like(Pvec),
tplug.origin().node.init_mu,k,Pvec) # Add shift
return (P,dP,mP,drate,mrate,trnodes)
