def tfrecord_pipeline(dspath, batch_size, num_threads, device="cpu", device_id=None,
shard_id=0, num_shards=1, reader_name="Reader",
seq=True, chroms=False, chroms_vlog=False, target=True, target_vlog=True, label=False, random_shuffle=True):
pipe = Pipeline(batch_size=batch_size, num_threads=num_threads, device_id=device_id)
feature_description = {}
feature_description["seq"] = tfrec.VarLenFeature(tfrec.float32, -1.0)
feature_description["label"] = tfrec.FixedLenFeature([], tfrec.int64, -1)
feature_description["target"] = tfrec.FixedLenFeature([], tfrec.float32, -1.0)
for ct in dspath["chromatin_tracks"]:
feature_description[ct] = tfrec.VarLenFeature(tfrec.float32, -1.0)
with pipe:
inputs = fn.readers.tfrecord(
name=reader_name,
path=dspath['TFRecord'],
index_path=dspath['TFRecord_idx'],
features=feature_description,
shard_id = shard_id,
num_shards = num_shards,
random_shuffle=random_shuffle,
read_ahead=True,
prefetch_queue_depth=20,
pad_last_batch=True)
if device=="gpu":
inputs['seq'] = inputs['seq'].gpu()
for ct in dspath["chromatin_tracks"]: inputs[ct] = inputs[ct].gpu()
inputs['target'] = inputs['target'].gpu()
inputs['label'] = inputs['label'].gpu()
seqdata = fn.expand_dims(inputs['seq'], axes=1, device=device)
seqdata = fn.reshape(seqdata, shape=(4, -1), device=device)
chromsdata = fn.cat(*[fn.expand_dims(inputs[ct], axes=0, device=device) for ct in dspath["chromatin_tracks"]], axis=0, device=device)
sample = []
if seq: sample.append(seqdata)
if chroms:
if chroms_vlog:
sample.append(log(chromsdata + 1))
else:
sample.append(chromsdata)
if target:
if target_vlog:
sample.append(log(inputs['target'] + 1))
else:
sample.append(inputs['target'])
if label: sample.append(inputs['label'])
pipe.set_outputs(*sample)
return pipe
def pipeline_arithm_ops_cpu(source):
data = fn.external_source(source=source, layout="HWC")
processed = (data * 2,
data + 2,
data - 2,
data / 2,
data // 2,
data ** 2,
data == 2,
data != 2,
data < 2,
data <= 2,
data > 2,
data >= 2,
data & 2,
data | 2,
data ^ 2,
dmath.abs(data),
dmath.fabs(data),
dmath.floor(data),
dmath.ceil(data),
dmath.pow(data, 2),
dmath.fpow(data, 1.5),
dmath.min(data, 2),
dmath.max(data, 50),
dmath.clamp(data, 10, 50),
dmath.sqrt(data),
dmath.rsqrt(data),
dmath.cbrt(data),
dmath.exp(data),
dmath.exp(data),
dmath.log(data),
dmath.log2(data),
dmath.log10(data),
dmath.sin(data),
dmath.cos(data),
dmath.tan(data),
dmath.asin(data),
dmath.acos(data),
dmath.atan(data),
dmath.atan2(data, 3),
dmath.sinh(data),
dmath.cosh(data),
dmath.tanh(data),
dmath.asinh(data),
dmath.acosh(data),
dmath.atanh(data))
return processed
def test_arithm_ops_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
data = fn.external_source(source=get_data, layout="HWC")
processed = [
data * 2, data + 2, data - 2, data / 2, data // 2, data**2, data == 2,
data != 2, data < 2, data <= 2, data > 2, data >= 2, data & 2,
data | 2, data ^ 2,
dmath.abs(data),
dmath.fabs(data),
dmath.floor(data),
dmath.ceil(data),
dmath.pow(data, 2),
dmath.fpow(data, 1.5),
dmath.min(data, 2),
dmath.max(data, 50),
dmath.clamp(data, 10, 50),
dmath.sqrt(data),
dmath.rsqrt(data),
dmath.cbrt(data),
dmath.exp(data),
dmath.exp(data),
dmath.log(data),
dmath.log2(data),
dmath.log10(data),
dmath.sin(data),
dmath.cos(data),
dmath.tan(data),
dmath.asin(data),
dmath.acos(data),
dmath.atan(data),
dmath.atan2(data, 3),
dmath.sinh(data),
dmath.cosh(data),
dmath.tanh(data),
dmath.asinh(data),
dmath.acosh(data),
dmath.atanh(data)
]
pipe.set_outputs(*processed)
pipe.build()
for _ in range(3):
pipe.run()
# Limit the range so we do not end with comparing just the infinities in results.
def limited_range(*types):
return [(-30, 30) for _ in types]
def pow_range(*_):
return [(-15, 15), (-4, 4)]
def default_range(*types):
return [None for _ in types]
math_function_operations = [
((lambda x: math.sqrt(x)), (lambda x: np.sqrt(x)), "sqrt", pos_range, 1e-6),
((lambda x: math.rsqrt(x)), (lambda x: 1.0 / np.sqrt(x)), "rsqrt", pos_range, 1e-5),
((lambda x: math.cbrt(x)), (lambda x: np.cbrt(x)), "cbrt", default_range, 1e-6),
((lambda x: math.exp(x)), (lambda x: np.exp(x)), "exp", limited_range, 1e-6),
((lambda x: math.log(x)), (lambda x: np.log(x)), "log", pos_range, 1e-6),
((lambda x: math.log2(x)), (lambda x: np.log2(x)), "log2", pos_range, 1e-6),
((lambda x: math.log10(x)), (lambda x: np.log10(x)), "log10", pos_range, 1e-6),
((lambda x: math.fabs(x)), (lambda x: np.fabs(x)), "fabs", default_range, 1e-6),
((lambda x: math.floor(x)), (lambda x: np.floor(x)), "floor", default_range, 1e-6),
((lambda x: math.ceil(x)), (lambda x: np.ceil(x)), "ceil", default_range, 1e-6),
((lambda x: math.sin(x)), (lambda x: np.sin(x)), "sin", default_range, 1e-6),
((lambda x: math.cos(x)), (lambda x: np.cos(x)), "cos", default_range, 1e-6),
((lambda x: math.tan(x)), (lambda x: np.tan(x)), "tan", default_range, 1e-6),
((lambda x: math.asin(x)), (lambda x: np.arcsin(x)), "asin", one_range, 1e-6),
((lambda x: math.acos(x)), (lambda x: np.arccos(x)), "acos", one_range, 1e-6),
((lambda x: math.atan(x)), (lambda x: np.arctan(x)), "atan", default_range, 1e-6),
((lambda x: math.sinh(x)), (lambda x: np.sinh(x)), "sinh", default_range, 1e-6),
((lambda x: math.cosh(x)), (lambda x: np.cosh(x)), "cosh", default_range, 1e-6),
((lambda x: math.tanh(x)), (lambda x: np.tanh(x)), "tanh", default_range, 1e-6),
((lambda x: math.asinh(x)), (lambda x: np.arcsinh(x)), "asinh", limited_range, 1e-6),