def maxthreshold(self, track: Track, key: int,
data: np.ndarray) -> Optional[float]:
"return the min threshold"
if self.highfactor is None:
return None
hfs = track.rawprecision(key)
out = track.phaseposition(self.high, data) + hfs * self.highfactor
return out if np.isfinite(out) else None
def _m3u_to_playlist(m3u_path: Path):
tracks = []
with open(str(m3u_path.resolve()), "rt") as fout:
for track in fout:
tracks.append(Track(Path(track.strip().lower()), dict(), None))
return Playlist(m3u_path.name[:-4], tracks)
def bypeakevents(self, nbeads, seed = None):
"Creates events grouped by peaks"
self.seed(seed)
track = Track(data = {i: None for i in range(nbeads)},
phases = self.phases,
key = 'bypeakevents')
def _create(cycles):
events = tuple(self.__events(cycles))
labels = [np.array([i[0] for i in evt['data']]) for evt in events]
curs = []
for lab in np.unique(np.concatenate(labels)):
cur = np.empty((len(events),), dtype = EVENTS_DTYPE)
cur['start'] = 0
cur['data'] = None
for i, (cevt, clab) in enumerate(zip(events, labels)):
val = cevt[clab == lab]
if len(val):
cur[i] = (val[0]['start'], val[0]['data'])
curs.append(cur)
return cycles.ravel(), curs
def _generator(curs):
for cur in curs:
peak = np.mean([i[1].mean() for i in cur if i[1] is not None])
yield (peak, cur)
def _action(_, bead):
track.data[bead[0]], curs = self.__apply(_create)
return bead[0], _generator(curs)
return (TrackView(track = track, data = dict(track.data), level = Level.peak)
.withaction(_action))
def test_minbiasalignment():
"test min bias alignment of peaks"
data = Experiment(baseline=None, thermaldrift=None).track(seed=1)
track = Track(**data)
lst = (InMemoryTrackTask(track), EventDetectionTask(),
PeakSelectorTask(peakalign=None), MinBiasPeakAlignmentTask())
peaks = next(create(*lst).run())
_ = peaks[0] # test everything runs
cycles = np.array([(1. if i > 5 else 0., 0.) for i in range(10)],
dtype=MinBiasPeakAlignmentTask.DTYPE)
stats = np.array([np.roll(cycles, i) for i in range(4)],
dtype=MinBiasPeakAlignmentTask.DTYPE)
for i in range(4):
stats[i, :]['mean'][:] += i * 10
truth = np.arange(10, dtype='f4') * .1
truth -= np.median(truth)
for i in range(10):
stats[:, i]['mean'][:] -= truth[i]
found = lst[-1](stats)
truth = np.array([
-0.44999883, -0.34998798, -0.24997711, 0., 0., 0., 0., 0.24997902,
0.34999132, 0.45000142
],
dtype='f4')
assert_allclose(found, truth)
def track(self, nbeads = 1, seed = None):
"creates a simulated track"
self.seed(seed)
track = Track(data = {},
phases = self.phases,
framerate = self.framerate,
key = 'tracksimulator')
setattr(track,"_lazyfy_",False)
sim = {}
for i in range(nbeads):
track.data[i] = self()
if len(getattr(self.events, 'stored', tuple())):
sim[i] = dict(self.events.stored)
if len(sim):
setattr(track, 'simulator', sim)
return track
def test_clippingtask():
"tests clipping task"
track = Track(
**(Experiment(baseline=None, thermaldrift=None).track(1, 5, seed=0)))
arr = np.copy(track.data[0])
assert np.all(np.isfinite(arr))
ClippingTask()(track, 0, arr)
assert_equal(arr, track.data[0])
extr = np.min(arr), np.max(arr)
cyc = track.cycles.withphases(5)
cyc[0, 0][10] = extr[0] - .1
cyc[0, 0][12] = extr[1] + .001
cyc[0, 1][10] = extr[0] - .1
cyc[0, 1][12] = extr[1] + .001
cyc[0, 2][12] = np.NaN
arr = np.copy(track.data[0])
ClippingTask()(track, 0, arr)
cyc = cyc.withdata({0: arr})
assert_equal(np.nonzero(np.isnan(cyc[0, 0]))[0], [10, 12])
assert_equal(np.nonzero(np.isnan(cyc[0, 1]))[0], [10, 12])
nzer = np.nonzero(arr - track.data[0])[0]
assert len(nzer) == 5
assert np.all(np.isnan(arr[nzer]))
def test_gels():
"test min bias alignment of peaks"
data = Experiment(baseline=None, thermaldrift=None).track(seed=1)
track = Track(**data)
lst = (InMemoryTrackTask(track), EventDetectionTask(),
PeakSelectorTask(peakalign=None), GELSPeakAlignmentTask())
peaks = next(create(*lst).run())
_ = peaks[0] # test everything runs
cycles = np.array([(1. if i > 5 else 0., 0.) for i in range(10)],
dtype=GELSPeakAlignmentTask.DTYPE)
stats = np.array([np.roll(cycles, i) for i in range(4)],
dtype=GELSPeakAlignmentTask.DTYPE)
for i in range(4):
stats[i, :]['mean'][:] += i * 10
truth = np.arange(10, dtype='f4') * .1
truth -= np.median(truth)
for i in range(10):
stats[:, i]['mean'][:] -= truth[i]
found = lst[-1](stats)
truth = np.array([
-0.47142908, -0.37142903, -0.27142864, 0., 0., 0., 0.12857169,
0.22857153, 0.32857174, 0.4285718
],
dtype='f4')
assert_allclose(found, truth)
def test_baselinepeak():
"test single strand peak"
data = Experiment(baseline=None, thermaldrift=None).track(seed=1)
track = Track(**data)
lst = (InMemoryTrackTask(track), EventDetectionTask(), PeakSelectorTask(),
BaselinePeakTask())
out1 = [i for i, _ in next(create(*lst).run())[0]]
out2 = [i for i, _ in next(create(*lst[:-1]).run())[0]]
assert out1 == out2[1:]
def get_tracks(
collection_nml: str,
volume: str,
auto_generated_playlists_folder: str,
playlist_manager: AutoGeneratedPlaylistManager) -> Dict[str, Track]:
result = dict()
collection = TraktorCollection(Path(collection_nml))
auto_generated_playlists = list_playlists_in_collection(collection, auto_generated_playlists_folder)
tagged_tracks_not_flattened = playlist_manager.tagged_tracks_from_playlists(auto_generated_playlists)
for t in collection.nml.collection.entry:
if t.location.volume != volume:
continue
path = str(traktor_path_to_pathlib_path(t.location.dir, t.location.file)).lower()
tags = dict()
comment = None
if t.info and t.info.comment:
comment = t.info.comment
if path in tagged_tracks_not_flattened:
for tagged_track in tagged_tracks_not_flattened[path]:
for tag_key, tag_value in tagged_track.tags.items():
if tag_key not in tags:
tags[tag_key] = tag_value
elif tags[tag_key] != tag_value:
# Conflict between playlists, this means the user has placed the track in multiple playlists
# To see which tags are the previous/new values, let's check the comment
if tags[tag_key] in comment and \
(tag_value not in comment or len(tag_value) < len(tags[tag_key])):
tags[tag_key] = tag_value
track = Track(path=Path(path), tags=tags, rating=None, comment=comment)
if t.info.ranking is not None and int(t.info.ranking) >= 51:
track.rating = t.info.ranking / 51
if t.album is not None and t.album.title:
track.album = t.album.title
result[path] = track
return result
def get_tracks(db_file, tag_list: List[str]) -> Dict[str, Track]:
result = dict()
library = Library(db_file)
tag_list = [
tag for tag in tag_list if tag != "rating" and not tag.startswith("_")
]
for item in library.items():
path = convert_attr_to_string(item.path).lower()
tags = {tag: _get_attr_dont_throw(item, tag) for tag in tag_list}
rating = _get_int_attr_dont_throw(item, "rating")
if not (rating is not None and 0 <= rating <= 5):
rating = None
result[path] = Track(Path(path), tags, rating)
return result
def test_singlestrandpeak():
"test single strand peak"
data = Experiment(baseline=None, thermaldrift=None).track(seed=1)
track = Track(**data)
lst = (InMemoryTrackTask(track), EventDetectionTask(), PeakSelectorTask(),
SingleStrandTask())
peaks = next(create(*lst[:-1]).run())
proc = SingleStrandProcessor()
ncl = proc.nonclosingramps(peaks, 0)
truth = np.where(data['truth'][0].strandclosing.duration >=
track.phase.duration(..., range(5)))[0]
assert set(ncl) == set(truth)
out1 = [i for i, _ in next(create(*lst).run())[0]]
out2 = [i for i, _ in next(create(*lst[:-1]).run())[0]]
assert out1 == out2[:-1]
def list_playlists_in_node(node: TraktorModels.Nodetype) -> List[Playlist]:
result = []
if node.playlist and node.playlist.entry:
tracks = []
for entry in node.playlist.entry:
if not entry.primarykey or entry.primarykey.type != "TRACK":
continue
track_path = Path(str(traktor_absolute_path_to_pathlib_path(entry.primarykey.key)).lower())
tracks.append(Track(track_path, dict(), None))
result.append(Playlist(node.name, tracks))
if node.subnodes and node.subnodes.node:
for n in node.subnodes.node:
result.extend(list_playlists_in_node(n))
return result
def test_subtract_med():
"tests subtractions"
_ = dict(dtype='f4')
agg = lambda x: SubtractMedianSignal.apply([i.astype('f4') for i in x],
(0, 5)) # noqa
assert_allclose(agg([np.arange(5, **_)] * 5), np.arange(5, **_))
assert_allclose(agg([np.arange(5, **_), np.ones(5, **_)]),
np.arange(5, **_) * .5 + .5)
assert_allclose(agg([np.arange(5, **_)] + [np.ones(5, **_)] * 2),
np.ones(5, **_))
assert_allclose(agg([np.arange(6, **_), np.ones(5, **_)]),
list(np.arange(5, **_) * .5 + .5) + [4.5])
tsk = BeadSubtractionTask(beads=[1, 2, 3, 4])
tsk.agg.average = True
cache = {} # type: ignore
frame = BeadSubtractionProcessor.apply(
Track(**Experiment().track(1, 5)).beads, cache=cache, **tsk.config())
out = frame[0]
assert out is not None
def bybeadevents(self, nbeads, seed = None) -> Cycles:
"Creates events in a Events object"
self.seed(seed)
track = Track(data = None,
phases = self.phases,
key = 'bybeadeventssimulator')
def _createall(_):
evts: Dict[Tuple[int,int], np.ndarray] = OrderedDict()
def _createone(cycs, bead):
evts.update(((bead, cid), evt) for cid, evt in enumerate(self.__events(cycs)))
return (bead, cycs.ravel())
track.data = dict(self.__apply(_createone, i) for i in range(nbeads))
return evts
return Cycles(track = track,
data = _createall,
direct = True,
level = Level.event)
class _Frame:
track = Track(path="mypath",
data=dat,
framerate=1. / 30.,
phases=(np.cumsum(cyc) - 10).reshape((3, 9)))
new = lambda self, _: self
withdata = lambda self, fcn: fcn(self)
@staticmethod
def values():
"-"
pks = lambda x: np.array(x, dtype=PEAKS_DTYPE)
evts = lambda x: np.array(x, dtype=EVENTS_DTYPE)
empty = np.empty(0)
tmp2 = evts([(5, np.zeros(5)), (5, np.zeros(5))])
evts1 = [(0., np.array([empty, empty, empty])),
(.01, np.array([empty,
evts([(0, np.zeros(5))]), tmp2])),
(.1, np.array([empty, empty,
evts([(0, np.zeros(5))])])),
(.5, np.array([empty, empty, tmp2])),
(1., np.array([empty, empty,
evts([(0, np.zeros(5))])]))]
return [
ByHairpinGroup('hp100', [
ByHairpinBead(
100, .95, Distance(.1, 1000., 0.0),
pks([(-1., 0.), (.01, np.iinfo('i4').min), (.1, 100.),
(.5, 500.), (1., 1000.)]), evts1)
]),
ByHairpinGroup(None, [
ByHairpinBead(
101, -3, Distance(.1, 1000., 0.0), pks([(0., 0.)]),
[(0., np.array(
[empty, empty,
evts([(0, np.zeros(5))])]))]),
ByHairpinBead(102, -3, Distance(.1, 1000., 0.0), pks([]),
[])
])
]
def test_excel():
"tests reporting"
for path in Path(gettempdir()).glob("*_hybridstattest*.*"):
path.unlink()
truth = [
np.array([0., .1, .2, .5, 1., 1.5], dtype='f4') / 1e-3,
np.array([0., .1, .5, 1.2, 1.5], dtype='f4') / 1e-3
]
sequences = {
'hp100': np.array(['c'] * 1500),
'hp101': np.array(['c'] * 1500)
}
for i in (100, 500, 1000):
sequences['hp100'][i - 3:i + 1] = list('atgc')
sequences['hp100'] = ''.join(sequences['hp100'])
for i in (100, 500, 1200, 1000):
sequences['hp101'][i - 3:i + 1] = list('atgc')
sequences['hp101'] = ''.join(sequences['hp101'])
pks = lambda x: np.array(x, dtype=PEAKS_DTYPE)
evts = lambda x: np.array(x, dtype=EVENTS_DTYPE)
empty = evts([])
tmp2 = evts([(5, np.zeros(5)), (5, np.zeros(5))])
evts1 = [(0., np.array([empty, empty, empty])),
(.01, np.array([empty, evts([(0, np.zeros(5))]), tmp2])),
(.1, np.array([empty, empty,
evts([(0, np.zeros(5))])])),
(.5, np.array([empty, empty, tmp2])),
(1., np.array([empty, empty,
evts([(0, np.zeros(5))])]))]
groups = [
ByHairpinGroup('hp100', [
ByHairpinBead(
100, .95, Distance(.1, 1000., 0.0),
pks([(0., 0.), (.01, np.iinfo('i4').min), (.1, 100.),
(.5, 500.), (1., 1000.)]), evts1)
]),
ByHairpinGroup(None, [
ByHairpinBead(101, -3, Distance(.1, 1000., 0.0), pks([
(0., 0.)
]), [(0., np.array([empty, empty,
evts([(0, np.zeros(5))])]))]),
ByHairpinBead(102, -3, Distance(.1, 1000., 0.0), pks([]), [])
])
]
dat = {
100: np.arange(10) * .1,
101: np.arange(10) * .2,
102: np.arange(10) * .3,
104: np.arange(10) * .2
}
cyc = np.array([0, 10, 5, 10, 5, 100, 5, 10, 5] * 3)
track = Track(path="mypath",
data=dat,
framerate=1. / 30.,
phases=(np.cumsum(cyc) - 10).reshape((3, 9)))
fcn = lambda x: run(path=x,
track=track,
config="myconfig",
hairpins={
'hp100': HairpinFitter(peaks=truth[0]),
'hp101': HairpinFitter(peaks=truth[1])
},
sequences=sequences,
oligos=['atgc'],
knownbeads=(98, ),
minduration=2,
groups=groups)
fname = mktemp() + "_hybridstattest.xlsx"
assert not Path(fname).exists()
fcn(fname)
assert Path(fname).exists()
fname = mktemp() + "_hybridstattest.csv"
assert not Path(fname).exists()
fcn(fname)
assert Path(fname).exists()
fname = mktemp() + "_hybridstattest.pkz"
assert not Path(fname).exists()
fcn(fname)
assert Path(fname).exists()
def track(self):
"create a track"
exp = self.experiment.experiment()
trk = Track()
trk.__setstate__(exp.track(self.nbeads))
return exp, trk