concepts = res_spade['patterns']
##### PSF filtering #####
concepts_psf = list(
filter(
lambda c: spade._pattern_spectrum_filter(
c, ns_sgnt, spectrum, winlen), concepts))
##### PSR filtering ######
# Decide whether filter the concepts using psr
if psr_param is not None:
# Filter using conditional tests (psr)
concepts_psr = spade.pattern_set_reduction(
concepts_psf,
ns_sgnt,
winlen=winlen,
h=psr_param[0],
k=psr_param[1],
l=psr_param[2],
min_spikes=min_spikes,
min_occ=min_occ)
else:
concepts_psr = concepts_psf
patterns = spade.concept_output_to_patterns(
concepts=concepts_psr,
pvalue_spectrum=pval_spectrum,
winlen=winlen,
binsize=binsize)
# Storing filtered results
params['alpha'] = alpha
params['psr_param'] = psr_param
params['correction'] = correction
def test_pattern_set_reduction(self):
winlen = 6
# intent(concept1) is a superset of intent(concept2)
# extent(concept1) is a subset of extent(concept2)
# intent(concept2) is a subset of intent(concept3)
# when taking into account the shift due to the window positions
# intent(concept1) has a non-empty intersection with intent(concept3)
# when taking into account the shift due to the window positions
# intent(concept4) is disjoint from all others
concept1 = ((12, 19, 26), (2, 10, 18))
concept2 = ((12, 19), (2, 10, 18, 26))
concept3 = ((0, 7, 14, 21), (0, 8))
concept4 = ((1, 6), (0, 8))
# reject concept2 using min_occ
# make sure to keep concept1 by setting k_superset_filtering = 1
concepts = spade.pattern_set_reduction([concept1, concept2],
ns_signatures=[],
winlen=winlen,
spectrum='#',
h_subset_filtering=0,
min_occ=2,
k_superset_filtering=1)
self.assertEqual(concepts, [concept1])
# keep concept2 by increasing h_subset_filtering
concepts = spade.pattern_set_reduction([concept1, concept2],
ns_signatures=[],
winlen=winlen,
spectrum='#',
h_subset_filtering=2,
min_occ=2,
k_superset_filtering=1)
self.assertEqual(concepts, [concept1, concept2])
# reject concept1 using min_spikes
concepts = spade.pattern_set_reduction([concept1, concept2],
ns_signatures=[],
winlen=winlen,
spectrum='#',
h_subset_filtering=2,
min_spikes=2,
k_superset_filtering=0)
self.assertEqual(concepts, [concept2])
# reject concept2 using ns_signatures
concepts = spade.pattern_set_reduction([concept1, concept2],
ns_signatures=[(2, 2)],
winlen=winlen,
spectrum='#',
h_subset_filtering=1,
min_occ=2,
k_superset_filtering=1)
self.assertEqual(concepts, [concept1])
# reject concept1 using ns_signatures
# make sure to keep concept2 by increasing h_subset_filtering
concepts = spade.pattern_set_reduction([concept1, concept2],
ns_signatures=[(2, 3)],
winlen=winlen,
spectrum='#',
h_subset_filtering=3,
min_spikes=2,
min_occ=2,
k_superset_filtering=1)
self.assertEqual(concepts, [concept2])
# reject concept2 using the covered spikes criterion
concepts = spade.pattern_set_reduction([concept1, concept2],
ns_signatures=[(2, 2)],
winlen=winlen,
spectrum='#',
h_subset_filtering=0,
min_occ=2,
k_superset_filtering=0,
l_covered_spikes=0)
self.assertEqual(concepts, [concept1])
# reject concept1 using superset filtering
# (case with non-empty intersection but no superset)
concepts = spade.pattern_set_reduction([concept1, concept3],
ns_signatures=[],
min_spikes=2,
winlen=winlen,
spectrum='#',
k_superset_filtering=0)
self.assertEqual(concepts, [concept3])
# keep concept1 by increasing k_superset_filtering
concepts = spade.pattern_set_reduction([concept1, concept3],
ns_signatures=[],
min_spikes=2,
winlen=winlen,
spectrum='#',
k_superset_filtering=1)
self.assertEqual(concepts, [concept1, concept3])
# reject concept3 using ns_signatures
concepts = spade.pattern_set_reduction([concept1, concept3],
ns_signatures=[(3, 2)],
min_spikes=2,
winlen=winlen,
spectrum='#',
k_superset_filtering=1)
self.assertEqual(concepts, [concept1])
# reject concept3 using the covered spikes criterion
concepts = spade.pattern_set_reduction([concept1, concept3],
ns_signatures=[(3, 2), (2, 3)],
min_spikes=2,
winlen=winlen,
spectrum='#',
k_superset_filtering=1,
l_covered_spikes=0)
self.assertEqual(concepts, [concept1])
# check that two concepts with disjoint intents are both kept
concepts = spade.pattern_set_reduction([concept3, concept4],
ns_signatures=[],
winlen=winlen,
spectrum='#')
self.assertEqual(concepts, [concept3, concept4])
spectrum=spectrum)
concepts_psf = list(filter(
lambda c: spade._pattern_spectrum_filter(
c, ns_sgnt, spectrum, winlen), concepts))
print('Winlen:', winlen)
print('Non significant signatures:', sorted(ns_sgnt))
print('Number of significant patterns before psr:', len(concepts_psf))
# PSR filtering
# Decide whether filter the concepts using psr
if psr_param is not None:
# Filter using conditional tests (psr)
if 0 < alpha < 1 and n_surr > 0:
concepts_psr = spade.pattern_set_reduction(concepts_psf, ns_sgnt,
winlen=winlen,
spectrum=spectrum,
h_subset_filtering=psr_param[0],
k_superset_filtering=psr_param[1],
l_covered_spikes=psr_param[2],
min_spikes=min_spikes,
min_occ=min_occ)
else:
concepts_psr = spade.pattern_set_reduction(concepts_psf, [],
winlen=winlen,
spectrum=spectrum,
h_subset_filtering=psr_param[0],
k_superset_filtering=psr_param[1],
l_covered_spikes=psr_param[2],
min_spikes=min_spikes,
min_occ=min_occ)
patterns = spade.concept_output_to_patterns(
concepts_psr, winlen, binsize, pval_spectrum) # notice 't_start' parameter is missing here as of 2020.10.29
#TODO correction should be applied before further processing of data.