def setUpClass(cls):
super(AoudaTests, cls).setUpClass()
dirname = os.path.dirname(__file__)
cls.filename = os.path.join(dirname, 'small_dataset.dat')
cls.resolution = 1000
cls.base_datetime = datetime.now() - timedelta(seconds=15)
cls.data = extract_hr_acc(read_data(cls.filename, cls.base_datetime))
def setUpClass(cls):
super(AoudaTests, cls).setUpClass()
dirname = os.path.dirname(__file__)
cls.filename = os.path.join(dirname, 'small_dataset.dat')
cls.resolution = 1000
cls.base_datetime = datetime.now() - timedelta(seconds=15)
cls.data = extract_hr_acc(read_data(cls.filename, cls.base_datetime))
def test_read_data_unlabeled(self):
""" Tests the function read_data for unlabeled data. """
references = pre.read_data(self.testfilename)
truth = [
[Reference(0, 'm jones',
'symbol intersect detect method improv spatial intersect join',
['e rundensteiner', 'y huang'], 'geoinformatica', None),
Reference(1, 'matthew c jones',
'improv spatial intersect join symbol intersect detect',
['e rundensteiner', 'h kuno', 'p marron', 'v taube', 'y ra'],
'sigmodels.intern manag data', None),
Reference(2, 'matthew c jones',
'view materi techniqu complex hirarch object', ['e rundensteiner',
'y huang'], 'ssd symposium larg spatial databas', None)],
[Reference(3, 'mike w miller', 'domin draw bipartit graph',
['l berg'], 'sigucc special interest group univers comput servic',
None),
Reference(4, 'mike w miller', 'rel compromis statist databas',
[], 'sigucc special interest group univers comput servic', None)],
[Reference(5, 'c chen', 'formal approach scenario analysi',
['d kung', 'j samuel', 'j gao', 'p hsia', 'y toyoshima'],
'ieee softwar', None)],
[Reference(6, 'jane j robinson', 'discours code clue context', [],
'acl meet the associ comput linguist', None),
Reference(7, 'jane j robinson', 'diagram grammar dialogu', [],
'cooper interfac inform system', None)],
[Reference(8, 'a gupta', 'iri h java distanc educ', ['a gonzalez',
'a hamid', 'c overstreet', 'h wahab', 'j wild', 'k maly', 's ghanem',
'x zhu'], 'acm journal educ resourc comput', None)],
[Reference(9, 'mary d brown',
'intern redund represent limit bypass support pipelin adder regist'
'file', ['y patt'], 'proceed the th ieee intern symposium high '
'perform comput architectur hpca intern symposium high perform '
'comput architectur talk slide', None)]]
self.assertEquals(references, truth)
def setUpClass(cls):
"""
Reads a test data set into memory
"""
dirname = os.path.dirname(__file__)
filename = os.path.join(dirname, 'dataset.dat')
cls._df = extract_hr_acc(read_data(filename))
def test_read_data_labeled(self):
""" Tests the function read_data for labeled data. """
references = pre.read_data(self.testfilename, labeled=True)
truth = [
[
Reference(
0, 'm jones',
'symbol intersect detect method improv spatial intersect join',
['e rundensteiner', 'y huang'], 'geoinformatica', '81'),
Reference(
1, 'matthew c jones',
'improv spatial intersect join symbol intersect detect', [
'e rundensteiner', 'h kuno', 'p marron', 'v taube',
'y ra'
], 'sigmodels.intern manag data', '81'),
Reference(2, 'matthew c jones',
'view materi techniqu complex hirarch object',
['e rundensteiner', 'y huang'],
'ssd symposium larg spatial databas', '81')
],
[
Reference(
3, 'mike w miller', 'domin draw bipartit graph',
['l berg'],
'sigucc special interest group univers comput servic',
'185'),
Reference(
4, 'mike w miller', 'rel compromis statist databas', [],
'sigucc special interest group univers comput servic',
'185')
],
[
Reference(
5, 'c chen', 'formal approach scenario analysi',
['d kung', 'j samuel', 'j gao', 'p hsia', 'y toyoshima'],
'ieee softwar', '94')
],
[
Reference(6, 'jane j robinson', 'discours code clue context',
[], 'acl meet the associ comput linguist', '69'),
Reference(7, 'jane j robinson', 'diagram grammar dialogu', [],
'cooper interfac inform system', '69')
],
[
Reference(8, 'a gupta', 'iri h java distanc educ', [
'a gonzalez', 'a hamid', 'c overstreet', 'h wahab',
'j wild', 'k maly', 's ghanem', 'x zhu'
], 'acm journal educ resourc comput', '0')
],
[
Reference(
9, 'mary d brown',
'intern redund represent limit bypass support pipelin adder regist '
'file', ['y patt'],
'proceed the th ieee intern symposium high '
'perform comput architectur hpca intern symposium high perform comput'
' architectur talk slide', '43')
]
]
self.assertEquals(references, truth)
def test_extract_hr_acc(self):
data = read_data(path=PreprocessingTests.filename,
base_datetime=PreprocessingTests.base_datetime)
frame = extract_hr_acc(data)
columns = set(frame.columns.tolist())
self.assertIn('hr', columns)
self.assertIn('acc', columns)
self.assertIn('acc_x', columns)
self.assertIn('acc_y', columns)
self.assertIn('acc_z', columns)
self.assertIn('ratio', columns)
self.assertIn('ratio_log', columns)
self.assertItemsEqual(data.index.tolist(), frame.index.tolist())
self.assertEqual(len(frame), len(notnull(frame)))
data['eratio'] = (data.hr / data.IMU_Chest_Magnitude)
data['eratio_log'] = log(data['eratio'])
data = data.fillna(method='ffill').fillna(method='bfill')
self.assertItemsEqual(data.hr.tolist(), frame.hr.tolist())
self.assertItemsEqual(data.IMU_Chest_Magnitude.tolist(),
frame.acc.tolist())
self.assertItemsEqual(data.IMU_Chest_x.tolist(), frame.acc_x.tolist())
self.assertItemsEqual(data.IMU_Chest_y.tolist(), frame.acc_y.tolist())
self.assertItemsEqual(data.IMU_Chest_z.tolist(), frame.acc_z.tolist())
self.assertItemsEqual(data.eratio.tolist(), frame.ratio.tolist())
self.assertItemsEqual(data.eratio_log.tolist(),
frame.ratio_log.tolist())
def test_tfidf(self):
""" Tests the tfidf function. """
references = pre.read_data(self.testfilename)
corpus = pre.get_corpus(references) + ['matt']
idf = mod.get_idf(corpus)
tfidf = mod.tfidf('matt',
mod.get_words('matt matt huang kuno jones c marron brown'),
idf)
truth = idf['matt'] * 2
self.assertEquals(tfidf, truth)
def test_read_data_no_base_datetime(self):
frame = read_data(path=PreprocessingTests.filename)
columns = set(frame.columns.tolist())
self.assertIn('activityID', columns)
self.assertIn('hr', columns)
self.assertIn('IMU_Chest_Magnitude', columns)
self.assertIn('IMU_Chest_x', columns)
self.assertIn('IMU_Chest_y', columns)
self.assertIn('IMU_Chest_z', columns)
self.assertLess(datetime.now(), frame.index[0])
def test_get_corpus(self):
""" Tests the function get_corpus. """
references = pre.read_data(self.testfilename)
corpus = pre.get_corpus(references)
truth = ['m jones', 'e rundensteiner', 'y huang', 'matthew c jones',
'e rundensteiner', 'h kuno', 'p marron', 'v taube', 'y ra',
'matthew c jones', 'e rundensteiner', 'y huang', 'mike w miller',
'l berg', 'mike w miller', 'c chen', 'd kung', 'j samuel', 'j gao',
'p hsia', 'y toyoshima', 'jane j robinson', 'jane j robinson',
'a gupta', 'a gonzalez', 'a hamid', 'c overstreet', 'h wahab', 'j wild',
'k maly', 's ghanem', 'x zhu', 'mary d brown', 'y patt']
self.assertEquals(corpus, truth)
def test_norm_tfidf(self):
""" Tests the normalized tfidf. """
references = pre.read_data(self.testfilename)
corpus = pre.get_corpus(references) + ['matt']
idf = mod.get_idf(corpus)
words = mod.get_words('matt matt huang kuno jones c marron brown')
n_tfidf = mod.norm_tfidf('matt', words, idf)
truth = mod.tfidf('matt', words, idf)
denom = 0
for word in words:
denom += mod.tfidf(word, words, idf) ** 2
denom = math.sqrt(denom)
truth /= denom
self.assertEquals(n_tfidf, truth)
def test_get_corpus(self):
""" Tests the function get_corpus. """
references = pre.read_data(self.testfilename)
corpus = pre.get_corpus(references)
truth = [
'm jones', 'e rundensteiner', 'y huang', 'matthew c jones',
'e rundensteiner', 'h kuno', 'p marron', 'v taube', 'y ra',
'matthew c jones', 'e rundensteiner', 'y huang', 'mike w miller',
'l berg', 'mike w miller', 'c chen', 'd kung', 'j samuel', 'j gao',
'p hsia', 'y toyoshima', 'jane j robinson', 'jane j robinson',
'a gupta', 'a gonzalez', 'a hamid', 'c overstreet', 'h wahab',
'j wild', 'k maly', 's ghanem', 'x zhu', 'mary d brown', 'y patt'
]
self.assertEquals(corpus, truth)
def train(n, path_to_csv):
'''
train the model
Input
-----
n - polynomial degree
path_to_csv - training set path
'''
# prepare data
#-------------------------------------------------
# load data
raw_data = pre.read_data(path_to_csv)
# rescale data
feature_data, means, stds = pre.feature_scaling(raw_data, n)
# split to train and test sets
data = pre.split_data(feature_data)
# create instance of the neural network
#-------------------------------------------------
model = ionn.InOutNN(n, init_random=True)
# train the model
#-------------------------------------------------
model.train(data['train'][:, :-1],
data['train'][:, -1],
alpha=0.5,
itmax=20000,
verbose=False)
#compute polynomial coefficients (from rescaled features)
poly_coeffs = model.polynomial_coefficients(means, stds)
print(np.flip(poly_coeffs, axis=0))
# test the model
#-------------------------------------------------
train_err, test_err = model.test(data['train'][:, :-1], data['train'][:,
-1],
data['test'][:, :-1], data['test'][:, -1])
# dump the model - Python object serialisation
#-------------------------------------------------
dir_path = os.path.dirname(os.path.realpath(__file__))
model_file = dir_path + "/../MODEL/inoutnn_{}.pcl".format(n)
with open(model_file, 'wb') as f:
pickle.dump(model, f)
return poly_coeffs
def test_get_coauthorship_transactions(self):
""" Tests the function get_coauthorship_transactions. """
references = pre.read_data(self.testfilename)
truth = [[
['m jones', 'e rundensteiner', 'y huang'],
['matthew c jones', 'e rundensteiner', 'h kuno', 'p marron', 'v taube',
'y ra'],
['matthew c jones', 'e rundensteiner', 'y huang']],
[['mike w miller', 'l berg'],
['mike w miller']],
[['c chen', 'd kung', 'j samuel', 'j gao', 'p hsia', 'y toyoshima']],
[['jane j robinson'],
['jane j robinson']],
[['a gupta', 'a gonzalez', 'a hamid', 'c overstreet', 'h wahab',
'j wild', 'k maly', 's ghanem', 'x zhu']],
[['mary d brown', 'y patt']]]
for i in range(len(references)):
transactions = mod.get_coauthorship_transactions(references[i])
self.assertEquals(transactions, truth[i])
def test_soft_tfidf(self):
""" Tests the soft tfidf. """
references = pre.read_data(self.testfilename)
corpus = pre.get_corpus(references) + ['matt', 'brow']
idf = mod.get_idf(corpus)
name_a = 'matt huang kuno jones brow'
words_a = mod.get_words(name_a)
name_b = 'matthew jones c marron brown'
words_b = mod.get_words(name_b)
s_tfidf = mod.soft_tfidf(name_a, name_b, idf)
close = ['matt', 'jones', 'brow']
truth = \
mod.norm_tfidf('matt', words_a, idf) * \
mod.norm_tfidf('matthew', words_b, idf) * \
Levenshtein.ratio('matt', 'matthew') + \
mod.norm_tfidf('jones', words_a, idf) * \
mod.norm_tfidf('jones', words_b, idf) * \
Levenshtein.ratio('jones', 'jones') + \
mod.norm_tfidf('brow', words_a, idf) * \
mod.norm_tfidf('brown', words_b, idf) * \
Levenshtein.ratio('brow', 'brown')
self.assertEquals(s_tfidf, truth)
def setUp(self):
super(RankingSampleTestCase, self).setUp()
self.references = pre.read_data(self.testfilename)
self.base_partitioning = [[0, 0, 1], [0, 1], [0], [0, 0], [0], [0]]
self.alt_partitionings = [[[0, 1, 1], [1, 0], [0], [0, 0], [0], [0]],
[[1, 0, 1], [0, 1], [0], [0, 0], [0], [0]]]
def setUp(self):
super(RankingSampleTestCase, self).setUp()
self.references = pre.read_data(self.testfilename)
self.base_partitioning = [[0, 0, 1], [0, 1], [0], [0, 0], [0], [0]]
self.alt_partitionings = [[[0, 1, 1], [1, 0], [0], [0, 0], [0], [0]],
[[1, 0, 1], [0, 1], [0], [0, 0], [0], [0]]]
def test_get_coauthorship_rules(self):
""" Tests the function get_coauthorship_rules. """
references = pre.read_data(self.testfilename)
truth = [{'m jones': {'e rundensteiner': 1.0,
'y huang': 1.0},
'e rundensteiner': {'m jones': 1.0/3,
'y huang': 2.0/3,
'matthew c jones': 2.0/3,
'h kuno': 1.0/3,
'p marron': 1.0/3,
'v taube': 1.0/3,
'y ra': 1.0/3
},
'y huang': {'m jones': 1.0/2,
'e rundensteiner': 1.0,
'matthew c jones': 1.0/2
},
'matthew c jones': {'e rundensteiner': 1.0,
'y huang': 1.0/2,
'h kuno': 1.0/2,
'p marron': 1.0/2,
'v taube': 1.0/2,
'y ra': 1.0/2
},
'h kuno': {'e rundensteiner': 1.0,
'matthew c jones': 1.0,
'p marron': 1.0,
'v taube': 1.0,
'y ra': 1.0
},
'p marron': {'e rundensteiner': 1.0,
'matthew c jones': 1.0,
'h kuno': 1.0,
'v taube': 1.0,
'y ra': 1.0
},
'v taube': {'e rundensteiner': 1.0,
'matthew c jones': 1.0,
'h kuno': 1.0,
'p marron': 1.0,
'y ra': 1.0
},
'y ra': {'e rundensteiner': 1.0,
'matthew c jones': 1.0,
'h kuno': 1.0,
'p marron': 1.0,
'v taube': 1.0
}},
{'mike w miller': {'l berg': 1.0/2},
'l berg': {'mike w miller': 1.0}},
{'c chen': {'d kung': 1.0,
'j samuel': 1.0,
'j gao': 1.0,
'p hsia': 1.0,
'y toyoshima': 1.0
},
'd kung': {'c chen': 1.0,
'j samuel': 1.0,
'j gao': 1.0,
'p hsia': 1.0,
'y toyoshima': 1.0
},
'j samuel': {'c chen': 1.0,
'd kung': 1.0,
'j gao': 1.0,
'p hsia': 1.0,
'y toyoshima': 1.0
},
'j gao': {'c chen': 1.0,
'd kung': 1.0,
'j samuel': 1.0,
'p hsia': 1.0,
'y toyoshima': 1.0
},
'p hsia': {'c chen': 1.0,
'd kung': 1.0,
'j samuel': 1.0,
'j gao': 1.0,
'y toyoshima': 1.0
},
'y toyoshima': {'c chen': 1.0,
'd kung': 1.0,
'j samuel': 1.0,
'j gao': 1.0,
'p hsia': 1.0
}},
{},
{'a gupta': {'a gonzalez': 1.0,
'a hamid': 1.0,
'c overstreet': 1.0,
'h wahab': 1.0,
'j wild': 1.0,
'k maly': 1.0,
's ghanem': 1.0,
'x zhu': 1.0
},
'a gonzalez': {'a gupta': 1.0,
'a hamid': 1.0,
'c overstreet': 1.0,
'h wahab': 1.0,
'j wild': 1.0,
'k maly': 1.0,
's ghanem': 1.0,
'x zhu': 1.0
},
'a hamid': {'a gupta': 1.0,
'a gonzalez': 1.0,
'c overstreet': 1.0,
'h wahab': 1.0,
'j wild': 1.0,
'k maly': 1.0,
's ghanem': 1.0,
'x zhu': 1.0
},
'c overstreet': {'a gupta': 1.0,
'a gonzalez': 1.0,
'a hamid': 1.0,
'h wahab': 1.0,
'j wild': 1.0,
'k maly': 1.0,
's ghanem': 1.0,
'x zhu': 1.0
},
'h wahab': {'a gupta': 1.0,
'a gonzalez': 1.0,
'a hamid': 1.0,
'c overstreet': 1.0,
'j wild': 1.0,
'k maly': 1.0,
's ghanem': 1.0,
'x zhu': 1.0
},
'j wild': {'a gupta': 1.0,
'a gonzalez': 1.0,
'a hamid': 1.0,
'c overstreet': 1.0,
'h wahab': 1.0,
'k maly': 1.0,
's ghanem': 1.0,
'x zhu': 1.0
},
'k maly': {'a gupta': 1.0,
'a gonzalez': 1.0,
'a hamid': 1.0,
'c overstreet': 1.0,
'h wahab': 1.0,
'j wild': 1.0,
's ghanem': 1.0,
'x zhu': 1.0
},
's ghanem': {'a gupta': 1.0,
'a gonzalez': 1.0,
'a hamid': 1.0,
'c overstreet': 1.0,
'h wahab': 1.0,
'j wild': 1.0,
'k maly': 1.0,
'x zhu': 1.0
},
'x zhu': {'a gupta': 1.0,
'a gonzalez': 1.0,
'a hamid': 1.0,
'c overstreet': 1.0,
'h wahab': 1.0,
'j wild': 1.0,
'k maly': 1.0,
's ghanem': 1.0
}},
{'mary d brown': {'y patt': 1.0},
'y patt': {'mary d brown': 1.0}}
]
for i in range(len(references)):
rules = mod.get_coauthorship_rules(references[i])
for author_a in rules:
for author_b in rules[author_a]:
self.assertEquals(round(rules[author_a][author_b], 4),
round(truth[i][author_a][author_b], 4))
for author_a in truth[i]:
for author_b in truth[i][author_a]:
self.assertEquals(round(rules[author_a][author_b], 4),
round(truth[i][author_a][author_b], 4))
from PyTango import DeviceProxy
sys.path.append("../../")
from src.preprocessing import read_data, extract_hr_acc
if __name__ == '__main__':
proxy = DeviceProxy("C3/hr_monitor/1")
dirname = os.path.dirname(__file__)
filename = os.path.join(dirname, 'dataset.dat')
print(filename)
data = extract_hr_acc(read_data(filename))
data = data.resample('1000L')
DP = namedtuple("DP", ["timestamp", "hr", "acc_x", "acc_y", "acc_z"])
i = 0
prow, ptimestamp = None, None
for index, row in data.iterrows():
timestamp = float(index.to_datetime().strftime('%s.%f'))
progress = (i / len(data)) * 100
if prow is not None:
datapoint = DP(timestamp=ptimestamp, hr=prow['hr'],
acc_x=prow['acc_x'], acc_y=prow['acc_y'],
acc_z=prow['acc_z'])
print("%s - %s, %s - %s%%" % (datetime.now(), index.to_datetime(),
datapoint.hr, progress))
proxy.register_datapoint(datapoint)