def data_normalize():
for row in train_rows:
row[3] = feas.normalize(row[3])
#for i in range(len(row[3])):
# row[3][i] *= fea_weight[i]
for row in test_rows:
row[3] = feas.normalize(row[3])
def data_normalize():
for row in train_rows:
row[3] = feas.normalize(row[3])
#for i in range(len(row[3])):
# row[3][i] *= fea_weight[i]
for row in test_rows:
row[3] = feas.normalize(row[3])
def predict():
""" Predict sleep vs. non-sleep """
start_time = time.time()
device_uuid, date_time = request.args.get('deviceUUID'), request.args.get(
'datetime')
model_filename = get_model_filename(device_uuid, date_time)
with open(model_filename, 'rb') as f:
clf = pickle.load(f)
baseline_filename = get_baseline_filename(device_uuid, date_time)
with open(baseline_filename, 'rb') as f:
baseline = pickle.load(f)
data_filename = get_data_filename(device_uuid, date_time)
with open(data_filename, 'r') as f:
rows = f.readlines()
if len(rows) < config.prediction_data_size:
return jsonify({
"status": 1,
"message": "Not enough data! %d" % len(rows)
})
raw = np.zeros((config.prediction_data_size, 3))
for i, j in zip(range(config.prediction_data_size),
range(len(rows) - config.prediction_data_size, len(rows))):
raw[i] = [int(val) for val in rows[j].strip().split(',')]
norm = features.normalize(raw)
temp_features = features.extract_multi_features(norm,
step=config.step_size,
x_len=config.sample_size)
X = features.get_calibrated_features(temp_features, baseline['features'])
"""
json_ = request.json
n_features = X.shape[1]
feature_importance = np.zeros(n_features)
if json_ and 'feature_importance' in json_:
feature_importance[0] = json_['feature_importance']['flex']
feature_importance[1] = json_['feature_importance']['eda']
for i in range(2, n_features):
feature_importance[i] = json_['feature_importance']['ecg'] / float(n_features - 2)
else:
feature_importance[0] = 1 / 3.
feature_importance[1] = 1 / 3.
for i in range(2, n_features):
feature_importance[i] = 1 / float(3 * (n_features - 2))
y = clf.predict(X, feature_importance)
"""
y = clf.decision_function(X) - baseline['hboss_base']
return jsonify({
"status": 0,
"sleep": list(y),
"mean_sleep": np.mean(y),
"max_sleep": np.max(y),
"time": (time.time() - start_time)
})
def train():
""" Train the predictor on the data collected """
start_time = time.time()
device_uuid, date_time = request.args.get('deviceUUID'), request.args.get(
'datetime')
data_filename = get_data_filename(device_uuid, date_time)
with open(data_filename, 'r') as f:
rows = f.readlines()
with open(config.awake_filename, 'rb') as f:
awake_features = pickle.load(f)
if len(rows) < config.min_train_data_size:
return jsonify({
"status": 1,
"message": "Not enough training data! %d" % len(rows)
})
raw = np.zeros((len(rows), 3))
for i in range(len(rows)):
raw[i] = [int(val) for val in rows[i].strip().split(',')]
norm = features.normalize(raw)
temp_features = features.extract_multi_features(norm,
step=config.step_size,
x_len=config.sample_size)
baseline_features = features.get_baseline_features(temp_features)
norm_features = features.get_calibrated_features(temp_features,
baseline_features)
X = np.concatenate((awake_features, norm_features), axis=0)
X[:, 1] = np.abs(np.random.normal(0, 0.01, len(X)))
app.logger.info(
'Training classifier using %d feature sets, each containing %d features'
% (X.shape[0], X.shape[1]))
clf = HBOS(contamination=0.05)
clf.fit(X)
model_filename = get_model_filename(device_uuid, date_time)
with open(model_filename, 'wb') as f:
pickle.dump(clf, f)
pred = clf.decision_function(X)
baseline = {'features': baseline_features, 'hboss_base': np.min(pred)}
baseline_filename = get_baseline_filename(device_uuid, date_time)
with open(baseline_filename, 'wb') as f:
pickle.dump(baseline, f)
return jsonify({"status": 0, "time": (time.time() - start_time)})
train_path = 'data/train.tsv'
submission_path = 'data/test.tsv'
n = 10000
tags = pickle.load(open('cache/tagged.%s.pkl' % (n)))
custom_contents = np.array(pickle.load(open('cache/custom_contents.%s.pkl' % (n))))
submission_custom_contents = np.array(pickle.load(open('cache/s.custom_contents.pkl')))
submission_tags = pickle.load(open('cache/s.tagged.pkl'))
print 'Reading %s data' % (n)
data = data_io.read_data(train_path, n)
submission_data = data_io.read_data(submission_path, 10000) # use all
contents = get_doc_contents(data[:, 2])
contents = [ features.normalize(content) for content in contents ]
Y = data[:, -1].astype(int)
bestwords = get_bestwords(contents, Y, 100000, n)
submission_contents = get_doc_contents(submission_data[:, 2])
submission_contents = [ features.normalize(submission_content) for submission_content in submission_contents ]
X_submission_ids = submission_data[:, 1]
v = TfidfVectorizer(min_df = 2, binary = True, norm = 'l2', smooth_idf = True, sublinear_tf = True, strip_accents = 'unicode', vocabulary = bestwords, ngram_range = (1,2))
X = v.fit_transform(contents)
X_submission = v.transform(submission_contents)
del data
del submission_data
def train(num_labels=None,
gridcv=False,
randomcv=False,
kbest=None,
rfecv=False):
# Load the track library. Collect metadata labels. Generate a target
# matrix. Load features for each track in the target matrix.
libtracks = library.tracks()
labels = collect_labels(libtracks, num_labels)
tracklist, target = generate_target(labels)
data = Dataset(features.normalize(features.matrix(tracklist)), target)
feat_names = features.names()
if kbest:
reduce_kbest(data, feat_names, kbest)
if rfecv:
reduce_rfecv(data, feat_names)
train, test = split_dataset(data, test_size=0.4, random_state=0)
# A random forest should be able to handle the excessive dimensionality
# of our dataset relative to the number of samples.
clf = RandomForestClassifier(n_estimators=120, n_jobs=-1, verbose=1)
if randomcv:
print "random parameter search..."
randomsearch(
clf, train, 20, {
"max_depth": [3, None],
"max_features": scipy.stats.randint(50, 100),
"min_samples_split": scipy.stats.randint(2, 11),
"min_samples_leaf": scipy.stats.randint(1, 11),
"bootstrap": [True, False],
"criterion": ["gini", "entropy"]
})
if gridcv:
print "grid parameter search..."
gridsearch(
clf, train, {
"max_depth": [3, None],
"max_features": [50, 75, 100],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [True, False],
"criterion": ["gini", "entropy"]
})
print("training classifier...")
clf.fit(*train)
mean_importance = clf.feature_importances_.mean()
# Measure prediction accuracy for the original training run.
pred_target = clf.predict(test.input)
orig_score = accuracy_score(test.target, pred_target)
print("accuracy score with %d features: %.2f%%" %
(len(feat_names), orig_score * 100.0))
# Reduce the feature set.
print("selecting best features...")
sfm = SelectFromModel(clf, threshold='1.5*mean')
sfm.fit(*train)
# Print the names of the most important features
feature_subset = sfm.get_support(indices=True)
for i in feature_subset:
importance = clf.feature_importances_[i] / mean_importance
print " %.1f: '%s'" % (importance, feat_names[i])
# make a new training set with just the useful features.
print("preparing new training subset...")
slim_train = transform_input(sfm, train)
slim_test = transform_input(sfm, test)
feat_names = [feat_names[i] for i in feature_subset]
# train a new classifier using the reduced feature set.
print("training subset classifier...")
clf_slim = RandomForestClassifier(n_estimators=120, n_jobs=-1, verbose=1)
clf_slim.fit(*slim_train)
# measure accuracy of the retrained models
pred_slim = clf_slim.predict(slim_test.input)
slim_score = accuracy_score(slim_test.target, pred_slim)
print("subset accuracy with %d features: %.2f%%" %
(len(feature_subset), slim_score * 100.0))
category_ = get_category(label)
if category_ is None:
return None
else:
categories.add(category_)
return sorted(list(categories))
# Convert the instrument labels, and keep only the segments whose instruments are all covered by the taxonomy
for i, feature in enumerate(features_):
categories = get_categories(labels_[i])
if categories is not None:
features.append(feature)
labels.append(categories)
features = np.array(features)
features = normalize(features)
labels = np.array(labels)
# Result summary
sum_scores = dict()
# (C, sigma)
svm_params = (2, 1)
selected_feats = {
'Bass drum': [('obsir', 3), ('obsir', 2), ('mfcc', 0), ('mfcc', 10), ('obsir', 4), ('obsir', 1), ('temporal_shape', 0), ('temporal_shape', 3), ('obsir', 5), ('mfcc', 12), ('spectral_shape', 1), ('obsir', 7), ('mfcc', 7), ('spread', 0), ('spectral_shape', 2), ('mfcc', 1)],
'Snare drum': [('obsir', 2), ('mfcc', 2), ('spectral_shape', 3), ('spread', 0), ('mfcc', 4), ('lpc', 3), ('lpc', 5), ('obsir', 3), ('flatness', 0), ('spectral_shape', 0), ('mfcc', 0), ('lpc', 0), ('temporal_shape', 2), ('obsir', 7), ('zcr', 0), ('obsir', 4)],
'Hi-hat': [('lpc', 0), ('temporal_shape', 2), ('mfcc', 4), ('obsir', 8), ('lpc', 5), ('zcr', 0), ('lpc', 1), ('obsir', 2), ('temporal_shape', 1), ('lpc', 3), ('lpc', 2), ('spectral_shape', 3), ('mfcc', 9), ('mfcc', 10), ('mfcc', 11), ('energy', 0)],
}
number_feats = {
if category_ is None:
return None
else:
categories.add(category_)
return sorted(list(categories))
# Convert the instrument labels, and keep only the segments whose instruments are all covered by the taxonomy
for i, feature in enumerate(features_):
categories = get_categories(labels_[i])
if categories is not None:
features.append(feature)
labels.append(categories)
features = np.array(features)
features = normalize(features)
labels = np.array(labels)
# Result summary
sum_scores = dict()
# (C, sigma)
svm_params = (2, 1)
selected_feats = {
'Bass drum': [('obsir', 3), ('obsir', 2), ('mfcc', 0), ('mfcc', 10),
('obsir', 4), ('obsir', 1), ('temporal_shape', 0),
('temporal_shape', 3), ('obsir', 5), ('mfcc', 12),
('spectral_shape', 1), ('obsir', 7), ('mfcc', 7),
('spread', 0), ('spectral_shape', 2), ('mfcc', 1)],
'Snare drum': [('obsir', 2), ('mfcc', 2), ('spectral_shape', 3),