def cross_validation(ds):
alg.reset_algorithm([
svmObj, rfcObj, gbcObj, logObj, knnObj, mlpObj, gnbObj, ensObj,
apriObj, astaltObj, fib4Obj, annObj
])
reset_dataset(ds)
c = 0
if (ds.split == 'groupKFold'):
splitMethod = kf.split(ds.X, ds.Y, ds.MRNs.astype(int))
elif (ds.split == 'KFold'):
splitMethod = normalKF.split(ds.X, ds.Y)
for train_index, test_index in splitMethod:
print(c)
preprocessing(ds, train_index, test_index, c)
alg.svm_class(ds.X_tr_imp_scl[c][:, svmObj.features],
ds.X_ts_imp_scl[c][:, svmObj.features], ds.Y_tr[c],
ds.Y_ts[c], svmObj, c)
alg.rfc_class(ds.X_tr_imp_scl[c][:, rfcObj.features],
ds.X_ts_imp_scl[c][:, rfcObj.features], ds.Y_tr[c],
ds.Y_ts[c], rfcObj, c)
alg.gbc_class(ds.X_tr_imp_scl[c][:, gbcObj.features],
ds.X_ts_imp_scl[c][:, gbcObj.features], ds.Y_tr[c],
ds.Y_ts[c], gbcObj, c)
alg.log_class(ds.X_tr_imp_scl[c][:, logObj.features],
ds.X_ts_imp_scl[c][:, logObj.features], ds.Y_tr[c],
ds.Y_ts[c], logObj, c)
alg.knn_class(ds.X_tr_imp_scl[c][:, knnObj.features],
ds.X_ts_imp_scl[c][:, knnObj.features], ds.Y_tr[c],
ds.Y_ts[c], knnObj, c)
alg.mlp_class(ds.X_tr_imp_scl[c][:, mlpObj.features],
ds.X_ts_imp_scl[c][:, mlpObj.features], ds.Y_tr[c],
ds.Y_ts[c], mlpObj, c)
alg.gnb_class(ds.X_tr_imp_scl[c][:, gnbObj.features],
ds.X_ts_imp_scl[c][:, gnbObj.features], ds.Y_tr[c],
ds.Y_ts[c], gnbObj, c)
ann_class(ds.X_tr_imp_scl[c][:, annObj.features],
ds.X_ts_imp_scl[c][:, annObj.features], ds.Y_tr[c] / 4,
ds.Y_ts[c], annObj, c)
alg.ens_class(svmObj, rfcObj, gbcObj, logObj, knnObj, mlpObj, gnbObj,
ds.Y_ts[c], ensObj, c)
alg.apri_class(ds.X_ts_imp[c][:, apriObj.features], ds.Y_ts[c],
apriObj, c)
alg.astalt_class(ds.X_ts_imp[c][:, astaltObj.features], ds.Y_ts[c],
astaltObj, c)
alg.fib4_class(ds.X_ts_imp[c][:, fib4Obj.features], ds.Y_ts[c],
fib4Obj, c)
c = c + 1
# This is the point where I need to get all the AUROCs
print_results(algorithmArray)
print_table(
algorithmArray, len(list(itertools.chain.from_iterable(ds.ts_indxs))),
True
) # The second argument indicates whether uncertanties should be printed here
def find_top_similar_ask1(q, qlist):
#使用tf-idf的方法
q = data.preprocessing([q])
vectorizer = TfidfVectorizer()
#方法1：使输入与全部问题库的信息作比较
# x = vectorizer.fit_transform(qlist)
# input_vec = vectorizer.transform(q)
#方法2：从倒排表中取出相关联的索引，仅与有相同字的问题作比较
index_list = []
invert_table = load_inverse_table()
for c in cut(q):
for i in c:
if i in invert_table.keys():
values = invert_table[i]
for value in values:
index_list.append(int(value))
# index_list += invert_table[i]
index_list = list(set(index_list))
qlist = np.asarray(qlist)
x = qlist[index_list]
x = vectorizer.fit_transform(x)
input_vec = vectorizer.transform(q)
res = cosine_similarity(input_vec, x)
n = np.argmax(res)
return n
def gen(target_molecules):
for i in range(0, len(target_molecules)):
mol = target_molecules[i]
while (len(mol) > 50):
mol = mol[:-1]
target_molecules[i] = mol
# Create preprocessing instance
pp = preprocessing()
# Load data from small data set
X_train, y_train, X_test, y_test = pp.load_data()
# Remove non characters
for i in range(0, len(target_molecules)):
mol = target_molecules[i]
for char in mol:
if char in pp.charset:
print("We good.")
else:
mol = mol.replace(char, '')
target_molecules[i] = mol
print("Oopps. Removing bad char:", char)
# Create & load model
model = nn(X_train, y_train, X_test, y_test)
model.load(pp)
# Generate a molecule
molecules = model.generate(target=target_molecules,
preprocessing_instance=pp,
hit_rate=20)
return molecules
def train():
data_gen_args = dict(rotation_range=90,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.2)
image_datagen = tf.keras.preprocessing.image.ImageDataGenerator(
**data_gen_args)
mask_datagen = tf.keras.preprocessing.image.ImageDataGenerator(
**data_gen_args)
seed = 1
model = VariationalUnet()
optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
global_step = tf.Variable(0)
num_epochs = 50
batch_size = 4
image_generator = image_datagen.flow_from_directory(
'data/membrane/train/images/',
class_mode=None,
batch_size=batch_size,
seed=seed)
mask_generator = mask_datagen.flow_from_directory(
'data/membrane/train/masks/',
class_mode=None,
batch_size=batch_size,
seed=seed)
train_generator = zip(image_generator, mask_generator)
for epoch in range(num_epochs):
print("Epoch: ", epoch)
for (img, mask) in train_generator:
img, mask = preprocessing(img, mask)
loss_value, grads, reconstruction, mus_sigmas = grad(
model, img, mask)
optimizer.apply_gradients(zip(grads, model.trainable_variables),
global_step)
if global_step.numpy() % 1 == 0:
print("Step: {}, Loss: {}".format(
global_step.numpy(),
loss(mask, reconstruction, mus_sigmas).numpy()))
def find_top_similar_ask2(q, qlist):
#使用glove的方法
#输入问题q，返回答案
# 1 将问题和问题库里的问题进行embedding，转换成句子向量
# 2 从倒排表里筛选候选
# 3 计算问题与候选问题的相似度
# 4 选取相似度取值最大的
q = data.preprocessing([q])
q = "".join(q)
emb, vocab = get_glove_data()
emb = np.asarray(emb)
# 输入问题的vec值
q_vec = get_words_vec(q, emb, vocab)
qlist_vec = []
i = 0
for qlist_ in qlist:
qlist_vec.append(get_words_vec(qlist_, emb, vocab))
i += 1
if i % 5000 == 0:
print(i)
qlist_vec = np.asarray(qlist_vec)
abs_v = abs(qlist_vec[0])
# 存储所有返回结果的索引
res = []
# 从倒排表中取出相关联的索引
index_list = []
invert_table = load_inverse_table()
for c in cut([q]):
for i in c:
if i in invert_table.keys():
values = invert_table[i]
for value in values:
index_list.append(int(value))
# index_list += invert_table[i]
index_list = list(set(index_list))
# 遍历倒排表内所有值，将list中所有vec值与input_q的vec值做对比，绝对值差较小的数的索引存入res中
# for value in qlist_vec[index_list]:
# if abs(q_vec - value) < abs_v:
# abs_v = abs(q_vec - value)
# res.append(qlist_vec[index_list].tolist().index(value))
values = qlist_vec[index_list]
res = cosine_similarity(q_vec, values)
n = np.argmax(res)
return n
def detect_event(n_point_df,
columns=['app1'],
detect_position=4,
median_window=7,
N=4,
n=10,
niter=100,
kappa=35,
gamma=0.15,
state_threshold=70,
noise_level=80):
var_med_an_npdf = preprocessing(n_point_df,
columns=columns,
m=median_window,
S=1,
N=N,
niter=niter,
kappa=kappa,
gamma=gamma)
[cl_steady,
cl_transients] = find_steady_states(var_med_an_npdf,
n_point=True,
columns=columns,
n=detect_position,
state_threshold=state_threshold,
noise_level=noise_level)
n_point_df = var_med_an_npdf
if cl_transients.empty:
detected = False
else:
detected = True
return cl_transients, detected
if __name__ == '__main__':
# check for input arguments
if len(sys.argv) == 1:
print(
'No arguments passed. Please specify classification method ("log", "nn" or "linreg").'
)
sys.exit()
arg = sys.argv[1]
if arg == 'log':
# initialise features and targets using credit card data
X, y = data.preprocessing(remove_data=True)
store_acc_train = []
store_acc_test = []
list_of_etas = [1E-4, 1E-3, 1E-2, 1E-1, 1]
for eta in list_of_etas:
lr = LogisticRegression(X,
y,
eta=eta,
minibatch_size=100,
epochs=100,
folds=10,
benchmark=False)
lr.logistic_regression()
store_acc_train.append(lr.acc_epoch_train)
store_acc_test.append(lr.acc_epoch_test)