def Get_Original_Data(self):
dataset = FeatureCalculater.FeatureCalculater()
sql = 'select * from students_final'
dataset.executer.execute(sql)
original_data = dataset.executer.fetchall()
original_data = np.array(original_data)
return original_data
def get_label(self):
dataset = FeatureCalculater.FeatureCalculater()
sql = 'describe students_final'
dataset.executer.execute(sql)
labels = dataset.executer.fetchall()
label1 = []
for i in labels:
label1.append(i[0])
return label1
def find_null_index(self, xdata):
'''
xdata: 输入数据集
@return nullIndexList xdata 中 含有null的列下标
'''
nullIndexList = []
xdata = np.array(xdata)
dataset = FeatureCalculater.FeatureCalculater()
sql = 'describe students_final'
dataset.executer.execute(sql)
labels = dataset.executer.fetchall()
for i in range(xdata.shape[1]):
for j in range(xdata.shape[0]):
if xdata[j, i] == None:
nullIndexList.append(i)
print(labels[i][0])
break
pass
# tempdata=np.delete(xdata, nullIndexList, axis=1)
return nullIndexList
def update_dataSet():
'''
更新表‘students_final’与钱有关的部分列
将null更新成0
'''
dataset =FeatureCalculater.FeatureCalculater()
dataset.executer.execute("DESCRIBE students_final")
columnName = dataset.executer.fetchall()
index = -1
for i in range(len(columnName)):
if str(columnName[i][0]) == 'canteen_total_amount' :
index = i
break
if index == -1:
print('异常：未发现' + 'canteen_total_amount')
for i in range(index,len(columnName)):
sql = 'update students_final set {0}=0 where {1} is null'
print(sql.format(columnName[i][0], columnName[i][0]))
dataset.executer.execute(sql.format(columnName[i][0], columnName[i][0]))
def run(self):
tf.set_random_seed(1)
np.random.seed(1)
self.final_gan_data.clear()
with tf.variable_scope('Generator'):
G_in = tf.placeholder(
tf.float32,
[None, self.N_IDEAS
]) # random ideas (could from normal distribution)
G_l1 = tf.layers.dense(G_in, 128, tf.nn.relu)
G_out = tf.layers.dense(
G_l1, self.ART_COMPONENTS
) # making a painting from these random ideas
with tf.variable_scope('Discriminator'):
real_art = tf.placeholder(
tf.float32, [None, self.ART_COMPONENTS],
name='real_in') # receive art work from the famous artist
D_l0 = tf.layers.dense(real_art, 128, tf.nn.relu, name='l')
prob_artist0 = tf.layers.dense(
D_l0, 1, tf.nn.sigmoid,
name='out') # probability that the art work is made by artist
# reuse layers for generator
D_l1 = tf.layers.dense(
G_out, 128, tf.nn.relu, name='l',
reuse=True) # receive art work from a newbie like G
prob_artist1 = tf.layers.dense(
D_l1, 1, tf.nn.sigmoid, name='out',
reuse=True) # probability that the art work is made by artist
D_loss = -tf.reduce_mean(
tf.log(prob_artist0) + tf.log(1 - prob_artist1))
G_loss = tf.reduce_mean(tf.log(1 - prob_artist1))
# AdamOptimizer 是基于Adam算法的梯度下降算法
train_D = tf.train.AdamOptimizer(self.LR_D).minimize(
D_loss,
var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='Discriminator'))
train_G = tf.train.AdamOptimizer(self.LR_G).minimize(
G_loss,
var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='Generator'))
sess = tf.Session()
# writer = tf.summary.FileWriter('./visual',sess.graph)
sess.run(tf.global_variables_initializer())
plt.ion() # something about continuous plotting
print(type(self.all_data))
print(self.all_data)
artist_paintings, datamin, datamax = self.get_minibatch_data(
self.all_data) # real painting from artist
G_ideas = np.random.randn(self.MINIBATCH_SIZE, self.N_IDEAS)
G_paintings, pa0, Dl = sess.run(
[G_out, prob_artist0, D_loss, train_D, train_G
], # train and get results
{
G_in: G_ideas,
real_art: artist_paintings
})[:3]
dataset = FeatureCalculater.FeatureCalculater()
sql = 'truncate gan_float'
dataset.executer.execute(sql)
for step in range(100000):
artist_paintings, datamin, datamax = self.get_minibatch_data(
self.all_data) # real painting from artist
# print(artist_paintings[0])
# print(len(artist_paintings))
G_ideas = np.random.randn(self.MINIBATCH_SIZE, self.N_IDEAS)
G_paintings, pa0, Dl = sess.run(
[G_out, prob_artist0, D_loss, train_D, train_G
], # train and get results
{
G_in: G_ideas,
real_art: artist_paintings
})[:3]
if step > 99000:
# print('D',tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='Discriminator'))
# print('G',tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='Generator'))
#
lista = []
for i in range(len(G_paintings[0]) - 1):
lista.append(
abs(G_paintings[0, i] * (datamax[i] - datamin[i]) +
datamin[i]))
#
# if G_paintings[0,len(G_paintings[0])-1]>99:
# G_paintings[0,len(G_paintings[0])-1]=95
# elif G_paintings[0,len(G_paintings[0])-1]<30:
# G_paintings[0,len(G_paintings[0])-1]=36
#
lista.append(
abs(G_paintings[0, len(G_paintings[0]) - 1] *
(datamax[len(G_paintings[0]) - 1] -
datamin[len(G_paintings[0]) - 1]) +
datamin[len(G_paintings[0]) - 1]))
self.final_gan_data.append(lista.copy())
#
str1 = ''
for i in range(0, len(lista) - 1):
str1 = str1 + str(lista[i]) + ','
str1 = str1 + str(lista[len(lista) - 1])
sql = 'insert into gan_float_2 values({0})'
print(str(step) + ' ' + sql.format(str1))
print(sql.format(str1))
dataset.executer.execute(sql.format(str1))
self.final_gan_data = np.array(self.final_gan_data)
sess.close()
print('Gan结束')
def work(self):
# for i in self.nullIndexList:
for i in range(24, 25):
'''
需要先找出第i列缺失的行下标
'''
print('*************************')
defectRowIndex = []
defectRow = []
compeleteRow = []
colunm_i = []
print(i)
for j in range(self.realdata.shape[0]):
if self.realdata[j, i] == None:
defectRowIndex.append(j)
defectRow.append(self.tempdata[j])
else:
compeleteRow.append(self.tempdata[j])
colunm_i.append([self.realdata[j, i]])
colunm_i = np.array(colunm_i)
compeleteRow = np.array(compeleteRow)
compeleteRow = np.hstack((compeleteRow, colunm_i))
compeleteRow = np.delete(compeleteRow, [0, 1], axis=1)
listy = []
defectRow = np.array(defectRow)
for kk in range(defectRow.shape[0]):
# listy.append([(RD.uniform(0,5)+1)/2])
listy.append([70])
listy = np.array(listy)
defectRow = np.hstack((defectRow, listy))
defectRow = np.delete(defectRow, [0, 1], axis=1)
print('*************************')
'''
先用gan网络生成数据
'''
print('---------------------------')
ganitem = PlanB_gan.GanB()
ganitem.all_data = compeleteRow
ganitem.ART_COMPONENTS = ganitem.all_data.shape[1]
ganitem.N_IDEAS = ganitem.ART_COMPONENTS
ganitem.noise = defectRow
ganitem.MINIBATCH_SIZE = defectRow.shape[0]
ganitem.run()
# print(ganitem.final_gan_data[:,-1])
dataset = FeatureCalculater.FeatureCalculater()
liat = []
for xh in range(defectRow.shape[0]):
liat.append(
int(ganitem.final_gan_data[xh, -1]) -
int(int((ganitem.final_gan_data[xh, -1])) / 100) * 100)
# print(int(ganitem.final_gan_data[xh,-1]))
# print(int(int((ganitem.final_gan_data[xh,-1]))/100)*100)
# print(liat[xh])
sql = 'update students_final set score = {0} where student_num= "{1}"'
# print(sql.format(round(float(defectRow[xh,-1]),3),str(self.realdata[defectRowIndex[xh],0])))
# dataset.executer.execute(sql.format(int((defectRow[xh,-1]))/100*100,str(self.realdata[defectRowIndex[xh],0])))
# # print(ganitem.final_gan_data[:,-1])
print('---------------------------')
max = 10
min = 30
for xh in range(len(liat)):
if max < liat[xh]:
max = liat[xh]
# # print(max)
# print(min)
for xh in range(len(liat)):
print(liat[xh])
liat[xh] = liat[xh] - min
if liat[xh] < 0:
liat[xh] = abs(liat[xh]) + RD.uniform(0, 10)
liat[xh] = (liat[xh] / (max - min - 5)) * 100
print(liat[xh])
sql = 'update students_final set score = {0} where student_num= "{1}"'
# print(sql.format(round(float(defectRow[xh,-1]),3),str(self.realdata[defectRowIndex
dataset.executer.execute(
sql.format(liat[xh],
str(self.realdata[defectRowIndex[xh], 0])))