def Reinforced_learner(category, itr, sim, moves=100, match=1):
# In case of namer list
z = categories.index(category)
namer = namerlist[z]
# Loading data
prod = Feat_prodloader(category)
asindict = Asin_loader(category)
asinprod = Asinprod_loader(category)
prodlen = len(prod)
# Create action and probability matrixs
actionmat = np.zeros(prodlen)
actionmat[:] = 1 / prodlen
# Creating counters
count = 0
ctot = 0
cnum = 0
# Iteration loop
for i in range(itr):
s = time.time()
actions = actionmat
reward = []
statemat = []
actionlen = len(actionmat)
# Loop trough number of moves.
for m in range(0, moves):
n = 0
act = np.random.choice(len(actions), p=actions)
a = actions[act]
actionlen -= 1
actions[act] = 0
actions[np.where(actions > 0)] += a / actionlen
# loop trough number of comparisons - not included yet
while n < match:
comp = np.random.randint(prodlen)
# c = sp.cosine(prod[act],prod[comp]) # Old cosine similarity
ct = np.sum(np.dot(prod[act], prod[comp]))
cb = np.sqrt(np.sum(np.power(prod[act], 2))) * np.sqrt(
np.sum(np.power(prod[comp], 2)))
c = ct / cb # New cosine similarity
# Check for matching
if asinprod[act] in asindict:
n += 1
ctot += c
cnum += 1
# Check similarity treshold.
if c > sim:
count += 1
# else:
# continue
# print(c)
# break
e = time.time()
print(e - s)
print(ctot / cnum)
print(cnum)
print(count)
break
def Chunker(fname, spos, chunksize, size, worker, category):
catreg = re.compile(r'(\"categories\": \"%s\")' % category)
asinlist = Asin_loader(category)
start = time.time()
meta = open(fname, 'r')
meta.seek(spos)
z = 0
ap = open(worker + '_aset.json', 'w')
dp = open(worker + '_dset.json', 'w')
for i, l in enumerate(meta):
if i == chunksize:
end = time.time()
print("%8s Time Took: %8.3f" % (worker, end - start))
break
if i % size == 0:
if z > 0:
ap.write(aset)
dp.write(dset)
aset = ""
dset = ""
z = 0
else:
aset = ""
dset = ""
end = time.time()
per = i / chunksize * 100
print('%8s Progress: %6.3f - %7d/%7d Time: %8.3fs' %
(worker, per, i, chunksize, end - start))
catmatch = catreg.search(l)
desmatch = desreg.search(l)
asimatch = asireg.search(l)
if catmatch:
if asimatch.group(1) in asinlist:
if desmatch:
desstr = desmatch.group(1)
desstr = desstr.replace('\'', '')
desstr = desstr.split(', ')
z += 1
dset = dset + str(desstr) + "\n"
aset = aset + asimatch.group(1) + "\n"
else:
continue
else:
continue
else:
continue
if z > 0:
ap.write(aset)
dp.write(dset)
def Similarity_tester(category):
s = time.time()
m1 = 0
workers = []
manager = Manager()
return_dict = manager.dict()
global prod, asindict, asinprod
if category in categories:
z = categories.index(category)
namer = namerlist[z]
prod = Feat_prodloader_V2(category)
asindict = Asin_loader(category)
asinprod = Asinprod_loader_V2(category)
prodlen = len(prod)
m, n = prod.shape
print(m, n)
step = 16
spos = 0
# pool = Pool(processes = 8)
# x = pool.map()
for i in range(0, CPUs):
fname = 'worker' + str(i + 1)
epos = (i + 1) * step
if i == CPUs - 1:
epos = prodlen
worker = Process(target=runner,
args=(spos, epos, fname, m, n, return_dict))
workers.append(worker)
worker.start()
else:
worker = Process(target=runner,
args=(spos, epos, fname, m, n, return_dict))
workers.append(worker)
worker.start()
spos += step
# q = 0
for work in workers:
work.join()
workers = []
q = return_dict.values()
# print(q/prodlen)
return (np.sum(q) / (24))
else:
similaritylist = []
similarityid = []
lister = ['Appliances', 'Baby', 'Electronics', 'Home and Kitchen']
for category in lister:
prod = Feat_prodloader_V2(category)
asindict = Asin_loader(category)
asinprod = Asinprod_loader_V2(category)
prodlen = len(prod)
m, n = prod.shape
print(m, n)
m1 = 0
b1 = 0
b2 = 0
b3 = 0
b4 = 0
b5 = 0
for i in range(prodlen):
z = np.random.randint(prodlen)
supermat = np.zeros((1, n))
supermat[0, :] = prod[z]
q = 0
# supermat = np.repeat(supermat,m,axis=0)
# print(supermat.shape)
c = 1 - sp.cdist(supermat, prod,
'cosine') # Old cosine similarity
m1 += np.mean(c)
z1 = np.count_nonzero(np.where(c > 0.5))
b1 += z1 / prodlen
z2 = np.count_nonzero(np.where(c > 0.3))
b2 += z2 / prodlen
z3 = np.count_nonzero(np.where(c > 0.0))
b3 += z3 / prodlen
z4 = np.count_nonzero(np.where(c > -0.1))
b4 += z4 / prodlen
z5 = np.count_nonzero(np.where(c < -0.5))
b5 += z5 / prodlen
# if i % 500 == 0:
# e = time.time()
# print(i,(e-s))
print(category, m1 / prodlen)
print(category, b1 / prodlen)
print(category, b2 / prodlen)
print(category, b3 / prodlen)
print(category, b4 / prodlen)
print(category, b5 / prodlen)
def Train_model(epochs=15, bsize=100000, display_step=1, bpoint=0.0):
matpath = 'category/all/matrixs/'
usbpath = '/media/magnus/2FB8B2080D52768C/matrixs/'
path = 'category/all/tf/'
datpath = 'category/all/'
workers = []
s = time.time()
global valilist, valmat, asinprod, prodmat, qdict, asindict, permlist, questmat
if os.path.exists(path):
shutil.rmtree(path)
else:
os.mkdir(path)
if not os.path.exists(matpath):
os.mkdir(matpath)
if not os.path.exists(usbpath + 'prodmat.npy'):
permlist = []
permlen = 0
valilist = np.zeros(0)
valilist = valilist.astype(int)
valilen = 0
asindict = {}
asinlen = 0
qdict = {}
asinprod = np.zeros(0)
prodmat = np.zeros((0, 300))
questmat = np.zeros((0, 300))
valmat = np.zeros((0, 300))
print('--- Loading Data ---')
for category in categories:
z = categories.index(category)
namer = namerlist[z]
loc = namer
catlist = Perm_loader(category)
catlist = np.array(catlist) + permlen
permlist.extend(np.ndarray.tolist(catlist))
permlen += len(catlist)
catvali = Devl_loader(category)
catvali = catvali + valilen
valilist = np.append(valilist, catvali.astype(int))
valilen += len(catvali)
catdict = Asin_loader(category)
catlen = 0
for key in catdict.keys():
val = catdict[key]
val = np.array(val) + asinlen
catlen += len(val)
catdict[key] = np.ndarray.tolist(val)
asindict.update(catdict)
intdict = Asin_loader(category, 1)
for key in intdict.keys():
nkey = int(key) + asinlen
qdict[str(int(nkey))] = intdict[key]
asinlen += catlen
catprod = Asinprod_loader_V2(category)
asinprod = np.append(asinprod, catprod)
catprodmat = Feat_prodloader_V2(category)
prodmat = np.append(prodmat, catprodmat, axis=0)
print(prodmat.shape)
catquestmat = Feat_questloader_V2(category, 'data')
questmat = np.append(questmat, catquestmat, axis=0)
print(questmat.shape)
catvalmat = Feat_questloader_V2(category, 'devl')
valmat = np.append(valmat, catvalmat, axis=0)
np.save(usbpath + 'valmat', valmat)
np.save(usbpath + 'prodmat', prodmat)
np.save(usbpath + 'questmat', questmat)
np.save(usbpath + 'asinprod', asinprod)
np.save(usbpath + 'valilist', valilist)
np.save(usbpath + 'permlist', permlist)
with open(usbpath + 'asindict.json', 'w') as fp:
json.dump(asindict, fp)
with open(usbpath + 'qdict.json', 'w') as fp:
json.dump(qdict, fp)
# Release all variable
valmat = prodmat = questmat = asinprod = valilist = asindict = qdict = 0
catvalmat = catquestmat = catprodmat = intdict = catdict = catvali = catlist = 0
# m,n = prodmat.shape
# p,q = questmat.shape
if not os.path.exists(usbpath + 'Xmat.npy'):
valilist = np.load(usbpath + 'valilist.npy')
valmat = np.load(usbpath + 'valmat.npy')
asinprod = np.load(usbpath + 'asinprod.npy')
prodmat = np.load(usbpath + 'prodmat.npy')
with open(usbpath + 'qdict.json', 'r') as fp:
qdict = json.load(fp)
with open(usbpath + 'asindict.json', 'r') as fp:
asindict = json.load(fp)
valnum = valmat.shape[0]
qlist = list(qdict.keys())
alist = list(asindict.keys())
print('--- Creating Validation Matrix ---')
s = time.time()
spos = 0
step = round(len(valilist) / CPUs)
for i in range(0, CPUs):
fname = 'worker' + str(i + 1)
epos = (i + 1) * step
if i == CPUs - 1:
epos = len(valilist)
worker = Process(target=Valimat_creator,
args=(spos, epos, fname, bpoint, usbpath))
workers.append(worker)
worker.start()
else:
worker = Process(target=Valimat_creator,
args=(spos, epos, fname, bpoint, usbpath))
workers.append(worker)
worker.start()
spos += step
for work in workers:
work.join()
workers = []
Xmat, Ymat = Valcombiner(CPUs, usbpath)
np.save(usbpath + 'Xmat', Xmat)
np.save(usbpath + 'Ymat', Ymat)
else:
Xmat = np.load(usbpath + 'Xmat.npy')
Ymat = np.load(usbpath + 'Ymat.npy')
testmax = np.argmax(Xmat)
testmin = np.argmin(Xmat)
testmax = np.unravel_index(testmax, Xmat.shape)
testmin = np.unravel_index(testmin, Xmat.shape)
print(Xmat[testmax])
print(Xmat[testmin])
permute = np.random.permutation(Xmat.shape[0])
Xmat = Xmat[permute]
Ymat = Ymat[permute]
print(Xmat.shape)
e = time.time()
print('Validation matrix shape:', Xmat.shape,
'Time took: %8.3fs' % (e - s))
t_zero_count = 0
t_one_count = 0
for f in Ymat:
if f[1] == 1.:
t_one_count += 1
else:
t_zero_count += 1
total = t_zero_count + t_one_count
uid = 0
labelmat = np.zeros((1, 2))
resmat = np.zeros((1, n_input))
count = 0
total_parameters = 0
#iterating over all variables
for variable in tf.trainable_variables():
local_parameters = 1
shape = variable.get_shape()
#getting shape of a variable
for i in shape:
local_parameters *= i.value
#mutiplying dimension values
total_parameters += local_parameters
print('Number of params in model ' + str(total_parameters))
if not os.path.exists(datpath + 'c1' + 'res.npy'):
if not os.path.exists(datpath + str(bsize - 1) + 'res.npy'):
permlist = np.load(usbpath + 'permlist.npy')
permlist = permlist.tolist()
asinprod = np.load(usbpath + 'asinprod.npy')
questmat = np.load(usbpath + 'questmat.npy')
prodmat = np.load(usbpath + 'prodmat.npy')
with open(usbpath + 'asindict.json') as fp:
asindict = json.load(fp)
with open(usbpath + 'qdict.json') as fp:
qdict = json.load(fp)
spos = 0
step = round(len(asindict) / CPUs)
for i in range(0, CPUs):
fname = 'worker' + str(i + 1)
epos = (i + 1) * step
if i == CPUs - 1:
epos = len(asindict)
worker = Process(target=Train_creator,
args=(spos, epos, fname, bsize, bpoint,
datpath))
workers.append(worker)
worker.start()
else:
if i == 0:
worker = Process(target=Train_creator,
args=(spos, epos, fname, bsize,
bpoint, datpath))
workers.append(worker)
worker.start()
spos += step
for work in workers:
work.join()
workers = []
permlist = 0
asinprod = 0
questmat = 0
prodmat = 0
asindict = 0
qdict = 0
Rescombiner(datpath, 25000, maxint=500000, model=nfeat * 4)
with tf.Session() as sess:
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter(path, sess.graph)
trainwriter = tf.summary.FileWriter(path + 'train/')
valwriter = tf.summary.FileWriter(path + 'val/')
sess.run(init)
start = time.time()
for e in range(epochs):
# Loop trough all products that have questions
qwerty = 0
zero_count = 0
one_count = 0
for bname in range(1, 2):
if os.path.exists(datpath + 'c' + str(bname) + 'res.npy'):
resmat = np.load(datpath + 'c' + str(bname) + 'res.npy')
resmat = resmat[0:1250, :]
labelmat = np.load(datpath + 'c' + str(bname) +
'label.npy')
if resmat.shape[0] != labelmat.shape[0]:
labelmat = labelmat[:resmat.shape[0], :]
qwerty += resmat.shape[0] * resmat.shape[1]
_, c, output, sumsum = sess.run(
[train_op, loss_op, pred, merged],
feed_dict={
X: resmat,
Y: labelmat
})
uid += 1
trainwriter.add_summary(sumsum, uid)
else:
break
end = time.time()
if e % display_step == 0:
print("Epoch:", '%4d' % (e + 1), "cost = {:.9f}".format(c),
"Time took: %8.2f" % (end - start))
output = sess.run([accuracy, pred, merged],
feed_dict={
X: Xmat,
Y: Ymat
})
valwriter.add_summary(output[2], e)
# print('Validations results')
print("Accuracy", output[0])
print("Input Parameters", qwerty)
zero_count = 0
one_count = 0
TP = 0
FP = 0
TN = 0
FN = 0
for j in output[1]:
if np.argmax(j) == 1:
one_count += 1
else:
zero_count += 1
print('zero count =%7d/%7d' % (zero_count, t_zero_count))
print('one count =%7d/%7d' % (one_count, t_one_count))
for i in range(0, total):
if np.argmax(Ymat[i, :]) == 1 and np.argmax(output[1][i]) == 1:
TP += 1
elif np.argmax(Ymat[i, :]) == 1 and np.argmax(
output[1][i]) == 0:
FN += 1
elif np.argmax(Ymat[i, :]) == 0 and np.argmax(
output[1][i]) == 0:
TN += 1
else:
FP += 1
print('TP = %7d/%7d , FP = %7d/%7d' %
(TP, t_one_count, FP, t_zero_count))
print('TN = %7d/%7d , FN = %7d/%7d' %
(TN, t_zero_count, FN, t_one_count))
if c < 0.001:
# count += 1
# if count == 168:
break
def Train_model(category, epochs=15, bsize=100000, display_step=1):
print('--- Loading Data ---')
z = categories.index(category)
namer = namerlist[z]
path = 'category/' + namer + '/tf/'
matpath = 'category/' + namer + '/matrixs/'
loc = namer
if os.path.exists(path):
shutil.rmtree(path)
else:
os.mkdir(path)
if not os.path.exists(matpath):
os.mkdir(matpath)
permlist = Perm_loader(category)
permlist = list(permlist)
valilist = Devl_loader(category)
asindict = Asin_loader(category)
qdict = Asin_loader(category, 1)
asinprod = Asinprod_loader(category)
prodmat = Feat_prodloader(category)
questmat = Feat_questloader(category, 'data')
valmat = Feat_questloader(category, 'devl')
m, n = prodmat.shape
p, q = questmat.shape
count = 0
t = 0
z = 0
total = len(list(asindict.keys()))
Xmat = np.zeros((valmat.shape[0] * 2, 2 * valmat.shape[1]))
Ymat = np.zeros((valmat.shape[0] * 2, 2))
print('--- Creating Validation Matrix ---')
s = time.time()
for v, i in enumerate(valilist):
e = qdict[str(i)]
try:
xpos = np.argwhere(asinprod == e)
Xmat[v, :nfeat] = prodmat[xpos, :]
Xmat[v, nfeat:] = valmat[v, :]
Ymat[v, 1] = 1
except:
Xmat[v, :nfeat] = prodmat[np.random.randint(m), :]
Xmat[v, nfeat:] = valmat[v, :]
Ymat[v, 0] = 1
if z % 100 == 0:
e = time.time()
print('Progress: %6.3f - %7d/%7d Time: %8.3fs' %
(z / valmat.shape[0] * 100, z, valmat.shape[0], e - s))
z += 1
z = 0
qlist = list(qdict.keys())
qlist.append(list(asindict.keys()))
print('Fill with negative')
for d in range(v, Xmat.shape[0]):
select = np.random.randint(m)
prodasin = asinprod[select]
while prodasin in qlist:
select = np.random.randint(m)
prodasin = asinprod[select]
Xmat[d, :nfeat] = prodmat[select, :]
Xmat[d, nfeat:] = questmat[np.random.randint(p), :]
Ymat[d, 0] = 1
if z % 100 == 0:
e = time.time()
print('Progress: %6.3f - %7d/%7d Time: %8.3fs' %
(z / valmat.shape[0] * 100, z, valmat.shape[0], e - s))
z += 1
e = time.time()
print('Validation matrix shape:', Xmat.shape,
'Time took: %8.3fs' % (e - s))
t_zero_count = 0
t_one_count = 0
for f in Ymat:
if f[1] == 1.:
t_one_count += 1
else:
t_zero_count += 1
total = t_zero_count + t_one_count
t_zero_count += 1
t_one_count -= 1
uid = 0
labelmat = np.zeros((1, 2))
resmat = np.zeros((1, n_input))
count = 0
with tf.Session() as sess:
print(path)
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter(path, sess.graph)
trainwriter = tf.summary.FileWriter(path + 'train/')
valwriter = tf.summary.FileWriter(path + 'val/')
sess.run(init)
start = time.time()
# Epoch Loop
plist = asinprod
qtlist = []
keepresmat = np.zeros((1, 2 * nfeat))
keeplabelmat = np.zeros((1, 2))
nfiles = []
for e in range(epochs):
# Loop trough all products that have questions
if e == 0:
for prog, i in enumerate(asindict.keys()):
d = 0
# Check wether product exists in the dataset
if i in asinprod:
pos = np.argwhere(asinprod == i)
if i in plist:
plist = np.delete(plist, np.where(asinprod == i))
labellist = np.zeros((1, 2))
tempmat = np.zeros((1, 2 * nfeat))
# For each true question
tlist = []
for t in asindict[i]:
tempmat[0, :nfeat] = prodmat[pos, :]
# Ensure it is in the trainingset.
if t in permlist:
labellist[0, 1] = 1
labelmat = np.append(labelmat,
labellist,
axis=0)
d += 1
tpos = permlist.index(t)
tlist.append(tpos)
test = questmat[tpos, :]
tempmat[0, nfeat:] = questmat[tpos, :]
resmat = np.append(resmat, tempmat, axis=0)
labellist = np.zeros((1, 2))
if tlist not in qtlist:
qtlist.append(tlist)
# Create equally amounts of negative samples
for fill in range(d + 1):
tempmat[0, :nfeat] = prodmat[pos, :]
select = np.random.randint(p)
while select in tlist:
select = np.random.randint(p)
tempmat[0, nfeat:] = questmat[select, :]
resmat = np.append(resmat, tempmat, axis=0)
labellist[0, 0] = 1
labelmat = np.append(labelmat, labellist, axis=0)
for fill in range(2 * d):
pselect = np.random.randint(len(plist))
while pselect in qlist:
pselect = np.random.randint(len(plist))
qselect = np.random.randint(p)
tempmat[0, :nfeat] = prodmat[pselect, :]
tempmat[0, nfeat:] = questmat[qselect, :]
resmat = np.append(resmat, tempmat, axis=0)
labellist[0, 0] = 1
labelmat = np.append(labelmat, labellist, axis=0)
if prog % bsize == bsize - 1:
nfiles.append(prog)
resmat = np.delete(resmat, 0, 0)
labelmat = np.delete(labelmat, 0, 0)
np.save(matpath + str(prog) + 'res', resmat)
np.save(matpath + str(prog) + 'label', labelmat)
resmat = np.zeros((1, 2 * nfeat))
labelmat = np.zeros((1, 2))
_, c, sumsum = sess.run([train_op, loss_op, merged],
feed_dict={
X: resmat,
Y: labelmat
})
resmat = np.zeros((1, n_input))
labelmat = np.zeros((1, 2))
uid += 1
trainwriter.add_summary(sumsum, uid)
# if prog > 16:
# break
if resmat.shape[0] > 1:
nfiles.append(prog)
resmat = np.delete(resmat, 0, 0)
labelmat = np.delete(labelmat, 0, 0)
np.save(matpath + str(prog) + 'res', resmat)
np.save(matpath + str(prog) + 'label', labelmat)
resmat = np.zeros((1, 2 * nfeat))
labelmat = np.zeros((1, 2))
_, c, sumsum = sess.run([train_op, loss_op, merged],
feed_dict={
X: resmat,
Y: labelmat
})
resmat = np.zeros((1, n_input))
labelmat = np.zeros((1, 2))
uid += 1
trainwriter.add_summary(sumsum, uid)
qwerty = 0
else:
qwerty = 0
for bname in nfiles:
resmat = np.load(matpath + str(bname) + 'res.npy')
labelmat = np.load(matpath + str(bname) + 'label.npy')
qwerty += resmat.shape[0] * resmat.shape[1]
_, c, sumsum = sess.run([train_op, loss_op, merged],
feed_dict={
X: resmat,
Y: labelmat
})
uid += 1
trainwriter.add_summary(sumsum, uid)
end = time.time()
if e % display_step == 0:
print("Epoch:", '%4d' % (e + 1), "cost = {:.9f}".format(c),
"Time took: %8.2f" % (end - start))
output = sess.run([accuracy, pred, merged],
feed_dict={
X: Xmat,
Y: Ymat
})
valwriter.add_summary(output[2], e)
print("Accuracy", output[0])
print("Input Parameters", qwerty)
zero_count = 0
one_count = 0
TP = 0
FP = 0
TN = 0
FN = 0
zcount = 0
for j in output[1]:
if np.argmax(j) == 1:
one_count += 1
else:
zero_count += 1
print('zero count =%7d/%7d' % (zero_count, t_zero_count))
print('one count =%7d/%7d' % (one_count, t_one_count))
for i in range(0, total):
if np.argmax(Ymat[i, :]) == 1 and np.argmax(output[1][i]) == 1:
TP += 1
elif np.argmax(Ymat[i, :]) == 1 and np.argmax(
output[1][i]) == 0:
FN += 1
elif np.argmax(Ymat[i, :]) == 0 and np.argmax(
output[1][i]) == 0:
TN += 1
else:
FP += 1
print('TP = %7d/%7d , FP = %7d/%7d' %
(TP, t_one_count, FP, t_zero_count))
print('TN = %7d/%7d , FN = %7d/%7d' %
(TN, t_zero_count, FN, t_one_count))
if c < 0.001:
count += 1
if count == 168:
break
else:
count = 0
if os.path.exists(matpath):
shutil.rmtree(matpath)
save_path = saver.save(sess, path + loc)
def Train_model(category, epochs=15, bsize=100000, display_step=1, bpoint=0.0):
print('--- Loading Data ---')
z = categories.index(category)
namer = namerlist[z]
path = 'category/' + namer + '/tf/'
matpath = 'category/' + namer + '/matrixs/'
loc = namer
if os.path.exists(path):
shutil.rmtree(path)
else:
os.mkdir(path)
if not os.path.exists(matpath):
os.mkdir(matpath)
permlist = Perm_loader(category)
permlist = list(permlist)
valilist = Devl_loader(category)
asindict = Asin_loader(category)
qdict = Asin_loader(category, 1)
asinprod = Asinprod_loader(category)
prodmat = Feat_prodloader(category)
questmat = Feat_questloader(category, 'data')
# questmat = Feat_questloader_V2(category,'data')
valmat = Feat_questloader(category, 'devl')
m, n = prodmat.shape
p, q = questmat.shape
v = 0
t = 0
z = 0
total = len(list(asindict.keys()))
valnum = valmat.shape[0]
Xmat = np.zeros((valnum * 2, 4 * valmat.shape[1]))
Ymat = np.zeros((valnum * 2, 2))
xlist = []
qlist = list(qdict.keys())
alist = list(asindict.keys())
# print(len(qlist))
# print(len(valilist))
# print(qdict.keys())
# print(kage)
print('--- Creating Validation Matrix ---')
s = time.time()
# print(asinprod)
for i in (valilist):
e = qdict[str(i)]
if e in asinprod:
xpos = np.argwhere(asinprod == e)
Xmat[v, :nfeat] = prodmat[xpos, :]
Xmat[v, nfeat:2 * nfeat] = valmat[z, :]
Xmat[v,
2 * nfeat:3 * nfeat] = np.multiply(Xmat[v, :nfeat],
Xmat[v, nfeat:2 * nfeat])
Xmat[v,
3 * nfeat:4 * nfeat] = np.subtract(Xmat[v, :nfeat],
Xmat[v, nfeat:2 * nfeat])
Ymat[v, 1] = 1
xlist.append(e)
v += 1
Xmat[v, :nfeat] = prodmat[xpos, :]
c = 1
while c > bpoint:
select = np.random.randint(len(valilist))
sel = valilist[select]
sel = qdict[str(sel)]
if sel in asinprod:
selpos = np.argwhere(asinprod == sel)
p1 = prodmat[int(xpos)]
p2 = prodmat[int(selpos)]
ct = np.sum(np.dot(p1, p2))
cb = np.sqrt(np.sum(np.power(p1, 2))) * np.sqrt(
np.sum(np.power(p2, 2)))
c = ct / cb
Xmat[v, nfeat:2 * nfeat] = valmat[select, :]
Xmat[v,
2 * nfeat:3 * nfeat] = np.multiply(Xmat[v, :nfeat],
Xmat[v, nfeat:2 * nfeat])
Xmat[v,
3 * nfeat:4 * nfeat] = np.subtract(Xmat[v, :nfeat],
Xmat[v, nfeat:2 * nfeat])
Ymat[v, 0] = 1
v += 1
else:
Xmat = np.delete(Xmat, v, 0)
Xmat = np.delete(Xmat, v, 0)
Ymat = np.delete(Ymat, v, 0)
Ymat = np.delete(Ymat, v, 0)
continue
# Xmat[v,:nfeat] = prodmat[np.random.randint(m),:]
# Xmat[v,nfeat:2*nfeat] = valmat[v,:]
# Ymat[v,0] = 1
# Xmat[v,2*nfeat:3*nfeat] = np.multiply(Xmat[v,:nfeat],Xmat[v,nfeat:2*nfeat])
# Xmat[v,3*nfeat:4*nfeat] = np.subtract(Xmat[v,:nfeat],Xmat[v,nfeat:2*nfeat])
if z % 1000 == 0:
e = time.time()
print('Progress: %6.3f - %7d/%7d Time: %8.3fs' %
(z / valmat.shape[0] * 100, z, valmat.shape[0], e - s))
z += 1
z = 0
print('Fill with negative')
testmax = np.argmax(Xmat)
testmin = np.argmin(Xmat)
testmax = np.unravel_index(testmax, Xmat.shape)
testmin = np.unravel_index(testmin, Xmat.shape)
print(Xmat[testmax])
print(Xmat[testmin])
permute = np.random.permutation(Xmat.shape[0])
Xmat = Xmat[permute]
Ymat = Ymat[permute]
print(Xmat.shape)
# for d in range(v,Xmat.shape[0]):
# select = np.random.randint(len(xlist))
# prodasin = np.argwhere(asinprod == xlist[select])
# while select in asinprod[qlist]:
# select = np.random.randint(len(xlist))
# prodasin = np.argwhere(asinprod == xlist[select])
# Xmat[d,:nfeat] = prodmat[select,:]
# Xmat[d,nfeat:2*nfeat] = questmat[np.random.randint(p),:]
# Xmat[d,2*nfeat:3*nfeat] = np.multiply(Xmat[d,:nfeat],Xmat[d,nfeat:2*nfeat])
# Xmat[d,3*nfeat:4*nfeat] = np.subtract(Xmat[d,:nfeat],Xmat[d,nfeat:2*nfeat])
# Ymat[d,0] = 1
# if z % 1000 == 0:
# e = time.time()
# print('Progress: %6.3f - %7d/%7d Time: %8.3fs'%(z/valmat.shape[0]*100,z,valmat.shape[0],e-s))
# z += 1
e = time.time()
print('Validation matrix shape:', Xmat.shape,
'Time took: %8.3fs' % (e - s))
t_zero_count = 0
t_one_count = 0
for f in Ymat:
if f[1] == 1.:
t_one_count += 1
else:
t_zero_count += 1
total = t_zero_count + t_one_count
uid = 0
labelmat = np.zeros((1, 2))
resmat = np.zeros((1, n_input))
count = 0
total_parameters = 0
#iterating over all variables
for variable in tf.trainable_variables():
local_parameters = 1
shape = variable.get_shape()
#getting shape of a variable
for i in shape:
local_parameters *= i.value
#mutiplying dimension values
total_parameters += local_parameters
print('Number of params in model ' + str(total_parameters))
with tf.Session() as sess:
print(path)
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter(path, sess.graph)
trainwriter = tf.summary.FileWriter(path + 'train/')
valwriter = tf.summary.FileWriter(path + 'val/')
sess.run(init)
start = time.time()
# Epoch Loop
plist = asinprod
qtlist = []
keepresmat = np.zeros((1, 4 * nfeat))
keeplabelmat = np.zeros((1, 2))
nfiles = []
zerolabelmat = 0
onelabelmat = 0
for e in range(epochs):
# Loop trough all products that have questions
if e == 0:
if os.path.exists(matpath + '/' + str(bsize - 1) + 'res.npy'):
continue
for prog, i in enumerate(asindict.keys()):
d = 0
# Check wether product exists in the dataset
if i in asinprod:
pos = np.argwhere(asinprod == i)
if i in plist:
plist = np.delete(plist, np.where(asinprod == i))
labellist = np.zeros((1, 2))
tempmat = np.zeros((1, 4 * nfeat))
# For each true question
tlist = []
for t in asindict[i]:
tempmat[0, :nfeat] = prodmat[pos, :]
# Ensure it is in the trainingset.
if t in permlist:
labellist[0, 1] = 1
labelmat = np.append(labelmat,
labellist,
axis=0)
d += 1
tpos = permlist.index(t)
tlist.append(tpos)
test = questmat[tpos, :]
tempmat[0, nfeat:2 * nfeat] = questmat[tpos, :]
tempmat[0, 2 * nfeat:3 * nfeat] = np.multiply(
tempmat[0, :nfeat],
tempmat[0, nfeat:2 * nfeat])
tempmat[0, 3 * nfeat:4 * nfeat] = np.subtract(
tempmat[0, :nfeat],
tempmat[0, nfeat:2 * nfeat])
resmat = np.append(resmat, tempmat, axis=0)
labellist = np.zeros((1, 2))
if tlist not in qtlist:
qtlist.append(tlist)
# Create equally amounts of negative samples
for fill in range(d + 1):
tempmat[0, :nfeat] = prodmat[pos, :]
d = 1
# select = np.random.randint(p)
# while select in tlist:
# select = np.random.randint(p)
while d > bpoint:
select = np.random.randint(len(permlist))
# sel = plist[select]
sel = qdict[str(select)]
if sel in asinprod:
selpos = np.argwhere(asinprod == sel)
p1 = prodmat[int(pos)]
p2 = prodmat[int(selpos)]
ct = np.sum(np.dot(p1, p2))
cb = np.sqrt(np.sum(np.power(
p1, 2))) * np.sqrt(
np.sum(np.power(p2, 2)))
d = ct / cb
# print(d)
tempmat[0, nfeat:2 * nfeat] = questmat[select, :]
tempmat[0, 2 * nfeat:3 * nfeat] = np.multiply(
tempmat[0, :nfeat], tempmat[0,
nfeat:2 * nfeat])
tempmat[0, 3 * nfeat:4 * nfeat] = np.subtract(
tempmat[0, :nfeat], tempmat[0,
nfeat:2 * nfeat])
resmat = np.append(resmat, tempmat, axis=0)
labellist[0, 0] = 1
labelmat = np.append(labelmat, labellist, axis=0)
# for fill in range(2*d):
# pselect = np.random.randint(len(plist))
# while pselect in qlist:
# pselect = np.random.randint(len(plist))
# qselect = np.random.randint(p)
# tempmat[0,:nfeat] = prodmat[pselect,:]
# tempmat[0,nfeat:2*nfeat] = questmat[qselect,:]
# tempmat[0,2*nfeat:3*nfeat] = np.multiply(tempmat[0,:nfeat],tempmat[0,nfeat:2*nfeat])
# tempmat[0,3*nfeat:4*nfeat] = np.subtract(tempmat[0,:nfeat],tempmat[0,nfeat:2*nfeat])
# resmat = np.append(resmat,tempmat,axis=0)
# labellist[0,0] = 1
# labelmat = np.append(labelmat,labellist,axis=0)
if prog % bsize == bsize - 1:
testmax = np.argmax(resmat)
testmin = np.argmin(resmat)
testmax = np.unravel_index(testmax, resmat.shape)
testmin = np.unravel_index(testmin, resmat.shape)
print(resmat[testmax])
print(resmat[testmin])
print(resmat.shape)
nfiles.append(prog)
resmat = np.delete(resmat, 0, 0)
labelmat = np.delete(labelmat, 0, 0)
permute = np.random.permutation(resmat.shape[0])
resmat = resmat[permute]
labelmat = labelmat[permute]
# zerolabelmat += np.sum(labelmat[:,0])
# onelabelmat += np.sum(labelmat[:,1])
np.save(matpath + str(prog) + 'res', resmat)
np.save(matpath + str(prog) + 'label', labelmat)
_, c, sumsum = sess.run([train_op, loss_op, merged],
feed_dict={
X: resmat,
Y: labelmat
})
resmat = np.zeros((1, n_input))
labelmat = np.zeros((1, 2))
uid += 1
trainwriter.add_summary(sumsum, uid)
resmat = np.zeros((1, 4 * nfeat))
labelmat = np.zeros((1, 2))
# if prog % bsize == bsize-1:
# break
if resmat.shape[0] > 1:
testmax = np.argmax(resmat)
testmin = np.argmin(resmat)
testmax = np.unravel_index(testmax, resmat.shape)
testmin = np.unravel_index(testmin, resmat.shape)
print(resmat[testmax])
print(resmat[testmin])
print(resmat.shape)
nfiles.append(prog)
resmat = np.delete(resmat, 0, 0)
labelmat = np.delete(labelmat, 0, 0)
permute = np.random.permutation(resmat.shape[0])
resmat = resmat[permute]
labelmat = labelmat[permute]
np.save(matpath + str(prog) + 'res', resmat)
np.save(matpath + str(prog) + 'label', labelmat)
_, c, sumsum = sess.run([train_op, loss_op, merged],
feed_dict={
X: resmat,
Y: labelmat
})
uid += 1
trainwriter.add_summary(sumsum, uid)
resmat = np.zeros((1, 4 * nfeat))
labelmat = np.zeros((1, 2))
qwerty = 0
else:
qwerty = 0
zero_count = 0
one_count = 0
if len(nfiles) > 1:
for bname in nfiles:
resmat = np.load(matpath + str(bname) + 'res.npy')
labelmat = np.load(matpath + str(bname) + 'label.npy')
qwerty += resmat.shape[0] * resmat.shape[1]
_, c, output, sumsum = sess.run(
[train_op, loss_op, pred, merged],
feed_dict={
X: resmat,
Y: labelmat
})
uid += 1
trainwriter.add_summary(sumsum, uid)
else:
for bname in range(bsize, 100000, bsize):
if os.path.exists(matpath + '/' + str(bname - 1) +
'res.npy'):
resmat = np.load(matpath + str(bname - 1) +
'res.npy')
labelmat = np.load(matpath + str(bname - 1) +
'label.npy')
qwerty += resmat.shape[0] * resmat.shape[1]
_, c, output, sumsum = sess.run(
[train_op, loss_op, pred, merged],
feed_dict={
X: resmat,
Y: labelmat
})
uid += 1
trainwriter.add_summary(sumsum, uid)
else:
break
end = time.time()
if e % display_step == 0:
print("Epoch:", '%4d' % (e), "cost = {:.9f}".format(c),
"Time took: %8.2f" % (end - start))
output = sess.run([accuracy, pred, merged],
feed_dict={
X: Xmat,
Y: Ymat
})
valwriter.add_summary(output[2], e)
# print('Validations results')
print("Accuracy", output[0])
print("Input Parameters", qwerty)
zero_count = 0
one_count = 0
TP = 0
FP = 0
TN = 0
FN = 0
for j in output[1]:
if np.argmax(j) == 1:
one_count += 1
else:
zero_count += 1
print('zero count =%7d/%7d' % (zero_count, t_zero_count))
print('one count =%7d/%7d' % (one_count, t_one_count))
for i in range(0, total):
if np.argmax(Ymat[i, :]) == 1 and np.argmax(output[1][i]) == 1:
TP += 1
elif np.argmax(Ymat[i, :]) == 1 and np.argmax(
output[1][i]) == 0:
FN += 1
elif np.argmax(Ymat[i, :]) == 0 and np.argmax(
output[1][i]) == 0:
TN += 1
else:
FP += 1
print('TP = %7d/%7d , FP = %7d/%7d' %
(TP, t_one_count, FP, t_zero_count))
print('TN = %7d/%7d , FN = %7d/%7d' %
(TN, t_zero_count, FN, t_one_count))
if c < 0.001:
count += 1
if count == 168:
break
else:
count = 0