def seqLearnCM(cate,
ratio,
r,
numCenterList,
alphaList,
QList,
KList,
lenPara,
dbname,
iter='ZZZ',
mode='ZZZ'):
# setting and starting ppservers ...
ppservers = ()
ncpus = 2
job_server = pp.Server(ncpus, ppservers=ppservers)
print str(datetime.datetime.now()
)[:19] + " Starting python parallel with", job_server.get_ncpus(
), "workers. "
paraPerm = list(itertools.product(numCenterList, alphaList, QList, KList))
path4 = util.getPath2(dbname + "_ycobjs_r" + str(r) + ".cpkl")
if os.path.exists(path4):
fnResults = util.getPath2(dbname + "_results_r" + str(r) + "_" + iter +
"_" + mode + ".npy")
fnRBFobjs = util.getPath2(dbname + "_results_r" + str(r) + "_" + iter +
"_" + mode + ".cpkl")
if not os.path.exists(fnResults):
ctrainM, ctestM, ytrainM, ytestM = dataLoader3.cpklload(path4)
# ctrainM, ctestM = numpy.mat(ctrain).T, numpy.mat(ctest).T
results = numpy.mat(numpy.zeros((0, lenPara)))
objsRBF = []
jobs = [
job_server.submit(
classifyCM,
(ratio, r, para[0], para[1], para[2], para[3], lenPara,
ytrainM, ytestM, ctrainM, ctestM, dbname), (),
("numpy", "dataProcess", "RBFNetwork", "dataLoader3",
"datetime", "util")) for para in paraPerm
]
for job in jobs:
cnm = job()
result, objs = cnm[0], cnm[1]
results = numpy.concatenate((results, result), axis=0)
objsRBF = objsRBF + objs
print "\n"
if not (numpy.shape(results)[0] == 0 or len(objsRBF) == 0):
dataLoader3.saveMatrix([results], [fnResults])
dataLoader3.saveObjects([objsRBF], [fnRBFobjs])
else:
util.infoErr([fnResults, fnRBFobjs])
else:
util.exPath([fnResults, fnRBFobjs])
else:
util.noPath([path4])
def reload(stipfile, actions, dbname):
path1 = util.getPath(dbname + ".cpkl")
if not os.path.exists(path1):
print str(datetime.datetime.now())[:19] + " loading datasets ... "
flags, inputSet, targetSet = dataLoader3.loadDataSet(
stipfile, actions, path1)
else:
util.exPath([path1])
flags, inputSet, targetSet = dataLoader3.cpklload(path1)
return flags, inputSet, targetSet
def seqLearnTest(cate,
ratio,
r,
numCenterList,
alphaList,
QList,
KList,
lenPara,
dbname,
iter='ZZZ',
mode='ZZZ'):
paraPerm = list(itertools.product(numCenterList, alphaList, QList, KList))
path4 = util.getPath2(dbname + "_ycobjs_r" + str(r) + ".cpkl")
if os.path.exists(path4):
fnResults = util.getPath2(dbname + "_results_r" + str(r) + "_" + iter +
"_" + mode + ".npy")
# fnRBFobjs = util.getPath2(dbname + "_results_r" + str(r) + "_" + iter + ".cpkl")
if not os.path.exists(fnResults):
ctrainM, ctestM, ytrainM, ytestM = dataLoader3.cpklload(path4)
# ctrainM, ctestM = numpy.mat(ctrain).T, numpy.mat(ctest).T
results = numpy.mat(numpy.zeros((0, lenPara)))
objsRBF = []
for para in paraPerm:
result = classify(ratio, r, para[0], para[1], para[2], para[3],
lenPara, ytrainM, ytestM, ctrainM, ctestM,
dbname)
# cnm = job()
# result, obj = cnm[0], cnm[1]
results = numpy.concatenate((results, result), axis=0)
# objsRBF = objsRBF + obj
# results = numpy.concatenate((results, job()[0]), axis = 0)
# objsRBF = objsRBF + job()[1]
print "\n"
if not numpy.shape(results)[0] == 0:
dataLoader3.saveMatrix([results], [fnResults])
# dataLoader3.saveObjects([objsRBF], [fnRBFobjs])
else:
util.infoErr([fnResults])
else:
util.exPath([fnResults])
else:
util.noPath([path4])
def cluster1(r, KList, actions, stipfile, dbname):
path4 = util.getPath(dbname + "_ycobjs_r" + str(r) + ".cpkl")
'''bitter of changing api frequently'''
'''*********************************************************'''
splitlist, xtrain, xtest, xtrainM, xtestM, ytrainM, ytestM, xrandMX = split1(
r, actions, stipfile, dbname)
if not os.path.exists(path4):
print str(datetime.datetime.now()
)[:19] + " counting datasets ..., round: " + str(r)
ctrain, ctest = splitter.count(xtrain, xtest)
ycobjs = [ctrain, ctest, ytrainM, ytestM]
dataLoader3.cpklsave(ycobjs, path4)
else:
util.exPath([path4])
'''************************************************************************************************'''
ctrain, ctest, ytrainM, ytestM = dataLoader3.cpklload(path4)
if numpy.shape(xtrainM)[0] > 100000:
'''idx = idxrandom(numpy.shape(xtrainM)[0], 10000)'''
idx = splitter.idxrandom(numpy.shape(xtrainM)[0], 100000)
xrandMX2 = xtrainM[idx, :]
else:
xrandMX2 = xtrainM
print str(
datetime.datetime.now())[:19] + " The shape of xrandMX2 is : " + str(
numpy.shape(xrandMX2))
'''************************************************************************************************'''
for k in KList:
path3 = util.getPath(dbname + "_bow" + "_r" + str(r) + "_k" + str(k) +
".cpkl")
if not os.path.exists(path3):
cluster.kMeans(r, k, ctrain, ctest, xtrainM, xtestM, xrandMX2,
path3)
else:
util.exPath([path3])
'''bitter of changing api frequetly'''
def cluster2(r, KList, actions, stipfile, dbname):
path4 = util.getPath(dbname + "_ycobjs_r" + str(r) + ".cpkl")
'''bitter of changing api frequetly'''
'''*********************************************************'''
splitlist, ctrain, ctest, ctrainM, ctestM, xtrainM, xtestM, ytrainM, ytestM, xrandMX = split2(
r, actions, stipfile, dbname)
if not os.path.exists(path4):
ycobjs = [ctrainM, ctestM, ytrainM, ytestM]
dataLoader3.cpklsave(ycobjs, path4)
else:
util.exPath([path4])
for k in KList:
path3 = util.getPath(dbname + "_bow" + "_r" + str(r) + "_k" + str(k) +
".cpkl")
if not os.path.exists(path3):
cluster.kMeans(r, k, ctrain, ctest, xtrainM, xtestM, xrandMX,
path3)
else:
util.exPath([path3])
'''bitter of changing api frequetly'''