def analyze_one_batch_of_models(savedmodel, matfilesdir, output, filt=False):
"""
Main job, from a saved model, find other saved model in same dir,
test them all, return plotting info (nSamples and dist)
"""
# hack
if savedmodel[-1] == os.path.sep:
savedmodel = savedmodel[:-1]
# GET SAVED MODELS
parentdir,tmp = os.path.split(savedmodel)
# traceback
tb = ANALYZE.traceback(savedmodel)
# find everything in parent folder, then just folders
all_in_folder = glob.glob(os.path.join(parentdir,'*'))
all_in_folder = filter(lambda x: os.path.isdir(x), all_in_folder)
# keep those that have same origin
leaves = filter(lambda x: ANALYZE.traceback(x)[0]==tb[0],all_in_folder)
# everything to test, matfile at the end
all_to_test = set()
for f in tb:
if os.path.isdir(f):
all_to_test.add(f)
for f in leaves:
all_to_test.add(f)
all_to_test = list(all_to_test)
# GET PARAMS / LOAD DATA
params = ANALYZE.unpickle(os.path.join(savedmodel,'params.p'))
oracle = ORACLE.OracleMatfiles(params,matfilesdir,oneFullIter=True)
data = [x for x in oracle]
data = filter(lambda x: x != None, data)
data = np.concatenate(data)
data = data[np.where(np.sum(data,axis=1)>0)]
if data.shape[0] == 0:
print_write('No patterns loaded, quit.',output)
sys.exit(0)
# PREDICT ON EVERY MODEL
dists = []
patterns = []
for f in all_to_test:
a,b,c,d = test_saved_model_folder(f,data,output,filt=filt)
dist,nPatterns,nIters,totalTime = a,b,c,d
dists.append(dist)
patterns.append(nPatterns)
# delete data before plotting
del data
# plot data
dists = np.array(dists)
patterns = np.array(patterns)
order = np.argsort(patterns)
# return 2 lists: nPatterns,dists and filt ordered by increasing patterns
return patterns[order],dists[order]
def analyze_one_batch_of_models(savedmodel, matfilesdir, output, filt=False):
"""
Main job, from a saved model, find other saved model in same dir,
test them all, return plotting info (nSamples and dist)
"""
# hack
if savedmodel[-1] == os.path.sep:
savedmodel = savedmodel[:-1]
# GET SAVED MODELS
parentdir, tmp = os.path.split(savedmodel)
# traceback
tb = ANALYZE.traceback(savedmodel)
# find everything in parent folder, then just folders
all_in_folder = glob.glob(os.path.join(parentdir, '*'))
all_in_folder = filter(lambda x: os.path.isdir(x), all_in_folder)
# keep those that have same origin
leaves = filter(lambda x: ANALYZE.traceback(x)[0] == tb[0], all_in_folder)
# everything to test, matfile at the end
all_to_test = set()
for f in tb:
if os.path.isdir(f):
all_to_test.add(f)
for f in leaves:
all_to_test.add(f)
all_to_test = list(all_to_test)
# GET PARAMS / LOAD DATA
params = ANALYZE.unpickle(os.path.join(savedmodel, 'params.p'))
oracle = ORACLE.OracleMatfiles(params, matfilesdir, oneFullIter=True)
data = [x for x in oracle]
data = filter(lambda x: x != None, data)
data = np.concatenate(data)
data = data[np.where(np.sum(data, axis=1) > 0)]
if data.shape[0] == 0:
print_write('No patterns loaded, quit.', output)
sys.exit(0)
# PREDICT ON EVERY MODEL
dists = []
patterns = []
for f in all_to_test:
a, b, c, d = test_saved_model_folder(f, data, output, filt=filt)
dist, nPatterns, nIters, totalTime = a, b, c, d
dists.append(dist)
patterns.append(nPatterns)
# delete data before plotting
del data
# plot data
dists = np.array(dists)
patterns = np.array(patterns)
order = np.argsort(patterns)
# return 2 lists: nPatterns,dists and filt ordered by increasing patterns
return patterns[order], dists[order]
def safe_traceback(folder):
"""
Performs traceback, but returns [None] if an error is raised.
"""
try:
return ANALYZE.traceback(folder)
except IOError:
return [None]
def safe_traceback(folder):
"""
Performs traceback, but returns [None] if an error is raised.
"""
try:
return ANALYZE.traceback(folder)
except IOError:
return [None]
# params
savedmodelFILT = os.path.abspath(sys.argv[1])
savedmodelNONFILT = os.path.abspath(sys.argv[2])
matfilesdir = os.path.abspath(sys.argv[3])
output = os.path.abspath(sys.argv[4])
print_write('saved model FILT= '+ savedmodelFILT,output)
print_write('saved model NONFILT= '+ savedmodelNONFILT,output)
print_write('matfiles dir = '+ matfilesdir,output)
print_write('output = '+output,output)
# GET SAVED MODELS FILT
parentdir,tmp = os.path.split(savedmodelFILT)
# traceback
tb = ANALYZE.traceback(savedmodelFILT)
# find everything in parent folder, then just folders
all_in_folder = glob.glob(os.path.join(parentdir,'*'))
all_in_folder = filter(lambda x: os.path.isdir(x), all_in_folder)
# keep those that have same origin
leaves = filter(lambda x: ANALYZE.traceback(x)[0]==tb[0],all_in_folder)
# everything to test, matfile at the end
all_to_testFILT = set()
for f in tb:
if os.path.isdir(f):
all_to_testFILT.add(f)
for f in leaves:
all_to_testFILT.add(f)
all_to_testFILT = list(all_to_testFILT)
# GET SAVED MODELS NON FILT
parentdir,tmp = os.path.split(savedmodelNONFILT)
# gather in a set all models to try
# also gather the number of iterations associated with each model
if os.path.isfile(savedmodel):
# WRONG CASE, matfile
print_write('Cannot do it on just a codebook.',output)
print_write('We dont know the params!.',output)
sys.exit(0)
elif testone:
# load one things and move on
print_write('Doing only one file.',output)
all_to_test = [savedmodel]
else:
# main algorithm
parentdir,tmp = os.path.split(savedmodel)
# traceback
tb = ANALYZE.traceback(savedmodel)
print_write('*** TRACEBACK ****************',output)
for f in tb:
print_write(str(f),output)
print_write('******************************',output)
# find everything in parent folder, then just folders
all_in_folder = glob.glob(os.path.join(parentdir,'*'))
all_in_folder = filter(lambda x: os.path.isdir(x), all_in_folder)
# keep those that have same origin
leaves = filter(lambda x: safe_traceback(x)[0]==tb[0],all_in_folder)
# everything to test, matfile at the end
all_to_test = set()
for f in tb:
if os.path.isdir(f):
all_to_test.add(f)
for f in leaves:
print 'we do a dry run, nothing will be deleted'
dryrun = True
else:
break
sys.argv.pop(1)
# last saved model
savedmodel = os.path.realpath(sys.argv[1])
if savedmodel[-1] == os.path.sep:
savedmodel = savedmodel[:-1] # hack
trimdir = os.path.dirname(savedmodel)
print 'most recent saved model:',savedmodel
print 'directory to trim:',trimdir
# get traceback
traceback = ANALYZE.traceback(savedmodel)
traceback = np.sort(traceback)
print '*********** TRACEBACK *************'
for k in traceback:
print k
print '***********************************'
# keep only folders
traceback = filter(lambda x: os.path.isdir(x), traceback)
# get all dirs in trimdir, in order
all_in_folder = glob.glob(os.path.join(trimdir,'*'))
all_in_folder = filter(lambda x: os.path.isdir(x), all_in_folder)
all_in_folder = np.sort(all_in_folder)
print 'number of folders found in main folder:',len(all_in_folder)
# go through them, add some to the 'remove set'
sys.argv.pop(1)
# params
savedmodelFILT = os.path.abspath(sys.argv[1])
savedmodelNONFILT = os.path.abspath(sys.argv[2])
matfilesdir = os.path.abspath(sys.argv[3])
output = os.path.abspath(sys.argv[4])
print_write('saved model FILT= ' + savedmodelFILT, output)
print_write('saved model NONFILT= ' + savedmodelNONFILT, output)
print_write('matfiles dir = ' + matfilesdir, output)
print_write('output = ' + output, output)
# GET SAVED MODELS FILT
parentdir, tmp = os.path.split(savedmodelFILT)
# traceback
tb = ANALYZE.traceback(savedmodelFILT)
# find everything in parent folder, then just folders
all_in_folder = glob.glob(os.path.join(parentdir, '*'))
all_in_folder = filter(lambda x: os.path.isdir(x), all_in_folder)
# keep those that have same origin
leaves = filter(lambda x: ANALYZE.traceback(x)[0] == tb[0], all_in_folder)
# everything to test, matfile at the end
all_to_testFILT = set()
for f in tb:
if os.path.isdir(f):
all_to_testFILT.add(f)
for f in leaves:
all_to_testFILT.add(f)
all_to_testFILT = list(all_to_testFILT)
# GET SAVED MODELS NON FILT
parentdir, tmp = os.path.split(savedmodelNONFILT)
