def classifier_predict(listname, modelname, outdir=None, n_jobs=None):
if outdir == None:
outdir = tempfile.mkdtemp(dir=os.curdir, prefix='out')
else:
if not os.path.exists(outdir):
tsh.makedirs(outdir)
inputname = os.path.splitext(os.path.basename(listname))[0]
if listname.endswith('.gz'):
inputname = os.path.splitext(inputname)[0]
meta, data = read_listfile(listname)
classifier = read_classifierfile(modelname)
feature_method = classifier['features']['meta']['feature_method']
feature_args = meta.copy()
# Training input_name would shadow the current one.
del classifier['features']['meta']['input_name']
featurename = os.path.join(outdir, inputname + '-feats.csv.gz')
if os.path.exists(featurename):
_, features = read_listfile(featurename)
else:
feature_args.update(classifier['features']['meta'])
args, features = compute_features(feature_method, feature_args, data,
input_name=inputname, n_jobs=n_jobs, output_dir=outdir)
assert (data['id'] == features['id']).all()
clean_args(args)
write_listfile(featurename, features, input_name=inputname, **args)
labels_name = classifier['meta']['truth'] + '_labels'
labels = classifier['meta'][labels_name]
pred = predict(classifier['classifier'], sorted(labels.keys()), features,
output_dir=outdir)
write_listfile(os.path.join(outdir, inputname + '-predictions.csv.gz'), pred,
classifier_name=modelname, truth=classifier['meta']['truth'],
labels_name=labels, input_name=inputname)
def dissimilarities(methodname,
listname,
argsname=None,
n_jobs=None,
outdir=None):
if outdir == None:
outdir = tempfile.mkdtemp(dir=os.curdir, prefix='out')
else:
if not os.path.exists(outdir):
tsh.makedirs(outdir)
inputname = os.path.splitext(os.path.basename(listname))[0]
if listname.endswith('.gz'):
inputname = os.path.splitext(inputname)[0]
meta, data = read_listfile(listname)
args = meta
if argsname != None:
args.update(read_argsfile(argsname))
args, w = compute_dissimilarity(methodname,
args,
data,
n_jobs=n_jobs,
output_dir=outdir,
input_name=inputname)
if 'threshold' in args and args['threshold'] != 'False':
args, w = threshold_dissimilarity(args['threshold'], args, w)
dissim = prepare_weights_data(data['id'], data.dtype['id'], w)
clean_args(args)
write_listfile(os.path.join(outdir, inputname + '-dissim.csv.gz'),
dissim,
input_name=inputname,
**args)
def propagate_predict(modelname, dissimname, predictionsname, outdir=None):
if outdir == None:
outdir = tempfile.mkdtemp(dir=os.curdir, prefix='out')
else:
if not os.path.exists(outdir):
tsh.makedirs(outdir)
args = {}
predictions_meta, predictions = read_listfile(predictionsname)
args.update(predictions_meta)
dissim_meta, sample_ids, dissim = read_weightsfile(dissimname)
assert (predictions['id'] == np.array(sample_ids)).all()
assert predictions_meta['input_name'] == dissim_meta['input_name'], \
'Expecting same input names (%s x %s)' % (predictions_meta['input_name'], dissim_meta['input_name'])
inputname = predictions_meta['input_name']
args.update(dissim_meta)
model = read_propagatorfile(modelname)
args.update(model['meta'])
method_name = model['propagator']['method_name']
del model['propagator']['method_name']
args.update(model['propagator'])
args, prop = propagate(method_name,
args,
predictions,
dissim,
output_dir=outdir)
clean_args(args)
del args['cv_results']
write_listfile(os.path.join(outdir, inputname + '-propagated.csv.gz'),
prop, **args)
def dissimilarities(methodname, listname, argsname=None, n_jobs=None, outdir=None):
if outdir == None:
outdir = tempfile.mkdtemp(dir=os.curdir, prefix='out')
else:
if not os.path.exists(outdir):
tsh.makedirs(outdir)
inputname = os.path.splitext(os.path.basename(listname))[0]
if listname.endswith('.gz'):
inputname = os.path.splitext(inputname)[0]
meta, data = read_listfile(listname)
args = meta
if argsname != None:
args.update(read_argsfile(argsname))
args, w = compute_dissimilarity(methodname, args, data, n_jobs=n_jobs, output_dir=outdir, input_name=inputname)
if 'threshold' in args and args['threshold'] != 'False':
args, w = threshold_dissimilarity(args['threshold'], args, w)
dissim = prepare_weights_data(data['id'], data.dtype['id'], w)
clean_args(args)
write_listfile(os.path.join(outdir, inputname + '-dissim.csv.gz'), dissim, input_name=inputname, **args)
def propagate_predict(modelname, dissimname, predictionsname, outdir=None):
if outdir == None:
outdir = tempfile.mkdtemp(dir=os.curdir, prefix='out')
else:
if not os.path.exists(outdir):
tsh.makedirs(outdir)
args = {}
predictions_meta, predictions = read_listfile(predictionsname)
args.update(predictions_meta)
dissim_meta, sample_ids, dissim = read_weightsfile(dissimname)
assert (predictions['id'] == np.array(sample_ids)).all()
assert predictions_meta['input_name'] == dissim_meta['input_name'], \
'Expecting same input names (%s x %s)' % (predictions_meta['input_name'], dissim_meta['input_name'])
inputname = predictions_meta['input_name']
args.update(dissim_meta)
model = read_propagatorfile(modelname)
args.update(model['meta'])
method_name = model['propagator']['method_name']
del model['propagator']['method_name']
args.update(model['propagator'])
args, prop = propagate(method_name, args, predictions, dissim, output_dir=outdir)
clean_args(args)
del args['cv_results']
write_listfile(os.path.join(outdir, inputname + '-propagated.csv.gz'), prop, **args)
help='Propagator file.')
parser.add_argument('-o',
'--output',
dest='output',
required=False,
action='store',
default=None,
help='Output directory.')
opts = parser.parse_args()
if opts.output == None:
outdir = tempfile.mkdtemp(dir=os.curdir, prefix='out')
logger.info('Output directory %s', outdir)
else:
outdir = opts.output
if not os.path.exists(outdir):
tsh.makedirs(outdir)
config = tsh.read_config(opts, __file__)
basename = os.path.splitext(os.path.basename(opts.model))[0]
propagator = read_propagatorfile(opts.model)
cv_results = propagator['meta']['cv_results']
data = {}
for score, params, cm in cv_results:
key = tuple(params.items())
if key not in data:
data[key] = []
data[key] += [score]
all_param_keys = sorted(data.keys())
data = tsh.dict_values(data, all_param_keys)
plt.errorbar(range(len(all_param_keys)),
np.mean(data, axis=1),
fmt='ro',
import argparse
parser = argparse.ArgumentParser(description='Computes features for all the input data.')
parser.add_argument('-c', '--config', dest='config', required=False, action='store', default=None, help='Path to the config file')
parser.add_argument('-m', '--method', dest='method', required=True, action='store', choices=method_table.keys(), default=None, help='Method name.')
parser.add_argument('-a', '--args', dest='args', required=False, action='store', default=None, help='Method arguments file.')
parser.add_argument('-l', '--list', dest='list', required=True, action='store', default=None, help='List file.')
parser.add_argument('-j', '--jobs', dest='jobs', required=False, action='store', default=None, type=int, help='Number of parallel processes.')
parser.add_argument('-o', '--output', dest='output', required=False, action='store', default=None, help='Output directory.')
opts = parser.parse_args()
if opts.output == None:
outdir = tempfile.mkdtemp(dir=os.curdir, prefix='out')
logger.info('Output directory %s', outdir)
else:
outdir = opts.output
if not os.path.exists(outdir):
tsh.makedirs(outdir)
inputname = os.path.splitext(os.path.basename(opts.list))[0]
if opts.list.endswith('.gz'):
inputname = os.path.splitext(inputname)[0]
outputname = os.path.join(outdir, inputname + '-feats.csv.gz')
if os.path.exists(outputname):
logger.info('Skipping file %s, already exists.', outputname)
else:
config = tsh.read_config(opts, __file__)
meta, data = read_listfile(opts.list)
args = meta
if opts.args != None:
args.update(read_argsfile(opts.args))
args, features = compute_features(opts.method, args, data, input_name=inputname, n_jobs=opts.jobs, output_dir=outdir)
clean_args(args)
write_listfile(outputname, features, input_name=inputname, **args)
def get_dissimilarities(data,
output_dir=None,
input_name=None,
image_prefix=None,
mask_prefix=None,
n_jobs=None,
**kwargs):
assert image_prefix != None
assert mask_prefix != None
assert output_dir != None
assert input_name != None
measure = kwargs['measure']
distance_name = kwargs['distance_name']
rotation_invariance = kwargs['rotation_invariance']
normalized_width = kwargs['normalized_width']
normalized_height = kwargs['normalized_height']
if n_jobs == None:
n_jobs = 1
image_prefix = os.path.expanduser(image_prefix)
mask_prefix = os.path.expanduser(image_prefix)
distance_name = distance_name.format(OUT=output_dir,
INPUTNAME=input_name,
**kwargs)
tsh.makedirs(os.path.dirname(distance_name))
kwargs['distance_name'] = distance_name
expr_dir = os.path.join(
output_dir, 'expr/%04dx%04d' % (normalized_width, normalized_height))
tsh.makedirs(expr_dir)
# Make it easier for evaluate.py to create nice html reports.
if 'create_links' in kwargs and kwargs['create_links'] == True:
if os.path.exists('expr'):
os.unlink('expr')
try:
os.symlink(expr_dir, 'expr')
except:
pass
if os.path.exists('distance'):
os.unlink('distance')
try:
os.symlink(os.path.dirname(distance_name), 'distance')
except:
pass
save_expr_images = kwargs[
'save_expr_images'] if 'save_expr_images' in kwargs else False
if os.path.exists(distance_name):
D = tsh.deserialize(distance_name)['D']
else:
imagenames = [
os.path.join(image_prefix, sample['image']) for sample in data
]
masknames = [
os.path.join(mask_prefix, sample['mask']) for sample in data
]
n = len(data)
logger.info('Extracting %d expressions...', n)
Parallel(n_jobs=n_jobs, verbose=True, pre_dispatch='2*n_jobs')(
delayed(_extract_expression)
(imagenames[j], masknames[j],
os.path.join(expr_dir, 'inside%02d.png' %
data[j]['id']) if save_expr_images else None,
os.path.join(expr_dir, 'expr%02d.png' %
data[j]['id']) if save_expr_images else None,
os.path.join(expr_dir, 'expr%02d.dat' %
data[j]['id']), normalized_width, normalized_height)
for j in range(n))
logger.info('Computing %d dissimilarities...', (n * (n - 1)) / 2)
results = Parallel(
n_jobs=n_jobs, verbose=True,
pre_dispatch='2*n_jobs')(delayed(_get_dissimilarity)(
i, j, measure, rotation_invariance,
os.path.join(expr_dir, 'expr%02d.dat' % data[i]['id']),
os.path.join(expr_dir, 'expr%02d.dat' % data[j]['id']))
for j in range(n)
for i in range(j + 1, n))
logger.info('Transforming results...')
D = np.zeros((n, n), dtype=np.float)
tfxs = np.array([['I'] * n] * n)
for i, j, d, t in results:
D[j, i] = d
D[i, j] = d
tfxs[j, i] = t
tfxs[i, j] = t
logger.info('Saving results...')
tsh.serialize(
distance_name, {
'D': D,
'min': None,
'max': None,
'tfxs': tfxs,
'measure': measure,
'rotation_invariance': rotation_invariance
})
return kwargs, D
def get_dissimilarities(data, output_dir=None, input_name=None, image_prefix=None, mask_prefix=None, n_jobs=None, **kwargs):
assert image_prefix != None
assert mask_prefix != None
assert output_dir != None
assert input_name != None
measure = kwargs['measure']
distance_name = kwargs['distance_name']
rotation_invariance = kwargs['rotation_invariance']
normalized_width = kwargs['normalized_width']
normalized_height = kwargs['normalized_height']
if n_jobs == None:
n_jobs = 1
image_prefix = os.path.expanduser(image_prefix)
mask_prefix = os.path.expanduser(image_prefix)
distance_name = distance_name.format(OUT=output_dir, INPUTNAME=input_name, **kwargs)
tsh.makedirs(os.path.dirname(distance_name))
kwargs['distance_name'] = distance_name
expr_dir = os.path.join(output_dir, 'expr/%04dx%04d' % (normalized_width, normalized_height))
tsh.makedirs(expr_dir)
# Make it easier for evaluate.py to create nice html reports.
if 'create_links' in kwargs and kwargs['create_links'] == True:
if os.path.exists('expr'):
os.unlink('expr')
try:
os.symlink(expr_dir, 'expr')
except:
pass
if os.path.exists('distance'):
os.unlink('distance')
try:
os.symlink(os.path.dirname(distance_name), 'distance')
except:
pass
save_expr_images = kwargs['save_expr_images'] if 'save_expr_images' in kwargs else False
if os.path.exists(distance_name):
D = tsh.deserialize(distance_name)['D']
else:
imagenames = [ os.path.join(image_prefix, sample['image']) for sample in data ]
masknames = [ os.path.join(mask_prefix, sample['mask']) for sample in data ]
n = len(data)
logger.info('Extracting %d expressions...', n)
Parallel(n_jobs=n_jobs, verbose=True,
pre_dispatch='2*n_jobs')(
delayed(_extract_expression)(
imagenames[j],
masknames[j],
os.path.join(expr_dir, 'inside%02d.png' % data[j]['id']) if save_expr_images else None,
os.path.join(expr_dir, 'expr%02d.png' % data[j]['id']) if save_expr_images else None,
os.path.join(expr_dir, 'expr%02d.dat' % data[j]['id']),
normalized_width,
normalized_height
) for j in range(n))
logger.info('Computing %d dissimilarities...', (n*(n-1))/2)
results = Parallel(n_jobs=n_jobs, verbose=True,
pre_dispatch='2*n_jobs')(
delayed(_get_dissimilarity)(
i, j,
measure, rotation_invariance,
os.path.join(expr_dir, 'expr%02d.dat' % data[i]['id']),
os.path.join(expr_dir, 'expr%02d.dat' % data[j]['id'])
) for j in range(n) for i in range(j+1, n))
logger.info('Transforming results...')
D = np.zeros((n, n), dtype=np.float)
tfxs = np.array([['I'] * n] * n)
for i, j, d, t in results:
D[j, i] = d
D[i, j] = d
tfxs[j, i] = t
tfxs[i, j] = t
logger.info('Saving results...')
tsh.serialize(distance_name, {
'D': D,
'min': None, 'max': None,
'tfxs': tfxs,
'measure': measure,
'rotation_invariance': rotation_invariance })
return kwargs, D
from utils import read_listfile
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='Copies images for all the listfiles.')
parser.add_argument('-r', '--relative', dest='relative', required=True, action='store', default=None, help='Make paths relative to this.')
parser.add_argument('lists', nargs='*', action='store', help='List file(s).')
parser.add_argument('-o', '--output', dest='output', required=False, action='store', default=None, help='Output directory.')
opts = parser.parse_args()
if opts.output == None:
outdir = tempfile.mkdtemp(dir=os.curdir, prefix='out')
logger.info('Output directory %s', outdir)
else:
outdir = opts.output
if not os.path.exists(outdir):
tsh.makedirs(outdir)
for listname in opts.lists:
meta, data = read_listfile(listname)
image_prefix = meta['image_prefix']
mask_prefix = meta['mask_prefix']
for d in data:
filename = os.path.join(image_prefix, d['image'])
destdir = os.path.dirname(os.path.join(outdir, os.path.relpath(filename, opts.relative)))
tsh.makedirs(destdir)
shutil.copy(filename, destdir)
filename = os.path.join(mask_prefix, d['mask'])
destdir = os.path.dirname(os.path.join(outdir, os.path.relpath(filename, opts.relative)))
tsh.makedirs(destdir)
shutil.copy(filename, destdir)