def test_check_context(self):
check('N', 1, N=1)
fail('N', 1, N=2)
self.assertRaises(ContractNotRespected, check, 'N', 1, N=2)
self.assertRaises(ValueError, fail, 'N', 1, N=1)
def test_check_context(self):
check('N', 1, N=1)
fail('N', 1, N=2)
self.assertRaises(ContractNotRespected, check, 'N', 1, N=2)
self.assertRaises(ValueError, fail, 'N', 1, N=1)
def test_check_context2():
""" Variable names must have only one letter. """
with raises(ValueError):
check('N', 1, NN=2)
with raises(ValueError):
check('N', 1, nn=2)
def _solve(self, R):
ndim = self.params['ndim']
num_iterations = self.params['num_iterations']
trust_R_top_perc = self.params['trust_R_top_perc']
check('>0,<100', trust_R_top_perc)
# Score of each datum -- must be computed only once
R_order = scale_score(R).astype('int32')
R_percentile = R_order * 100.0 / R_order.size
M = (R_order * 2.0 / (R.size - 1)) - 1
np.testing.assert_almost_equal(M.max(), +1)
np.testing.assert_almost_equal(M.min(), -1)
current_guess_for_S = best_embedding_on_sphere(M, ndim)
for iteration in range(num_iterations):
guess_for_C = cosines_from_directions(current_guess_for_S)
guess_for_C_sorted = np.sort(guess_for_C.flat)
new_estimated_C = guess_for_C_sorted[R_order]
careful_C = new_estimated_C.copy()
dont_trust = R_percentile < (100 - trust_R_top_perc)
# if iteration > 0:
careful_C[dont_trust] = guess_for_C[dont_trust]
# careful_C[dont_trust] = -1 # good for fov 360
# careful_C[dont_trust] = 0
new_guess_for_S = best_embedding_on_sphere(careful_C, ndim)
data = dict(S=new_guess_for_S, **locals())
self.iteration(data)
current_guess_for_S = new_guess_for_S
def _test_contracts():
try:
contracts.check('=0', numpy.arange(1, dtype=numpy.int8)[0])
except:
raise ImportError( \
"Invalid contracts[%s], does not support numpy types" \
% contracts.__version__)
def test_check(check, datum):
try:
contracts.check(check, datum)
return True
except contracts.ContractNotRespected as E:
#logging.warning("%r" % E)
return False
def generate_random_test_case(tcid,
ndim,
fov,
num,
kernel,
dist_noise=0,
abs_cos_noise_std=0):
S = random_directions_bounded(ndim=ndim, radius=fov / 2, num_points=num)
check('directions', S)
C = cosines_from_directions(S)
# get real distances
D = np.arccos(C)
# Multiplicative noise
noise = np.random.randn(*(D.shape))
D2 = D + dist_noise * D * noise
C2 = np.cos(D2)
R = kernel(C2)
# We don't want to mess with the diagonal
where = R < 5 * abs_cos_noise_std
R2 = R + where * np.random.randn(*(R.shape)) * abs_cos_noise_std
R2 = np.clip(R2, -1, 1)
tc = CalibTestCase(tcid, R2)
tc.set_ground_truth(S, kernel)
return tc
def from_yaml(spec):
try:
if not isinstance(spec, dict):
raise ValueError('Expected a dict, got %s' % spec)
s = dict(**spec)
id_stream = s.pop('id', None)
desc = s.pop('desc', None)
required = ['shape', 'format', 'range']
for x in required:
if not x in s:
raise ValueError('Missing entry %r.' % x)
shape = s.pop('shape')
check('list[>0](int,>0)', shape) # XXX: slow
format = s.pop('format') # @ReservedAssignment
range = s.pop('range') # @ReservedAssignment
extra = s.pop('extra', {})
filtered = s.pop('filtered', None)
default = s.pop('default', None)
names = s.pop('names', None) # TODO: do something useful with this
if names:
extra['names'] = names
if s.keys():
logger.warning('While reading\n%s\nextra keys detected: %s' %
((spec), s.keys()))
streamels = streamels_from_spec(shape, format, range, default)
return StreamSpec(id_stream, streamels, extra, filtered, desc)
except:
logger.error('Error while parsing the StreamSpec:\n%s' % spec)
raise
def _solve(self, R):
ndim = self.params['ndim']
num_iterations = self.params['num_iterations']
trust_R_top_perc = self.params['trust_R_top_perc']
check('>0,<100', trust_R_top_perc)
# Score of each datum -- must be computed only once
R_order = scale_score(R).astype('int32')
R_percentile = R_order * 100.0 / R_order.size
M = (R_order * 2.0 / (R.size - 1)) - 1
np.testing.assert_almost_equal(M.max(), +1)
np.testing.assert_almost_equal(M.min(), -1)
current_guess_for_S = best_embedding_on_sphere(M, ndim)
for iteration in range(num_iterations):
guess_for_C = cosines_from_directions(current_guess_for_S)
guess_for_C_sorted = np.sort(guess_for_C.flat)
new_estimated_C = guess_for_C_sorted[R_order]
careful_C = new_estimated_C.copy()
dont_trust = R_percentile < (100 - trust_R_top_perc)
# if iteration > 0:
careful_C[dont_trust] = guess_for_C[dont_trust]
# careful_C[dont_trust] = -1 # good for fov 360
# careful_C[dont_trust] = 0
new_guess_for_S = best_embedding_on_sphere(careful_C, ndim)
data = dict(S=new_guess_for_S, **locals())
self.iteration(data)
current_guess_for_S = new_guess_for_S
def best_orthogonal_transform(X, Y):
''' Finds the best orthogonal transform R between X and Y,
such that R X ~= Y. '''
YX = np.dot(Y, X.T)
check('array[KxK]', YX)
U, _, V = np.linalg.svd(YX)
best = np.dot(U, V)
return best
def best_orthogonal_transform(X, Y):
''' Finds the best orthogonal transform R between X and Y,
such that R X ~= Y. '''
YX = np.dot(Y, X.T)
check('array[KxK]', YX)
U, _, V = np.linalg.svd(YX)
best = np.dot(U, V)
return best
def test_check_context():
check('N', 1, N=1)
fail('N', 1, N=2)
with raises(ContractNotRespected):
check('N', 1, N=2)
with raises(ValueError):
fail('N', 1, N=1)
def init(self):
try:
with open(self.config.tensors, 'rb') as f:
tensors = pickle.load(f)
except Exception as e:
raise BadConfig("%s" % e, self, 'file')
self.T = tensors['Gnn']
check('array,(array[Kx2xHxW]|array[Kx1xN])', self.T)
def check_valid_identifier(e):
check_valid_identifier.__dict__['description'] = \
'check_valid_identifier(%r)' % e
identifier_expression.parseString(e, parseAll=True) #@UndefinedVariable
new_contract(e, '*')
check(e, 42)
def test_check_4():
score = (2, None)
msg = 'Player score must be a tuple of 2 int.'
try:
check('tuple(int,int)', score, msg)
except ContractNotRespected as e:
s = str(e)
assert msg in s
else:
assert False
def test_check_4(self):
score = (2, None)
msg = 'Player score must be a tuple of 2 int.'
try:
check('tuple(int,int)', score, msg)
except ContractNotRespected as e:
s = str(e)
assert msg in s
else:
self.assertFalse()
def solve(self, R, true_S=None):
self.R = R
self.R_order = scale_score(self.R).astype('int32')
self.R_sorted = np.sort(R.flat)
self.n = R.shape[0]
self.iterations = []
self.true_S = true_S
if self.is_spherical():
if true_S is not None:
check('directions', true_S)
self._solve(R)
for it in self.iterations:
if 'C' in it:
del it['C']
last_iteration = self.iterations[-1]
results = {}
copy_fields = [
'rel_error',
'rel_error_deg',
'spearman',
'spearman_robust',
'error',
'error_deg',
'S',
'S_aligned',
'diameter',
'diameter_deg',
'angles_corr',
'deriv_sign',
'phase',
'scaled_error',
'scaled_error_deg',
'scaled_rel_error',
'scaled_rel_error_deg',
]
for f in copy_fields:
if f in last_iteration:
results[f] = last_iteration[f]
results['R'] = R #.astype('float32')
results['n'] = R.shape[0]
results['params'] = self.params
results['true_S'] = true_S
results['iterations'] = self.iterations
results['geometry'] = self.geometry
self.results = results
return self.results
def best_similarity_transform(X, Y):
''' Finds the best transform (R,t) between X and Y,
such that R X + t ~= Y. '''
K = X.shape[0]
Xm = X.mean(axis=1).reshape(K, 1)
Ym = Y.mean(axis=1).reshape(K, 1)
X = X - Xm
Y = Y - Ym
# assert_allclose(X.mean(axis=1), 0, atol=1e-8)
# assert_allclose(Y.mean(axis=1), 0, atol=1e-8)
YX = np.dot(Y, X.T)
check('array[KxK]', YX)
U, _, V = np.linalg.svd(YX)
R = np.dot(U, V)
t = Ym - np.dot(R, Xm)
return R, t
def best_similarity_transform(X, Y):
''' Finds the best transform (R,t) between X and Y,
such that R X + t ~= Y. '''
K = X.shape[0]
Xm = X.mean(axis=1).reshape(K, 1)
Ym = Y.mean(axis=1).reshape(K, 1)
X = X - Xm
Y = Y - Ym
# assert_allclose(X.mean(axis=1), 0, atol=1e-8)
# assert_allclose(Y.mean(axis=1), 0, atol=1e-8)
YX = np.dot(Y, X.T)
check('array[KxK]', YX)
U, _, V = np.linalg.svd(YX)
R = np.dot(U, V)
t = Ym - np.dot(R, Xm)
return R, t
def solve(self, R, true_S=None):
self.R = R
self.R_order = scale_score(self.R).astype('int32')
self.R_sorted = np.sort(R.flat)
self.n = R.shape[0]
self.iterations = []
self.true_S = true_S
if self.is_spherical():
if true_S is not None:
check('directions', true_S)
self._solve(R)
for it in self.iterations:
if 'C' in it:
del it['C']
last_iteration = self.iterations[-1]
results = {}
copy_fields = ['rel_error',
'rel_error_deg',
'spearman', 'spearman_robust',
'error',
'error_deg',
'S', 'S_aligned', 'diameter',
'diameter_deg', 'angles_corr', 'deriv_sign', 'phase',
'scaled_error',
'scaled_error_deg',
'scaled_rel_error',
'scaled_rel_error_deg',
]
for f in copy_fields:
if f in last_iteration:
results[f] = last_iteration[f]
results['R'] = R #.astype('float32')
results['n'] = R.shape[0]
results['params'] = self.params
results['true_S'] = true_S
results['iterations'] = self.iterations
results['geometry'] = self.geometry
self.results = results
return self.results
def check_result(r, op):
if not ('bounds' in op):
return True
bounds = op['bounds']
if isinstance(bounds, function):
return bounds(r)
if isinstance(bounds, (tuple, list)): # min, max
return ((r >= bounds[0]) and (r <= bounds[1]))
if isinstance(bounds, (str, unicode)):
if has_contracts:
try:
contracts.check(bounds, r)
return True
except contracts.ContractNotRespected:
return False
else:
raise ImportError(
"import contracts failed, contract found[{}".format(bounds))
raise CheckError("Unknown bounds type {}, {}".format(type(bounds), bounds))
def structure_to_mongo(structure, course_context=None):
"""
Converts the 'blocks' key from a map {BlockKey: block_data} to
a list [block_data], inserting BlockKey.type as 'block_type'
and BlockKey.id as 'block_id'.
Doesn't convert 'root', since namedtuple's can be inserted
directly into mongo.
"""
with TIMER.timer('structure_to_mongo', course_context) as tagger:
tagger.measure('blocks', len(structure['blocks']))
check('BlockKey', structure['root'])
check('dict(BlockKey: BlockData)', structure['blocks'])
for block in structure['blocks'].itervalues():
if 'children' in block.fields:
check('list(BlockKey)', block.fields['children'])
new_structure = dict(structure)
new_structure['blocks'] = []
for block_key, block in structure['blocks'].iteritems():
new_block = dict(block.to_storable())
new_block.setdefault('block_type', block_key.type)
new_block['block_id'] = block_key.id
new_structure['blocks'].append(new_block)
return new_structure
def main():
parser = OptionParser()
parser.add_option("--file", help='Pickle file')
parser.add_option("--outdir")
(options, args) = parser.parse_args() #@UnusedVariable
assert not args
data = pickle.load(open(options.file, 'rb'))
select = range(181)
for x in ['y_cov', 'y_dot_cov', 'y_dot_sign_cov']:
data[x] = data[x][select, :][:, select]
check('array[NxN]', data[x])
n = data['y_cov'].shape[0]
#theta = numpy.linspace(0, numpy.pi * 2, n)
theta = numpy.linspace(0, numpy.pi, n)
d = OpenStruct(**data)
# groundtruth
d.theta = theta
d.S = create_s_from_theta(d.theta)
d.C = get_cosine_matrix_from_s(d.S)
d.D = get_distance_matrix_from_cosine(d.C)
r = Report('calibrator_analysis')
#r.add_child(simple_plots(d))
#r.add_child(ground_truth_plots(d))
#r.add_child(hist_plots(d))
r.add_child(iterations_plots(d))
filename = os.path.join(options.outdir, 'calib_1d_stats_plots.html')
print("Writing to %r" % filename)
r.to_html(filename)
def structure_from_mongo(structure):
"""
Converts the 'blocks' key from a list [block_data] to a map
{BlockKey: block_data}.
Converts 'root' from [block_type, block_id] to BlockKey.
Converts 'blocks.*.fields.children' from [[block_type, block_id]] to [BlockKey].
N.B. Does not convert any other ReferenceFields (because we don't know which fields they are at this level).
"""
check('seq[2]', structure['root'])
check('list(dict)', structure['blocks'])
for block in structure['blocks']:
if 'children' in block['fields']:
check('list(list[2])', block['fields']['children'])
structure['root'] = BlockKey(*structure['root'])
new_blocks = {}
for block in structure['blocks']:
if 'children' in block['fields']:
block['fields']['children'] = [
BlockKey(*child) for child in block['fields']['children']
]
new_blocks[BlockKey(block['block_type'],
block.pop('block_id'))] = BlockData(**block)
structure['blocks'] = new_blocks
return structure
def structure_from_mongo(structure, course_context=None):
"""
Converts the 'blocks' key from a list [block_data] to a map
{BlockKey: block_data}.
Converts 'root' from [block_type, block_id] to BlockKey.
Converts 'blocks.*.fields.children' from [[block_type, block_id]] to [BlockKey].
N.B. Does not convert any other ReferenceFields (because we don't know which fields they are at this level).
Arguments:
structure: The document structure to convert
course_context (CourseKey): For metrics gathering, the CourseKey
for the course that this data is being processed for.
"""
with TIMER.timer('structure_from_mongo', course_context) as tagger:
tagger.measure('blocks', len(structure['blocks']))
check('seq[2]', structure['root'])
check('list(dict)', structure['blocks'])
for block in structure['blocks']:
if 'children' in block['fields']:
check('list(list[2])', block['fields']['children'])
structure['root'] = BlockKey(*structure['root'])
new_blocks = {}
for block in structure['blocks']:
if 'children' in block['fields']:
block['fields']['children'] = [
BlockKey(*child) for child in block['fields']['children']
]
new_blocks[BlockKey(block['block_type'],
block.pop('block_id'))] = BlockData(**block)
structure['blocks'] = new_blocks
return structure
def structure_from_mongo(structure, course_context=None):
"""
Converts the 'blocks' key from a list [block_data] to a map
{BlockKey: block_data}.
Converts 'root' from [block_type, block_id] to BlockKey.
Converts 'blocks.*.fields.children' from [[block_type, block_id]] to [BlockKey].
N.B. Does not convert any other ReferenceFields (because we don't know which fields they are at this level).
Arguments:
structure: The document structure to convert
course_context (CourseKey): For metrics gathering, the CourseKey
for the course that this data is being processed for.
"""
with TIMER.timer('structure_from_mongo', course_context) as tagger:
tagger.measure('blocks', len(structure['blocks']))
check('seq[2]', structure['root'])
check('list(dict)', structure['blocks'])
for block in structure['blocks']:
if 'children' in block['fields']:
check('list(list[2])', block['fields']['children'])
structure['root'] = BlockKey(*structure['root'])
new_blocks = {}
for block in structure['blocks']:
if 'children' in block['fields']:
block['fields']['children'] = [BlockKey(*child) for child in block['fields']['children']]
new_blocks[BlockKey(block['block_type'], block.pop('block_id'))] = BlockData(**block)
structure['blocks'] = new_blocks
return structure
def structure_to_mongo(structure, course_context=None):
"""
Converts the 'blocks' key from a map {BlockKey: block_data} to
a list [block_data], inserting BlockKey.type as 'block_type'
and BlockKey.id as 'block_id'.
Doesn't convert 'root', since namedtuple's can be inserted
directly into mongo.
"""
with TIMER.timer('structure_to_mongo', course_context) as tagger:
tagger.measure('blocks', len(structure['blocks']))
check('BlockKey', structure['root'])
check('dict(BlockKey: BlockData)', structure['blocks'])
for block in six.itervalues(structure['blocks']):
if 'children' in block.fields:
check('list(BlockKey)', block.fields['children'])
new_structure = dict(structure)
new_structure['blocks'] = []
for block_key, block in six.iteritems(structure['blocks']):
new_block = dict(block.to_storable())
new_block.setdefault('block_type', block_key.type)
new_block['block_id'] = block_key.id
new_structure['blocks'].append(new_block)
return new_structure
def generate_random_test_case(tcid, ndim, fov, num, kernel,
dist_noise=0, abs_cos_noise_std=0):
S = random_directions_bounded(ndim=ndim, radius=fov / 2, num_points=num)
check('directions', S)
C = cosines_from_directions(S)
# get real distances
D = np.arccos(C)
# Multiplicative noise
noise = np.random.randn(*(D.shape))
D2 = D + dist_noise * D * noise
C2 = np.cos(D2)
R = kernel(C2)
# We don't want to mess with the diagonal
where = R < 5 * abs_cos_noise_std
R2 = R + where * np.random.randn(*(R.shape)) * abs_cos_noise_std
R2 = np.clip(R2, -1, 1)
tc = CalibTestCase(tcid, R2)
tc.set_ground_truth(S, kernel)
return tc
def structure_from_mongo(structure):
"""
Converts the 'blocks' key from a list [block_data] to a map
{BlockKey: block_data}.
Converts 'root' from [block_type, block_id] to BlockKey.
Converts 'blocks.*.fields.children' from [[block_type, block_id]] to [BlockKey].
N.B. Does not convert any other ReferenceFields (because we don't know which fields they are at this level).
"""
check('seq[2]', structure['root'])
check('list(dict)', structure['blocks'])
for block in structure['blocks']:
if 'children' in block['fields']:
check('list(list[2])', block['fields']['children'])
structure['root'] = BlockKey(*structure['root'])
new_blocks = {}
for block in structure['blocks']:
if 'children' in block['fields']:
block['fields']['children'] = [BlockKey(*child) for child in block['fields']['children']]
new_blocks[BlockKey(block['block_type'], block.pop('block_id'))] = block
structure['blocks'] = new_blocks
return structure
def structure_to_mongo(structure):
"""
Converts the 'blocks' key from a map {BlockKey: block_data} to
a list [block_data], inserting BlockKey.type as 'block_type'
and BlockKey.id as 'block_id'.
Doesn't convert 'root', since namedtuple's can be inserted
directly into mongo.
"""
check('BlockKey', structure['root'])
check('dict(BlockKey: dict)', structure['blocks'])
for block in structure['blocks'].itervalues():
if 'children' in block['fields']:
check('list(BlockKey)', block['fields']['children'])
new_structure = dict(structure)
new_structure['blocks'] = []
for block_key, block in structure['blocks'].iteritems():
new_block = dict(block)
new_block.setdefault('block_type', block_key.type)
new_block['block_id'] = block_key.id
new_structure['blocks'].append(new_block)
return new_structure
def structure_to_mongo(structure):
"""
Converts the 'blocks' key from a map {BlockKey: block_data} to
a list [block_data], inserting BlockKey.type as 'block_type'
and BlockKey.id as 'block_id'.
Doesn't convert 'root', since namedtuple's can be inserted
directly into mongo.
"""
check('BlockKey', structure['root'])
check('dict(BlockKey: dict)', structure['blocks'])
for block in structure['blocks'].itervalues():
if 'children' in block['fields']:
check('list(BlockKey)', block['fields']['children'])
new_structure = dict(structure)
new_structure['blocks'] = []
for block_key, block in structure['blocks'].iteritems():
new_block = dict(block)
new_block.setdefault('block_type', block_key.type)
new_block['block_id'] = block_key.id
new_structure['blocks'].append(new_block)
return new_structure
def test_check_1a():
with raises(ValueError):
check(1, 2)
def test_separate_context(self):
new_contract('my_list2', 'list[N]')
check('tuple(my_list2, my_list2)', ([1, 2], [1, 2]))
check('tuple(my_list2, my_list2)', ([1, 2], [1, 2, 3]))
def belongs(self, x):
check('array[N],unit_length', x)
def belongs(self, x):
# TODO: make this much more efficient
check('array[NxN],orthogonal', x, N=self.n)
det = np.linalg.det(x)
assert_allclose(det, 1, err_msg='I expect the determinant to be +1.')
def test_binding(self):
context = check('list[N](str), N>0', ['a', 'b', 'c'])
self.assertTrue('N' in context)
self.assertTrue(context['N'] == 3)
def test_unit_length(self):
check("unit_length", np.array([1]))
check("unit_length", np.array([0, 1]))
fail("unit_length", np.array([0, 2]))
def test_contract_not_cached():
z = 2
check('$z', 2)
z = 3
check('$z', 3)
def main():
from optparse import OptionParser
parser = OptionParser(description=__description__,
prog=__title__,
version='%prog ' + __version__)
parser.add_option('-t',
'--type',
action='store',
type='string',
dest='filetype',
help='Input file type (default: autodetect)',
default='autodetect')
parser.add_option('-o',
'--output_type',
action='store',
type='string',
dest='outtype',
help='Output file type (default: jpg)',
default='jpg')
parser.add_option('-O',
'--output_dir',
action='store',
type='string',
dest='outdir',
help="Output directory (default: input file's dir.)")
parser.add_option('-P',
'--postfix',
action='store',
type='string',
dest='postfix',
help='postfix for the output (default: _tone_mapped)',
default='_tone_mapped')
parser.add_option('-c',
'--contrast-limit',
action='store',
type='float',
dest='climit',
help='contrast limit in 1/N units (default: infinite)',
default=1e29)
parser.add_option(
'-e',
'--exponent',
action='append',
type='float',
dest='exps',
help='weight = factor/r^exponent (default exp=inf)',
)
parser.add_option(
'-f',
'--factor',
action='append',
type='float',
dest='factors',
help='weight = factor/r^exponent (default factor=1.0)',
)
# Not yet implemented.
# ~parser.add_option('-m', '--meta',
# ~action='store_true',
# ~dest='meta',
# ~help='Record metadata to OUTPUT_FILE.meta',
# ~default=True)
# ~parser.add_option('--no-meta',
# ~action='store_false',
# ~dest='meta',
# ~help='Do NOT record metadata.',
# ~default=True)
parser.add_option('-M',
'--maximum',
action='store',
type='float',
dest='MAX',
help='center-pixel weight (default: 1.0)',
default=1.0)
parser.add_option('-R',
'--R-cutoff',
action='store',
type='float',
dest='R_cutoff',
help='radial cutoff(default: no cutoff)',
default=np.inf)
parser.add_option('--dynamic_range_bottom',
action='store',
type='float',
dest='dynamic_bottom',
help='bottom of the dynamic range (default: image min)')
parser.add_option('--dynamic_range_top',
action='store',
type='float',
dest='dynamic_top',
help='top of the dynamic range (default: image max)')
parser.add_option('-v',
action='count',
dest='verbose',
help='Verbosity level, can be repeated,' +
' e.g. -vvv. (default: non-verbose mode)',
default=0)
parser.add_option('-b',
'--bin_count',
action='store',
type='int',
dest='bins',
help='number of bins (default: 0 - no binning)',
default=0)
parser.add_option('-s',
'--color_space',
action='store',
type='string',
dest='colorspace',
help='color space for processing (default: HSV)',
default='HSV')
parser.add_option(
'-S',
'--color_channel',
action='store',
type='int',
dest='color_channel',
help='color channel to be processed, starting ' +
'with 0 (default: 2 for HSV, 1 for XYZ, 0 for Y**)',
)
parser.add_option('-x',
'--downscale_ratio',
action='store',
type='float',
dest='downscale',
help='downscale factor, e.g. 2.0 (default: 1.0)')
parser.add_option('--distance_metric',
action='store',
type='string',
dest='distance',
default='eucledian',
help='Distance type. Valid parameters ' +
'"eucledian", "maximum", "manhattan"' +
'or any 0<p<inf float number.')
parser.add_option('--overwrite',
action='store_true',
dest='overwrite',
help='Overwrite output file, if it already exists.',
default=False)
parser.add_option('--cite',
dest='cite',
action='store_true',
default=False,
help='print citation information')
color_channel_mapping = {
'HSV': 2,
'YIQ': 0,
'YCbCr': 0,
'YPbPr': 0,
'YUV': 0
}
color_channel_scale_factor = {
'HSV': 1.0,
'YIQ': 1.0,
'YCbCr': 235.,
'YPbPr': 1.0,
'YUV': 1.0
}
(options, args) = parser.parse_args()
# Test of input parameters
check('int, >=0', options.bins, 'number of bins')
check('float, >0', options.climit, 'contrast limit')
if options.cite:
print('Reference for this software:')
print(__reference__)
print()
print('Bibtex format:')
print(__bibtex__)
sys.exit(0)
if len(args) == 0:
parser.print_help()
sys.exit(0)
# Check power function definition
if options.exps is None:
options.exps = [
1.0,
]
if options.factors is None:
options.factors = (1.0, ) * len(options.exps)
if len(options.factors) != len(options.exps):
eprint('The number of exponents and scaling factors must match!')
sys.exit(0)
for path in args:
check('filename', path)
if options.verbose > 0:
eprint('\nFile: {}'.format(path))
garbage_collector()
dirname, filename = os.path.split(path)
fileroot, ext = os.path.splitext(filename)
if not options.outtype.startswith('.'):
options.outtype = '.' + options.outtype
if options.outdir is None:
options.outdir = dirname
output_file = os.path.join(
options.outdir, fileroot + options.postfix + options.outtype)
if os.path.exists(output_file) and (not options.overwrite):
eprint('Output file already exists!')
eprint('Use "--overwrite", if you want to overwrite it.')
sys.exit(0)
# opening input file, and preprocessing
image, itype = read_image(path, options.filetype)
multi_channel = len(image.shape) > 2
# ~# color space conversion, if necessary
hidden_gray = False
if multi_channel:
if skimage.__version__ < '0.14.0':
eprint('Remark: Color conversion in skimage is buggy' +
' before 0.14. Use at least 0.14.')
sys.exit(1)
hidden_gray = True
for ch in range(1, image.shape[-1]):
hidden_gray = (hidden_gray
and np.all(image[..., 0] == image[..., ch]))
if not hidden_gray:
data = skimage.color.convert_colorspace(
image, 'RGB', options.colorspace)
color_channel = color_channel_mapping[options.colorspace]
img = (data[..., color_channel] /
color_channel_scale_factor[options.colorspace])
else:
image = image[..., 0]
img = image
multi_channel = False
else:
img = image
m = if_not_none(img.min(), options.dynamic_bottom)
M = if_not_none(img.max(), options.dynamic_top)
img = np.clip(img, m, M)
gain_limits = options.climit
if options.bins <= 1: # Use all
img = img.astype(np.float32).reshape(img.shape)
options.bins = int(img.max()) + 1
dtype = np.uint8 if options.bins < 255 else np.uint16
binned = img.astype(dtype)
else:
img = ((img - m) *
(float(options.bins - 1) / float(M - m))).astype(np.float32)
dtype = np.uint8 if options.bins < 255 else np.uint16
binned = dither(img, levels=options.bins, method='fs', dtype=dtype)
if options.verbose > 1:
eprint(' Image dimensions : {}'.format(image.shape))
if multi_channel:
eprint(' Color space, channel :' +
' {}, #{}'.format(options.colorspace, color_channel))
else:
eprint(' Color space, channel : single channel data')
eprint(' Dynamic range : {} - {} '.format(m, M))
if options.verbose > 2:
eprint('\n Command line: ', ' '.join(sys.argv))
# main processing
image = None
garbage_collector()
result = tone_mapping(img,
binned,
verbosity=options.verbose,
GAIN=gain_limits,
exps=options.exps,
factors=options.factors,
MAX=options.MAX,
R_cutoff=options.R_cutoff,
downscale=options.downscale,
distance_metric=options.distance)
result *= 255. / result.max()
result = dither(result, levels=256, method='fs', dtype=np.uint8)
img = None
binned = None
gain_limits = None
garbage_collector()
if options.verbose > 1:
eprint(' Output file: {}'.format(output_file))
if not multi_channel:
if hidden_gray:
result = np.dstack((result, result, result))
imageio.imsave(output_file, result)
else:
imageio.imsave(output_file, result)
# multichannel
else:
data[..., color_channel_mapping[options.colorspace]] = result * \
color_channel_scale_factor[options.colorspace] / result.max()
result = None
image = None
img = None
garbage_collector()
result = []
for d in np.array_split(data, 15):
result.append(
skimage.color.convert_colorspace(d, options.colorspace,
'RGB'))
data = None
garbage_collector()
rgb = np.concatenate(result)
result = None
garbage_collector()
imageio.imsave(output_file, rgb)
def belongs(self, x):
check('array[N],unit_length', x)
def test_check_fail():
z = 2
check('$z', 2)
fail('$z', 3)
def test_contract_not_cached():
z = 2
check('$z', 2)
z = 3
check('$z', 3)
def test_binding(self):
context = check('list[N](str), N>0', ['a', 'b', 'c'])
self.assertTrue('N' in context)
self.assertTrue(context['N'] == 3)
def test_check_2():
with raises(ContractNotRespected):
check('tuple(int,int)', (None, 2))
def test_check_1():
res = check('tuple(int,int)', (2, 2))
assert isinstance(res, dict)
def test_check_3():
with raises(ContractSyntaxError):
check('tuple(>>int,int)', (None, 2))
def test_unit_length(self):
check('unit_length', np.array([1]))
check('unit_length', np.array([0, 1]))
fail('unit_length', np.array([0, 2]))
def test_binding():
context = check('list[N](str), N>0', ['a', 'b', 'c'])
assert 'N' in context
assert context['N'] == 3
def test_callable2(self):
c = cname()
new_contract(c, ok2)
check('list(%s)' % c, [0])
def plot_results(label, results):
iterations = results['iterations']
r = Report(label)
R = results['R']
gt_C = results['gt_C']
f = r.figure(cols=3, caption='Ground truth')
with r.data_pylab('r_vs_c') as pylab:
pylab.plot(gt_C.flat, R.flat, '.', markersize=0.2)
pylab.xlabel('real cosine')
pylab.ylabel('correlation measure')
pylab.axis((-1, 1, -1, 1))
f.sub('r_vs_c', 'Unknown function correlation -> cosine')
r.data('gt_C', gt_C).display('posneg', max_value=1).add_to(f, 'ground truth cosine matrix')
dist = numpy.real(numpy.arccos(gt_C))
r.data('gt_dist', dist).display('scale').add_to(f, 'ground truth distance matrix')
f = r.figure(cols=12)
R = results['R']
R_order = results['R_order']
for i, it in enumerate(iterations):
singular_values = it['singular_values']
coords = it['coords']
coords_proj = it['coords_proj']
estimated_C = it['estimated_C']
estimated_C_order = it['estimated_C_order']
check('array[MxN],(M=2|M=3)', coords)
rit = r.node('iteration%2d' % i)
rit.data('Cest', it['Cest']).display('posneg', max_value=1).add_to(f, 'Cest')
rit.data('dont_trust', it['dont_trust'] * 1.0).display('scale').add_to(f, 'trust')
rit.data('Cestn', it['Cestn']).display('posneg', max_value=1).add_to(f, 'Cestn')
dist = numpy.real(numpy.arccos(it['Cestn']))
rit.data('dist', dist).display('scale', max_value=numpy.pi).add_to(f, 'corresponding distance')
distp = propagate(dist)
rit.data('distp', distp).display('scale', max_value=numpy.pi).add_to(f, 'propagated distance')
n = rit.data('singular_values', singular_values)
with n.data_pylab('plot') as pylab:
s = singular_values
s = s / s[0]
pylab.plot(s[:15], 'x-')
f.sub(n, 'Singular values')
n = rit.data('coords', coords)
with n.data_pylab('plot') as pylab:
pylab.plot(coords[0, :], coords[1, :], '.')
pylab.axis('equal')
f.sub(n, 'Coordinates')
n = rit.data('coords_proj', coords)
with n.data_pylab('plot') as pylab:
pylab.plot(coords_proj[0, :], coords_proj[1, :], '.')
pylab.axis((-1, 1, -1, 1))
f.sub(n, 'Coordinates (projected)')
with n.data_pylab('r_vs_est_c') as pylab:
pylab.plot(estimated_C.flat, R.flat, '.', markersize=0.2)
pylab.ylabel('estimated cosine')
pylab.xlabel('correlation measure')
pylab.axis((-1, 1, -1, 1))
f.sub('r_vs_est_c', 'R vs estimated C')
with n.data_pylab('order_order') as pylab:
pylab.plot(estimated_C_order.flat, R_order.flat, '.', markersize=0.2)
pylab.ylabel('est C order')
pylab.xlabel('R order')
f.sub('order_order')
# XXX: if mistake: add_child, nothing happens
rit.data('estimated_C', estimated_C).display('posneg').add_to(f, 'estimated_C')
rit.data('Cest_new', it['Cest_new']).display('posneg', max_value=1).add_to(f, 'Cest_new')
return r
def cbc(R, num_iterations=5, ground_truth=None):
'''
:type R: array[NxN]
:type num_iterations: int,>0
:rtype: dict
'''
# N = cbc.N
iterations = []
R_order = scale_score(R).astype('int32')
estimated_C = R
Cest = R
n = R.shape[0]
for iteration in range(num_iterations):
print('Iteration %d/%d' % (iteration, num_iterations))
Cestn = Cest.copy()
#dont_trust = numpy.logical_or(Cestn <= 0.8, R <= 0.8)
if iteration > 0:
dont_trust = Cestn <= numpy.cos(numpy.deg2rad(5))
#Cestn[dont_trust] = estimated_C[dont_trust]
# Cestn[dont_trust] = -1
else:
dont_trust = Cestn <= 10 # visualization
numpy.clip(Cestn, -1, 1, Cestn)
U, S, V = numpy.linalg.svd(Cestn, full_matrices=0)
check('array[NxN]', U)
check('array[N]', S)
check('array[NxN]', V)
# if iteration == 0:
# coords = V[[0, 2], :]
# else:
nvars = 2
coords = V[:nvars, :]
#check('array[2xN]', coords)
for i in range(nvars):
coords[i, :] = coords[i, :] * numpy.sqrt(S[i])
# remove mean
# if iteration == 0:
# for i in range(2):
# coords[i, :] -= coords[i, :].mean()
# project onto the sphere
nvars_project = 2
coords_proj = coords.copy()
for k in range(n):
v = coords_proj[:, k]
coords_proj[:, k] = v / numpy.linalg.norm(v[:nvars_project])
estimated_C = numpy.dot(coords_proj.T, coords_proj)
# find the best fit g() to
# estimated_C = g( R )
estimated_C_order = scale_score(estimated_C)
estimated_C_sorted = numpy.sort(estimated_C.flat)
Cest_new = estimated_C_sorted[R_order]
iteration = dict(Cest=Cest.copy(),
dont_trust=dont_trust.copy(),
Cestn=Cestn.copy(),
singular_values=S.copy(),
coords=coords.copy(),
coords_proj=coords_proj.copy(),
estimated_C=estimated_C.copy(),
estimated_C_order=estimated_C_order.copy(),
Cest_new=Cest_new.copy())
Cest = Cest_new
iterations.append(iteration)
#result = numpy.zeros((3, N))
results = dict(R=R, R_order=R_order,
gt_C=ground_truth.C,
gt_D=ground_truth.D,
gt_S=ground_truth.S,
iterations=iterations)
return results
def test_check_1(self):
res = check('tuple(int,int)', (2, 2))
assert isinstance(res, Context)
def main():
init_matplotlib()
reprep.RepRepDefaults.default_image_format = MIME_PDF
parser = OptionParser()
group = OptionGroup(parser, "Files and directories")
group.add_option("--outdir",
help='Directory with variables.pickle and where '
'the output will be placed.')
# group.add_option("--testdir", default=None)
group.add_option("--data_sick",
default=None,
help='.pickle file containing Sick data.')
group.add_option("--data_mino",
default=None,
help='directory containing Mino data.')
group.add_option("--data_fly",
default=None,
help='.pickle file containing fly simulation data.')
group.add_option("--test_cases",
default=None,
help='Base dire for test cases.')
parser.add_option_group(group)
group = OptionGroup(parser, "Experiments options")
group.add_option("--contracts",
default=False,
action='store_true',
help='Enables PyContacts sanity checks.')
group.add_option("--set",
default='*',
help='[= %default] Which combinations to run.')
group.add_option("--seed",
default=None,
type='int',
help='[= %default] Seed for random number generator.')
parser.add_option_group(group)
group = OptionGroup(parser, "Compmake options")
group.add_option("--remake",
default=False,
action='store_true',
help='Remakes all (non interactive).')
group.add_option("--report",
default=False,
action='store_true',
help='Cleans and redoes all reports (non interactive).')
group.add_option("--report_stats",
default=False,
action='store_true',
help='Cleans and redoes the reports for the stats. '
'(non interactive)')
parser.add_option_group(group)
(options, args) = parser.parse_args() #@UnusedVariable
np.random.seed(options.seed)
if not options.contracts:
disable_all()
assert options.outdir is not None
available_test_cases = {}
if options.test_cases is not None:
TCConfig.load(options.test_cases)
for tc_id in TCConfig.test_cases:
available_test_cases[tc_id] = \
(tc_load_spec, {'spec': TCConfig.test_cases[tc_id]})
print('Generating synthetic test cases...')
synthetic = get_syntethic_test_cases()
available_test_cases.update(synthetic)
euclidean = get_euclidean_test_cases()
available_test_cases.update(euclidean)
if options.data_sick is not None:
print('Preparing Sick data...')
real = get_real_test_cases(options.data_sick)
available_test_cases.update(real)
if options.data_fly is not None:
print('Preparing fly data...')
available_test_cases.update(get_fly_testcase(options.data_fly))
# if options.data_mino is not None:
# print('Preparing Mino data...')
# available_test_cases.update(get_mino_testcases(options.data_mino))
check('dict(str: tuple(Callable, dict))', available_test_cases)
print('Creating list of algorithms..')
algorithms = get_list_of_algorithms()
check('dict(str: tuple(Callable, dict))', algorithms)
print('Creating list of combinations..')
combinations = get_list_of_combinations()
which = expand_string(options.set, list(combinations.keys()))
print('I will use the sets: %s' % which)
if len(which) == 1:
compmake_storage = join(options.outdir, 'compmake', which[0])
else:
compmake_storage = join(options.outdir, 'compmake', 'common_storage')
use_filesystem(compmake_storage)
print('Available %d test cases and %d algorithms' %
(len(available_test_cases), len(algorithms)))
print('Staging creation of test cases reports')
test_cases = {}
test_case_reports = {}
def stage_test_case_report(tcid):
if not tcid in available_test_cases:
msg = ('Could not find test case %r \n %s' %
(tcid, available_test_cases.keys()))
raise Exception(msg)
if not tcid in test_cases:
f, args = available_test_cases[tcid]
job_id = 'test_case_data-%s' % tcid
test_cases[tcid] = comp(test_case_generate,
f=f,
args=args,
job_id=job_id)
if not tcid in test_case_reports:
job_id = 'test_case-%s-report' % tcid
report = comp(create_report_test_case,
tcid,
test_cases[tcid],
job_id=job_id)
job_id += '-write'
filename = join(options.outdir, 'test_cases', '%s.html' % tcid)
comp(write_report, report, filename, job_id=job_id)
test_case_reports[tcid] = report
return test_case_reports[tcid]
# set of tuple (algo, test_case)
executions = {}
def stage_execution(tcid, algid):
exc_id = '%s-%s' % (tcid, algid)
stage_test_case_report(tcid)
key = (tcid, algid)
if not key in executions:
test_case = test_cases[tcid]
if not algid in algorithms:
raise Exception('No %r known in %s' %
(algid, algorithms.keys()))
algo_class, algo_params = algorithms[algid]
executions[key] = comp(run_combination,
test_case,
algo_class,
algo_params,
job_id='calib-%s-run' % exc_id)
basename = join(options.outdir, 'results', '%s-%s' % (tcid, algid))
comp(save_results,
basename=basename,
results=executions[key],
job_id='calib-%s-save' % exc_id)
# Create iterations report
report = comp(create_report_iterations,
exc_id,
executions[key],
job_id='calib-%s-report' % exc_id)
filename = join(options.outdir, 'executions',
'%s-%s.html' % (tcid, algid))
comp(write_report,
report,
filename,
job_id='calib-%s-report-write' % exc_id)
return executions[key]
for comb_id in which:
comb = combinations[comb_id]
alg_ids = expand_string(comb.algorithms, algorithms.keys())
tc_ids = expand_string(comb.test_cases, available_test_cases.keys())
print(
'Set %r has %d test cases and %d algorithms (~%d jobs in total).' %
(comb_id, len(alg_ids), len(tc_ids),
len(alg_ids) * len(tc_ids) * 2))
deps = {}
for t, a in itertools.product(tc_ids, alg_ids):
deps[(t, a)] = stage_execution(t, a)
job_id = 'tex-%s' % comb_id
tables_dir = join(options.outdir, 'tables')
comp(create_tables_for_paper,
tables_dir,
comb_id,
tc_ids,
alg_ids,
deps,
job_id=job_id)
job_id = 'set-%s-report' % comb_id
report = comp(create_report_comb_stats,
comb_id,
tc_ids,
alg_ids,
deps,
job_id=job_id)
job_id += '-write'
filename = join(options.outdir, 'stats', '%s.html' % comb_id)
comp(write_report, report, filename, job_id=job_id)
if options.report or options.report_stats:
if options.report:
batch_command('clean *-report*')
elif options.report_stats:
batch_command('clean set-* tex*')
batch_command('parmake')
elif options.remake:
batch_command('clean *')
batch_command('make set-* tex-*')
else:
compmake_console()
def belongs_ts(self, bv):
# formatm('bv', bv)
check('tuple(shape(x),shape(x))', bv, x=self.shape)
base, vel = bv
self.belongs(base)
self.belongs(vel)
def test_valid(self):
c = new_contract('my_list', 'list[2]')
assert isinstance(c, Contract)
check('tuple(my_list, my_list)', ([1, 2], [1, 2]))
check_contracts_fail('tuple(my_list, my_list)', ([1, 2], [1, 2, 3]))
