def _density(self, element, ctx, engine):
args_result = []
for elem in element.get_args():
args_result.append(
self._deduce(
elem,
Parser.get_ctx_with(
ctx,
density_view=Density.View.SAMPLE,
),
engine,
))
if ctx.density_view == Density.View.SAMPLE:
assert not ctx.requested_shape is None, "Shape information is not provided to sample {}".format(
element)
logging.debug("Sampling {} with shape {}x{}".format(
element, self._batch_size, ctx.requested_shape))
return engine(
element,
Engine.make_ctx(
tuple(args_result),
structure=(self._batch_size, ctx.requested_shape),
density_view=ctx.density_view,
))
elif ctx.density_view == Density.View.PROBABILITY:
provided_input = None
if isinstance(
ctx.probability_output,
PartOfSequence) and not ctx.sequence_variables is None:
logging.debug(
"Got likelihood of part of sequence {}, will try to find data from input to RNN"
.format(ctx.probability_output))
assert ctx.probability_output in ctx.sequence_variables, "Couldn't find sequence data in sequence info for {}".format(
ctx.output)
provided_input = ctx.sequence_variables[ctx.probability_output]
var_structure = self._structure.get(ctx.probability_output)
assert not var_structure is None, "Need to provide structure information for `{}'".format(
element)
return engine(
element,
Engine.make_ctx(
tuple(args_result),
density_view=ctx.density_view,
structure=(self._batch_size, var_structure),
provided_input=provided_input,
input_variable=ctx.probability_output,
))
elif ctx.density_view == Density.View.DENSITY:
return engine(
element,
Engine.make_ctx(
tuple(args_result),
density_view=ctx.density_view,
))
else:
raise Exception("Unexpected density view")
def _function_result(self, element, ctx, engine):
if isinstance(element.get_fun(), Function):
elem_structure = self._structure.get(element.get_fun(),
ctx.requested_shape)
assert not elem_structure is None, "Need to provide structure information for {}".format(
element)
args_result = []
for arg_elem in element.get_args():
args_result.append(
self._deduce(
arg_elem,
Parser.get_ctx_with(ctx,
requested_shape=elem_structure),
engine))
return engine(
element.get_fun(),
Engine.make_ctx(tuple(args_result), elem_structure, ctx.scope))
else:
args_result = []
for elem in element.get_args():
args_result.append(self._deduce(elem, ctx, engine))
return engine(element.get_fun(),
Engine.make_ctx(tuple(args_result)))
class EngineTestCase(EngineBaseTestCase):
"""
Class for testing the Engine class.
"""
def setUp(self):
distillery = Distillery.objects.get_by_natural_key(
'elasticsearch.test_index.test_docs')
self.engine = Engine(distillery.collection)
def test_find_by_id(self):
"""
Tests the find_by_id method.
"""
with self.assertRaises(NotImplementedError):
self.engine.find_by_id('xyz')
def test_find(self):
"""
Tests the find method.
"""
with self.assertRaises(NotImplementedError):
self.engine.find({'_id': 'xyz'})
def test_filter_ids(self):
"""
Tests the find_by_id method.
"""
with self.assertRaises(NotImplementedError):
self.engine.filter_ids(['1', '2'], ['foo', 'bar'], 'foobar')
def test_insert(self):
"""
Tests the insert method.
"""
with self.assertRaises(NotImplementedError):
self.engine.insert({'_id': 'xyz'})
def test_remove_by_id(self):
"""
Tests the remove_by_id method.
"""
with self.assertRaises(NotImplementedError):
self.engine.remove_by_id('xyz')
def _metrics(self, element, ctx, engine):
args_result = []
for elem in element.get_args():
args_result.append(
self._deduce(
elem,
Parser.get_ctx_with(ctx,
density_view=Density.View.DENSITY),
engine))
return engine(element, Engine.make_ctx(tuple(args_result)))
def _part_of_sequence(self, element, ctx, engine):
idx = element.get_idx()
if idx == 0:
return self._variable(element, ctx, engine)
elif idx == -1:
t_elem = get_zero_offset(element)
ret = self._deduce(t_elem, ctx, engine)
return engine(element, Engine.make_ctx(ret))
elif isinstance(idx, Index):
seq = element.get_seq()
if idx.get_offset() == -1: # t-1
seq_init = self._deduce(seq[0], ctx, engine)
return engine(element, Engine.make_ctx(seq_init))
elif idx.get_offset() == 0: # t
seq_t = self._variable(element, ctx, engine)
return seq_t
else:
raise Exception("Unsupported index offset: {}".format(idx))
else:
raise Exception("Unsupported indexing {}".format(idx))
def _deduce_variable_density(self, variable, density, ctx, engine):
elem_structure = self._structure.get(variable)
assert not elem_structure is None, "Need to provide structure information for {}".format(
variable)
var_dict = OrderedDict()
arg_result = self._deduce(
density,
Parser.get_ctx_with(ctx,
variables_tree=var_dict,
requested_shape=elem_structure), engine)
ctx.variables_tree[variable] = var_dict
return engine(variable, Engine.make_ctx((arg_result, ),
elem_structure))
def setUp(self):
distillery = Distillery.objects.get_by_natural_key(
'elasticsearch.test_index.test_docs')
self.engine = Engine(distillery.collection)
def _variable(self, element, ctx, engine):
candidates = []
for m in element.get_models():
for var_statement_id, density, probability in m.get_var_probabilities(
element):
min_global_statement = ctx.statement_id is None or var_statement_id <= ctx.statement_id # intersected with current statement id
if min_global_statement:
candidates.append((var_statement_id, density, probability))
next_density, next_statement_id, next_probability = None, None, None
if len(candidates) == 1:
logging.debug(
"Got 1 candidate {} to describe {}. Easy choice".format(
candidates[0], element))
next_statement_id, next_density, next_probability = candidates[0]
elif len(candidates) > 1:
logging.debug(
"Got {} candidates to describe variable {}, need to choose ..."
.format(len(candidates), element))
assert not ctx.dependencies is None, "Got too many descriptions of variable {}: {}; failing to choose one of them (need more context)" \
.format(element, ", ".join([str(c[2]) for c in candidates]))
deps = set([d for d in ctx.dependencies if d != element])
if len(deps) == 0:
logging.debug("Looking for description that is unconditioned")
candidates = [(c_st, c_dens, c_prob)
for c_st, c_dens, c_prob in candidates
if len(c_prob.get_dependencies()) == 0]
assert len(
candidates
) > 0, "Failed to find any description of {} which includes unconditioned dependency".format(
element)
assert len(
candidates
) == 1, "Got too many descriptions of variable {} which includes unconditioned dependency".format(
element)
else:
logging.debug(
"Looking for description that has {} as subset".format(
deps))
candidates = [(c_st, c_dens, c_prob)
for c_st, c_dens, c_prob in candidates
if deps <= set(c_prob.get_dependencies())]
assert len(
candidates
) > 0, "Failed to find any description of {} which has dependencies that includes {}".format(
element, deps)
assert len(
candidates
) == 1, "Got too many descriptions of variable {} which has dependencies that includes {}".format(
element, deps)
logging.debug("Found this one: {}".format(candidates[0][2]))
next_statement_id, next_density, next_probability = candidates[0]
var_structure = self._structure.get(element)
assert not var_structure is None, "Need to provide structure information for `{}'".format(
element)
if not next_density is None:
return self._deduce_variable_density(
element, next_density,
Parser.get_ctx_with(
ctx,
statement_id=next_probability.get_statement_id(),
dependencies=next_probability.get_dependencies(),
scope=next_probability.get_scope_name(),
), engine)
ctx.variables_tree[element] = None
return engine(
element,
Engine.make_ctx(structure=(self._batch_size, var_structure)))
def _default_callback(self, element, ctx, engine):
args_result = []
for elem in element.get_args():
args_result.append(self._deduce(elem, ctx, engine))
return engine(element, Engine.make_ctx(tuple(args_result)))