class Trunc(FunctionExpr): # TODO: check that the dtype is correct at compilation time (__init__ is too # early since we do not have the context yet) # assert getdtype(self.args[0], context) == float def compute(self, context, expr): if isinstance(expr, np.ndarray): return expr.astype(int) else: return int(expr) dtype = always(int)
class Regression(CompoundExpression): """abstract base class for all regressions""" @staticmethod def add_filter(expr, filter): if filter is not None: missing_value = missing_values[getdtype(expr, None)] return Where(filter, expr, missing_value) else: return expr dtype = always(float)
class LogitRegr(Regression): funcname = 'logit_regr' def build_expr(self, context, expr, filter=None, align=None): score_expr = LogitScore(expr) if align is not None: # we do not need add_filter because Alignment already handles it return Alignment(score_expr, align, filter=filter) else: return self.add_filter(ComparisonOp('>', score_expr, 0.5), filter) dtype = always(bool)
class Count(FunctionExpr): def compute(self, context, filter=None): if filter is None: return context_length(context) else: # TODO: check this at "compile" time (in __init__), though for # that we need to know the type of all temporary variables # first if not np.issubdtype(filter.dtype, bool): raise ValueError("count filter must be a boolean expression") return np.sum(filter) dtype = always(int)
class TimeAverage(TimeFunction): funcname = 'tavg' def compute(self, context, expr): entity = context.entity baseperiod = entity.base_period period = context.period - 1 res_size = len(entity.array) num_values = np.zeros(res_size, dtype=np.int) # current period last_period_wh_value = np.full(res_size, context.period, dtype=np.int) sum_values = np.zeros(res_size, dtype=np.float) id_to_rownum = context.id_to_rownum while period >= baseperiod: ids, values = self.value_for_period(expr, period, context, fill=None) # filter out lines which are present because there was a value for # that individual at that period but not for that column acceptable_rows = hasvalue(values) acceptable_ids = ids[acceptable_rows] if len(acceptable_ids): acceptable_values = values[acceptable_rows] value_rows = id_to_rownum[acceptable_ids] has_value = np.zeros(res_size, dtype=bool) safe_put(has_value, value_rows, True) period_value = np.zeros(res_size, dtype=np.float) safe_put(period_value, value_rows, acceptable_values) num_values += has_value * (last_period_wh_value - period) sum_values += period_value last_period_wh_value[has_value] = period period -= 1 return sum_values / num_values dtype = always(float)
def make_np_class(baseclass, docstring, dtypefunc): pos = docstring.find('(') name = docstring[:pos] args = docstring[pos + 1:-1] if isinstance(dtypefunc, type): dtypefunc = always(dtypefunc) # we need to explicitly set funcname, because the usual mechanism of # getting it from the class name during class creation (in the metaclass) # does not work because the class name is not set yet. class FuncClass(baseclass): np_func = getattr(np.random, name) funcname = name argspec = argspec(args, **baseclass.kwonlyargs) if dtypefunc is not None: dtype = dtypefunc FuncClass.__name__ = name.capitalize() return FuncClass
class LogitScore(CompoundExpression): funcname = 'logit_score' def build_expr(self, context, expr): if isinstance(expr, basestring): # assume it is a filename expr = ExtExpr(expr) u = Uniform() # expr in (0, 0.0, False, '') if not isinstance(expr, Expr) and not expr: expr = u else: epsilon = Logit(u) # expr = logistic(expr - epsilon) expr = Logistic(BinaryOp('-', expr, epsilon)) return expr dtype = always(float)
def make_np_class(baseclass, docstring, dtypefunc): name, args = split_signature(docstring) if isinstance(dtypefunc, type): dtypefunc = always(dtypefunc) evalfunc = getattr(np.random, name) # we need to explicitly set funcname, because the usual mechanism of # getting it from the class name during class creation (in the metaclass) # does not work because the class name is not set yet. class FuncClass(baseclass): np_func = evalfunc funcname = name argspec = argspec(args, **baseclass.kwonlyargs) if dtypefunc is not None: dtype = dtypefunc FuncClass.__name__ = name.capitalize() # skip first two lines, skip 8 first chars on each line newdoclines = [line[8:] for line in evalfunc.__doc__.split('\n')[2:]] FuncClass.__doc__ = '\n'.join(newdoclines) return FuncClass
class Average(FilteredExpression): funcname = 'avg' no_eval = ('expr', ) def compute(self, context, expr, filter=None, skip_na=True): # FIXME: either take "contextual filter" into account here (by using # self._getfilter), or don't do it in sum & gini if filter is not None: tmpvar = self.add_tmp_var(context, filter) if getdtype(expr, context) is bool: # convert expr to int because mul_bbb is not implemented in # numexpr # expr *= 1 expr = BinaryOp('*', expr, 1) # expr *= filter_values expr = BinaryOp('*', expr, tmpvar) else: filter = True values = expr_eval(expr, context) values = np.asarray(values) if skip_na: # we should *not* use an inplace operation because filter can be a # simple variable filter = filter & ispresent(values) if filter is True: numrows = len(values) else: numrows = np.sum(filter) if numrows: if skip_na: return na_sum(values) / float(numrows) else: return np.sum(values) / float(numrows) else: return float('nan') dtype = always(float)
class Gini(FilteredExpression): no_eval = ('filter', ) def compute(self, context, expr, filter=None, skip_na=True): values = np.asarray(expr) filter_expr = self._getfilter(context, filter) if filter_expr is not None: filter_values = expr_eval(filter_expr, context) else: filter_values = True if skip_na: # we should *not* use an inplace operation because filter_values # can be a simple variable filter_values = filter_values & ispresent(values) if filter_values is not True: values = values[filter_values] # from Wikipedia: # G = 1/n * (n + 1 - 2 * (sum((n + 1 - i) * a[i]) / sum(a[i]))) # i=1..n i=1..n # but sum((n + 1 - i) * a[i]) # i=1..n # = sum((n - i) * a[i] for i in range(n)) # = sum(cumsum(a)) sorted_values = np.sort(values) n = len(values) # force float to avoid overflows with integer input expressions cumsum = np.cumsum(sorted_values, dtype=float) values_sum = cumsum[-1] if values_sum == 0: print("gini(%s, filter=%s): expression is all zeros (or nan) " "for filter" % (self.args[0], filter)) return (n + 1 - 2 * np.sum(cumsum) / values_sum) / n dtype = always(float)
class Exp(NumexprFunction): argspec = argspec('expr') dtype = always(float)
class Matching(FilteredExpression): """ Base class for matching functions """ dtype = always(int)
class Std(NumpyAggregate): np_func = np.std dtype = always(float)
class AlignmentAbsoluteValues(FilteredExpression): funcname = 'align_abs' no_eval = ('filter', 'secondary_axis', 'expressions', 'method') def __init__(self, *args, **kwargs): super(AlignmentAbsoluteValues, self).__init__(*args, **kwargs) need = self.args[1] if isinstance(need, basestring): fpath = os.path.join(config.input_directory, need) need = load_ndarray(fpath, float) # XXX: store args in a list so that we can modify it? self.args = (self.args[0], need) + self.args[2:] self.past_error = None def collect_variables(self): # args[9] is the "link" argument # if self.args.link is None: if self.args[9] is None: return FilteredExpression.collect_variables(self) else: # in this case, it's tricky return set() def _eval_need(self, context, need, expressions, possible_values, expressions_context=None): assert isinstance(need, np.ndarray) if expressions_context is None: expressions_context = context # When given a 0d array, we convert it to 1d. This can happen e.g. for # >>> b = True; x = ne.evaluate('where(b, 0.1, 0.2)') # >>> isinstance(x, np.ndarray) # True # >>> x.shape # () if not need.shape: need = np.array([need]) if isinstance(need, LabeledArray): if not expressions: expressions = [ Variable(expressions_context.entity, name) for name in need.dim_names ] if not possible_values: possible_values = need.pvalues assert isinstance(need, np.ndarray) if len(expressions) != len(possible_values): raise Exception("align() expressions and possible_values " "have different length: %d vs %d" % (len(expressions), len(possible_values))) if 'period' in [str(e) for e in expressions]: period = context.period expressions, possible_values, need = \ kill_axis('period', period, expressions, possible_values, need) # kill any axis where the value is constant for all individuals # satisfying the filter # tokill = [(expr, column[0]) # for expr, column in zip(expressions, columns) # if isconstant(column, filter_value)] # for expr, value in tokill: # expressions, possible_values, need = \ # kill_axis(str(expr), value, expressions, possible_values, # need) return need, expressions, possible_values def _handle_frac_need(self, need, method): # handle the "fractional people problem" if not np.issubdtype(need.dtype, np.integer): if method == 'uniform': int_need = need.astype(int) if config.debug and config.log_level == "processes": print() print("random sequence position before:", np.random.get_state()[2]) u = np.random.uniform(size=need.shape) if config.debug and config.log_level == "processes": print("random sequence position after:", np.random.get_state()[2]) need = int_need + (u < need - int_need) elif method == 'cutoff': int_need = need.astype(int) frac_need = need - int_need need = int_need # the sum of fractional objects is the number of extra objects # we want aligned extra_wanted = int(round(np.sum(frac_need))) if extra_wanted: # search the cutoff point yielding: # sum(frac_need >= cutoff) == extra_wanted sorted_frac_need = frac_need.flatten() sorted_frac_need.sort() cutoff = sorted_frac_need[-extra_wanted] extra = frac_need >= cutoff if np.sum(extra) > extra_wanted: # This case can only happen when several bins have the # same frac_need. In this case we could try to be even # closer to our target by randomly selecting X out of # the Y bins which have a frac_need equal to the # cutoff (in addition to all those which are strictly # greater than it). assert np.sum(frac_need == cutoff) > 1 need += extra elif method == 'round': # always use 0.5 as a cutoff point need = (need + 0.5).astype(int) assert np.issubdtype(need.dtype, np.integer) return need def _add_past_error(self, context, need, method='default'): if method == 'default': method = context.get('__on_align_error__') if method == 'carry': if self.past_error is None: # TODO: we should store this somewhere in the context instead # XXX: I wonder if past_error shouldn't be float # (and _add_past_error should not be called before # _handle_frac_need instead of after like it does now). self.past_error = np.zeros(need.shape, dtype=int) print("adding %d individuals from last period error" % np.sum(self.past_error)) need += self.past_error return need def _display_unaligned(self, expressions, ids, columns, unaligned): print("Warning: %d individual(s) do not fit in any alignment " "category" % np.sum(unaligned)) header = ['id'] + [str(e) for e in expressions] columns = [ids] + columns num_rows = len(ids) print( PrettyTable([header] + [[col[row] for col in columns] for row in range(num_rows) if unaligned[row]])) def compute(self, context, score, need=None, filter=None, take=None, leave=None, expressions=None, possible_values=None, errors='default', frac_need='uniform', link=None, secondary_axis=None, method='bysorting'): if method not in ("bysorting", "sidewalk"): raise Exception( "Method for alignment should be either 'bysorting' " "or 'sidewalk'") if method == 'bysorting' and need is None: raise Exception("need argument is required when using the " "'bysorting' method (which is the default)") if method == "sidewalk": # need is calculated over score and we could think of # calculate without leave_filter and without take_filter if need is None: need = sum(score) # XXX: move this to _eval_need? # need is a single scalar if np.isscalar(need): need = [need] # need is a non-ndarray sequence if isinstance(need, (tuple, list)): need = np.array(need) assert isinstance(need, np.ndarray) if expressions is None: expressions = [] if possible_values is None: possible_values = [] else: possible_values = [np.array(pv) for pv in possible_values] if frac_need not in ('uniform', 'cutoff', 'round'): cls = ValueError if isinstance(frac_need, basestring) else TypeError raise cls("frac_need should be one of: 'uniform', 'cutoff' or " "'round'") if secondary_axis is not None and link is None: raise Exception("the 'secondary_axis' argument is only valid in " "combination with the 'link' argument") if not isinstance(secondary_axis, (type(None), int, Variable)): raise Exception( "'secondary_axis' should be either an integer or " "an axis name (but got '%s' which is of type '%s')" % (secondary_axis, type(secondary_axis))) func = self.align_no_link if link is None else self.align_link return func(context, score, need, filter, take, leave, expressions, possible_values, errors, frac_need, link, secondary_axis, method) def align_no_link(self, context, score, need, filter, take, leave, expressions, possible_values, errors, frac_need, link, secondary_axis, method): ctx_length = context_length(context) need, expressions, possible_values = \ self._eval_need(context, need, expressions, possible_values) filter_value = expr_eval(self._getfilter(context, filter), context) if filter_value is not None: num_to_align = np.sum(filter_value) else: num_to_align = ctx_length # retrieve the columns we need to work with if expressions: columns = [expr_eval(expr, context) for expr in expressions] if filter_value is not None: groups = partition_nd(columns, filter_value, possible_values) else: groups = partition_nd(columns, True, possible_values) else: columns = [] if filter_value is not None: groups = [filter_to_indices(filter_value)] else: groups = [np.arange(num_to_align)] # the sum is not necessarily equal to len(a), because some individuals # might not fit in any group (eg if some alignment data is missing) if sum(len(g) for g in groups) < num_to_align: unaligned = np.ones(ctx_length, dtype=bool) if filter_value is not None: unaligned[~filter_value] = False for member_indices in groups: unaligned[member_indices] = False self._display_unaligned(expressions, context['id'], columns, unaligned) # noinspection PyAugmentAssignment need = need * self._get_need_correction(groups, possible_values) need = self._handle_frac_need(need, frac_need) need = self._add_past_error(context, need, errors) need = np.asarray(need) # FIXME: either handle past_error in no link (currently, the past # error is added... but never computed, so always 0 !) or raise # an error in case errors='carry" is used with no link. return align_get_indices_nd(ctx_length, groups, need, filter_value, score, take, leave, method) def align_link(self, context, score, need, filter, take, leave, expressions, possible_values, errors, frac_need, link, secondary_axis, method): target_context = link._target_context(context) need, expressions, possible_values = \ self._eval_need(context, need, expressions, possible_values, target_context) # handle secondary axis if isinstance(secondary_axis, Expr): axis_name = str(secondary_axis) try: secondary_axis = need.dim_names.index(axis_name) except ValueError: raise ValueError("invalid value for secondary_axis: there is " "no axis named '%s' in the need array" % axis_name) else: if secondary_axis >= need.ndim: raise Exception("%d is an invalid value for secondary_axis: " "it should be smaller than the number of " "dimension of the need array (%d)" % (secondary_axis, need.ndim)) # evaluate columns target_columns = [expr_eval(e, target_context) for e in expressions] # this is a one2many, so the link column is on the target side link_column = target_context[link._link_field] filter_expr = self._getfilter(context, filter) if filter_expr is not None: reverse_link = Many2One("reverse", link._link_field, context.entity.name) target_filter = LinkGet(reverse_link, filter_expr, False) target_filter_value = expr_eval(target_filter, target_context) # It is often not a good idea to pre-filter columns like this # because we loose information about "indices", but in this case, # it is fine, because we do not need that information afterwards. filtered_columns = [ col[target_filter_value] if isinstance(col, np.ndarray) and col.shape else [col] for col in target_columns ] link_column = link_column[target_filter_value] else: filtered_columns = target_columns target_filter_value = None # compute labels for filtered columns # ----------------------------------- # We can't use _group_labels_light because group_labels assigns labels # on a first come, first served basis, not using the order they are # in pvalues fcols_labels = [] filtered_length = len(filtered_columns[0]) unaligned = np.zeros(filtered_length, dtype=bool) for fcol, pvalues in zip(filtered_columns, need.pvalues): pvalues_index = dict((v, i) for i, v in enumerate(pvalues)) fcol_labels = np.empty(filtered_length, dtype=np.int32) for i in range(filtered_length): value_idx = pvalues_index.get(fcol[i], -1) if value_idx == -1: unaligned[i] = True fcol_labels[i] = value_idx fcols_labels.append(fcol_labels) num_unaligned = np.sum(unaligned) if num_unaligned: # further filter label columns and link_column validlabels = ~unaligned fcols_labels = [labels[validlabels] for labels in fcols_labels] link_column = link_column[validlabels] # display who are the evil ones ids = target_context['id'] if target_filter_value is not None: filtered_ids = ids[target_filter_value] else: filtered_ids = ids self._display_unaligned(expressions, filtered_ids, filtered_columns, unaligned) else: del unaligned id_to_rownum = context.id_to_rownum missing_int = missing_values[int] source_ids = link_column if len(id_to_rownum): source_rows = id_to_rownum[source_ids] # filter out missing values: those where the value of the link # points to nowhere (-1) source_rows[source_ids == missing_int] = missing_int else: assert np.all(source_ids == missing_int) source_rows = [] # filtered_columns are not filtered further on invalid labels # (num_unaligned) but this is not a problem since those will be # ignored by GroupBy anyway. # TODO: this is ugly because a groupby on "values", returns an LArray with those # values (ndarrays) as axes names. Ugh. groupby_expr = GroupBy(*filtered_columns, pvalues=possible_values) # FIXME: target_context is not correct, as it is not filtered while # filtered_columns are. Since we do not use the context "columns" it # mostly works but I had to disable an assertion in utils.expand # because the length of the context is not correct. num_candidates = expr_eval(groupby_expr, target_context) # fetch the list of linked individuals for each local individual. # e.g. the list of person ids for each household hh = np.empty(context_length(context), dtype=object) # we can't use .fill([]) because it reuses the same list for all # objects for i in range(len(hh)): hh[i] = [] # Even though this is highly sub-optimal, the time taken to create # those lists of ids is very small compared to the total time taken # for align_other (0.2s vs 4.26), so I shouldn't care too much about # it for now. # target_row (row of person) is an index valid for *filtered/label* # columns ! for target_row, source_row in enumerate(source_rows): if source_row == -1: continue hh[source_row].append(target_row) class FakeContainer(object): def __init__(self, length): self.length = length def __len__(self): return self.length groups = [FakeContainer(g) for g in num_candidates] need = need * self._get_need_correction(groups, possible_values) need = self._handle_frac_need(need, frac_need) need = self._add_past_error(context, need, errors) # need = np.asarray(need) need = np.asarray(need) aligned, error = \ align_link_nd(score, need, num_candidates, hh, fcols_labels, secondary_axis) self.past_error = error return aligned def _get_need_correction(self, groups, possible_values): return 1 dtype = always(bool)
class Median(NumpyAggregate): np_func = np.median dtype = always(float)
class Percentile(NumpyAggregate): np_func = np.percentile dtype = always(float)
class All(NumpyAggregate): np_func = np.all dtype = always(bool)
class Dump(TableExpression): no_eval = ('args', ) kwonlyargs = { 'filter': None, 'missing': None, 'header': True, 'limit': None } def compute(self, context, *args, **kwargs): filter_value = kwargs.pop('filter', None) missing = kwargs.pop('missing', None) # periods = kwargs.pop('periods', None) header = kwargs.pop('header', True) limit = kwargs.pop('limit', None) entity = context.entity if args: expressions = list(args) else: # extra=False because we don't want globals nor "system" variables # (nan, period, __xxx__) # FIXME: we should also somehow "traverse" expressions in this case # too (args is ()) => all keys in the current context expressions = [ Variable(entity, name) for name in context.keys(extra=False) ] str_expressions = [str(e) for e in expressions] if 'id' not in str_expressions: str_expressions.insert(0, 'id') expressions.insert(0, Variable(entity, 'id')) id_pos = 0 else: id_pos = str_expressions.index('id') # if (self.periods is not None and len(self.periods) and # 'period' not in str_expressions): # str_expressions.insert(0, 'period') # expressions.insert(0, Variable('period')) # id_pos += 1 columns = [] for expr in expressions: if filter_value is False: # dtype does not matter much expr_value = np.empty(0) else: expr_value = expr_eval(expr, context) if (filter_value is not None and isinstance(expr_value, np.ndarray) and expr_value.shape): expr_value = expr_value[filter_value] columns.append(expr_value) ids = columns[id_pos] if isinstance(ids, np.ndarray) and ids.shape: numrows = len(ids) else: # FIXME: we need a test for this case (no idea how this can happen) numrows = 1 # expand scalar columns to full columns in memory # TODO: handle or explicitly reject columns wh ndim > 1 for idx, col in enumerate(columns): dtype = None if not isinstance(col, np.ndarray): dtype = type(col) elif not col.shape: dtype = col.dtype.type if dtype is not None: # TODO: try using itertools.repeat instead as it seems to be a # bit faster and would consume less memory (however, it might # not play very well with Pandas.to_csv) newcol = np.full(numrows, col, dtype=dtype) columns[idx] = newcol if limit is not None: assert isinstance(limit, (int, long)) columns = [col[:limit] for col in columns] data = izip(*columns) table = chain([str_expressions], data) if header else data return PrettyTable(table, missing) dtype = always(None)
class New(FilteredExpression): no_eval = ('filter', 'kwargs') def _initial_values(self, array, to_give_birth, num_birth, default_values): return get_default_array(num_birth, array.dtype, default_values) @classmethod def _collect_kwargs_variables(cls, kwargs): used_variables = set() # kwargs are stored as a list of (k, v) pairs for k, v in kwargs.iteritems(): used_variables.update(collect_variables(v)) return used_variables def compute(self, context, entity_name=None, filter=None, number=None, **kwargs): if filter is not None and number is not None: # Having neither is allowed, though, as there can be a contextual # filter. Also, there is no reason to prevent the whole # population giving birth, even though the usefulness of such # usage seem dubious. raise ValueError("new() 'filter' and 'number' arguments are " "mutually exclusive") source_entity = context.entity if entity_name is None: target_entity = source_entity else: target_entity = context.entities[entity_name] # target context is the context where the new individuals will be # created if target_entity is source_entity: target_context = context else: # we do need to copy the data (.extra) because we will insert into # the entity.array anyway => fresh_data=True target_context = context.clone(fresh_data=True, entity_name=target_entity.name) filter_expr = self._getfilter(context, filter) if filter_expr is not None: to_give_birth = expr_eval(filter_expr, context) num_birth = to_give_birth.sum() elif number is not None: to_give_birth = None num_birth = number else: to_give_birth = np.ones(len(context), dtype=bool) num_birth = len(context) array = target_entity.array default_values = target_entity.fields.default_values id_to_rownum = target_entity.id_to_rownum num_individuals = len(id_to_rownum) children = self._initial_values(array, to_give_birth, num_birth, default_values) if num_birth: children['id'] = np.arange(num_individuals, num_individuals + num_birth) children['period'] = context.period used_variables = [ v.name for v in self._collect_kwargs_variables(kwargs) ] if to_give_birth is None: assert not used_variables child_context = context.empty(num_birth) else: child_context = context.subset(to_give_birth, used_variables, filter_expr) for k, v in kwargs.iteritems(): if k not in array.dtype.names: print("WARNING: {} is unknown, ignoring it!".format(k)) continue children[k] = expr_eval(v, child_context) add_individuals(target_context, children) expr_cache.invalidate(context.period, context.entity_name) # result is the ids of the new individuals corresponding to the source # entity if to_give_birth is not None: result = np.full(context_length(context), -1, dtype=int) if source_entity is target_entity: extra_bools = np.zeros(num_birth, dtype=bool) to_give_birth = np.concatenate((to_give_birth, extra_bools)) # Note that np.place is a bit faster, but is currently buggy when # working with columns of structured arrays. # See https://github.com/numpy/numpy/issues/2462 result[to_give_birth] = children['id'] return result else: return None dtype = always(int)
class Any(NumpyAggregate): np_func = np.any dtype = always(bool)