def __init__(self,proj_id): self.dtype = np.float32 try: self.vg = projector.volume_geometry(proj_id) self.pg = projector.projection_geometry(proj_id) self.data_mod = data2d self.appendString = "" if projector.is_cuda(proj_id): self.appendString += "_CUDA" except Exception: self.vg = projector3d.volume_geometry(proj_id) self.pg = projector3d.projection_geometry(proj_id) self.data_mod = data3d self.appendString = "3D" if projector3d.is_cuda(proj_id): self.appendString += "_CUDA" self.vshape = functions.geom_size(self.vg) self.vsize = reduce(operator.mul,self.vshape) self.sshape = functions.geom_size(self.pg) self.ssize = reduce(operator.mul,self.sshape) self.shape = (self.ssize, self.vsize) self.proj_id = proj_id self.T = OpTomoTranspose(self)
def as_list(self): # on top-level, I have type and depth, for easier parallelism, # but this is hacky as hell schema = ( self.schema.items if isinstance(self.schema, ArraySchema) else self.schema ) items = [InputAdapter(item, self.evaluator, schema) for item in self.value] if not self.prefix: return reduce(operator.add, [a.arg_list() for a in items], []) if self.separate and self.item_separator is None: return reduce(operator.add, [[self.prefix] + a.arg_list() for a in items], []) if not self.separate and self.item_separator is None: return [self.prefix + a.list_item() for a in items if a.list_item() is not None] joined = self.item_separator.join( filter(None, [a.list_item() for a in items]) ) if self.separate and self.item_separator is not None: return [self.prefix, joined] return [self.prefix + joined]
def evaluate(self, previousResult, original=None, fast=False): original = original if original is not None else previousResult if original._grouped_self and fast: name = GetName(self) # TODO: Rewrite this, this is terrible. go_to_index = len(self._queue) for idx, item in enumerate(self._queue): if isinstance(item, OperatorStep): go_to_index = idx break transform_input = lambda x: reduce( lambda prevResult, f: f.evaluate(prevResult, original, fast=fast), self._queue[1:go_to_index], x ) out = original._grouped_self[name].transform(transform_input) out = reduce(lambda prevResult, f: f.evaluate(prevResult, original, fast=fast), self._queue[go_to_index:], out) return out else: output = reduce(lambda prevResult, f: f.evaluate(prevResult, original), self._queue, original) return output
def generate(operator, *args): """Generates SparseInterval instance current and past results of RecurrentEvent.forward() generators with respect to 'operator' of given expression.""" operands = [] for arg in args: if isinstance(arg, dict): try: result = next(arg['generator']) arg['results'] = ( arg['results'].union(SparseInterval(*[result])) ) except StopIteration: arg['exhausted'] = True operands.append(arg['results']) elif isinstance(arg, SparseInterval): operands.append(arg) if operator == AND: return reduce(lambda m, v: m.intersection(v), operands) elif operator == OR: return reduce(lambda m, v: m.union(v), operands) elif operator == NOT: union = reduce(lambda m, v: m.union(v), operands) intervals = it.chain((MIN,), it.chain.from_iterable(union.intervals), (MAX,)) return SparseInterval(*zip(intervals, intervals)) else: raise AssertionError
def __purity(cls, mem, references, macro=True, verbose=False): clusters = {} avg = 0 memberships = [x for x in mem] for (i, x) in memberships: if x in clusters: clusters[x].append(i) else: clusters[x] = [i] for docs in clusters.values(): (_, count) = median(reduce(lambda s, x: s + x, [references[x] for x in docs])) if verbose: print('# %f = %d/%d, labels %s for cluster %s' % ( float(count) / float(len(docs)), count, len(docs), str(reduce(lambda s, x: s + x, [references[x] for x in docs])), str(docs))) if macro: avg += count else: avg += float(count) / float(len(docs)) if macro: return float(avg) / len(memberships) else: return float(avg) / len(clusters)
def reshape(self, *shape): """Gives a new shape to an array without changing its data.""" # TODO: add more error-checking, perhaps if not self.flags.forc: raise RuntimeError("only contiguous arrays may " "be used as arguments to this operation") if isinstance(shape[0], tuple) or isinstance(shape[0], list): shape = tuple(shape[0]) if shape == self.shape: return self if -1 in shape: shape = list(shape) idx = shape.index(-1) size = -reduce(lambda x, y: x * y, shape, 1) shape[idx] = self.size // size if -1 in shape[idx:]: raise ValueError("can only specify one unknown dimension") shape = tuple(shape) size = reduce(lambda x, y: x * y, shape, 1) if size != self.size: raise ValueError("total size of new array must be unchanged") return GPUArray( shape=shape, dtype=self.dtype, allocator=self.allocator, base=self, gpudata=int(self.gpudata))
def get_context_data(self, **kwargs): context = super(SearchableListMixin, self).get_context_data(**kwargs) qs = self.model.objects.all() total_objects = qs.count() query = self.get_search_query() is_filter = False if query: is_filter = True w_qs = [] search_pairs = self.get_search_fields_with_filters() for word in self.get_words(query): filters = [Q(**{'%s__%s' % (pair[0], pair[1]): word}) for pair in search_pairs] if self.search_date_fields: dt = self.try_convert_to_date(word) if dt: filters.extend([Q(**{field_name: dt}) for field_name in self.search_date_fields]) w_qs.append(reduce(operator.or_, filters)) qs = qs.filter(reduce(operator.and_, w_qs)) qs = qs.distinct() total_objects = qs.count() paginator = Paginator(qs, 10) page = self.request.GET.get('page') try: qs = paginator.page(page) except PageNotAnInteger: qs = paginator.page(1) except EmptyPage: qs = paginator.page(paginator.num_pages) context.update({'object_list': qs, 'is_filter': is_filter, 'total_objects': total_objects}) return context
def __init__(self,proj_id): self.dtype = np.float32 try: self.vg = projector.volume_geometry(proj_id) self.pg = projector.projection_geometry(proj_id) self.data_mod = data2d self.appendString = "" if projector.is_cuda(proj_id): self.appendString += "_CUDA" except Exception: self.vg = projector3d.volume_geometry(proj_id) self.pg = projector3d.projection_geometry(proj_id) self.data_mod = data3d self.appendString = "3D" if projector3d.is_cuda(proj_id): self.appendString += "_CUDA" self.vshape = functions.geom_size(self.vg) self.vsize = reduce(operator.mul,self.vshape) self.sshape = functions.geom_size(self.pg) self.ssize = reduce(operator.mul,self.sshape) self.shape = (self.ssize, self.vsize) self.proj_id = proj_id self.transposeOpTomo = OpTomoTranspose(self) try: self.T = self.transposeOpTomo except AttributeError: # Scipy >= 0.16 defines self.T using self._transpose() pass
def backtrack(self, node, tasks): candidates = self.fetch_tracks(node.op) tracks = [] def filter(node, depth): new_candidates = [] for candidate in candidates: track, i, lopt = candidate if i < depth: pass elif track[i-depth] in (None, node.op): if i == depth: tasks[node].append(lopt) else: tracks.append(candidate) else: new_candidates.append(candidate) return new_candidates depth = 0 nodes = [node] while candidates: for node in nodes: candidates = list(filter(node, depth)) depth += 1 _nodes = nodes nodes = reduce(list.__iadd__, [reduce(list.__iadd__, [[n for n, i in out.clients if not isinstance(n, string_types)] for out in node.outputs], []) for node in nodes], []) candidates = tracks tracks = []
def _get_merged_histograms(cursor, property_name, path, with_processes, histograms_url, additional_histograms): if path[0] == "histograms" and len(path) != 2: raise ValueError("Histogram access requires a histogram name.") elif path[0] == "keyedHistograms" and len(path) != 3: raise ValueError("Keyed histogram access requires both a histogram name and a label.") # Get parent property parent = _get_ping_property(cursor, path, histograms_url, additional_histograms) # Get children properties if not isinstance(cursor, dict): children = [] else: children = [_get_ping_property(cursor["processes"]["content"], path, histograms_url, additional_histograms)] children += [_get_ping_property(child, path, histograms_url, additional_histograms) for child in cursor.get("childPayloads", [])] children += [_get_ping_property(cursor["processes"]["gpu"], path, histograms_url, additional_histograms)] children = list(filter(lambda h: h is not None, children)) # Merge parent and children merged = ([parent] if parent else []) + children result = {} if with_processes: result[property_name + "_parent"] = parent result[property_name + "_children"] = reduce(add, children) if children else None result[property_name] = reduce(add, merged) if merged else None return result
def combineQualities(allowed_qualities, preferred_qualities): any_quality = 0 best_quality = 0 if allowed_qualities: any_quality = reduce(operator.or_, allowed_qualities) if preferred_qualities: best_quality = reduce(operator.or_, preferred_qualities) return any_quality | (best_quality << 16)
def get_usage(self, using="short"): if using not in set(["short", "long", "both"]): raise ValueError( "using has to be 'short', 'long' or 'both'; not %r" % using ) # The usage is generated in three steps: # 1. A "linked list" is generated with lists as nodes. Each node is a # list consisting of an optional positional as first item followed by # zero or more required positionals, optionally followed by a list # representing the next node. # # 2. The usage string is generated by enclosing the with a space joined # usage strings of the positionals in a given node with brackets. # The exception is the root which is not enclosed by brackets as the # first node is always required. # # 3. The usage string generated in `2` is prefixed with the short and/ # or the long and either or both joined with a comma are returned. def step(acc, next): root, current = acc if next.optional: current.append([next]) current = current[-1] else: current.append(next) return root, current def render(tree, _root=True): if isinstance(tree, Positional): return tree.usage else: nodes = u(" ").join(render(node, _root=False) for node in tree) if _root: return nodes return u("[{0}]").format(nodes) usage = render(reduce(step, self.positionals, ([], ) * 2)[0]) if using == "both" and self.short and self.long: return ( u("{0} {1}").format(self.short, usage).strip() + u(", ") + u("{0} {1}").format(self.long, usage).strip() ) else: return u("{0} {{0}}").format( self.short if using == "short" and self.short or using in set(["long", "both"]) and not self.long else self.long ).format(usage) return usage.format( short=self.short, long=self.long, usage=render(reduce(step, self.positionals, ([], ) * 2)[0]) ).strip()
def expr(cls): e = pp.Literal("@").suppress() + v_integer u = reduce(operator.or_, [pp.Literal(i) for i in human.SIZE_UNITS.keys()]).leaveWhitespace() e = e + pp.Optional(u, default=None) s = pp.Literal(",").suppress() s += reduce(operator.or_, [pp.Literal(i) for i in generators.DATATYPES.keys()]) e += pp.Optional(s, default="bytes") return e.setParseAction(lambda x: cls(*x))
def lcm( *a ): """Least common multiple. Usage: lcm( [ 3, 4, 5 ] ) or: lcm( 3, 4, 5 ) """ if len( a ) > 1: return reduce( lcm2, a ) if hasattr( a[0], "__iter__" ): return reduce( lcm2, a[0] ) return a[0]
def gcd( *a ): """Greatest common divisor. Usage: gcd( [ 2, 4, 6 ] ) or: gcd( 2, 4, 6 ) """ if len( a ) > 1: return reduce( gcd2, a ) if hasattr( a[0], "__iter__" ): return reduce( gcd2, a[0] ) return a[0]
def __setattr__(self, name, value): if isinstance(self._parent, Style.Style): setattr( reduce(getattr, self._attribute.split('.'), self._parent), name, value) else: for cell in self._parent: setattr( reduce(getattr, self._attribute.split('.'), cell), name, value)
def _get_total_hits(self): """ Returns the total number of hits across all teams and players. """ return sm.reduce( add, [ sm.reduce(add, [p.hits for p in t.players], 0) for t in self.model.teams ], 0, )
def add_trading_days(self, n, date): if n > 0: return reduce( lambda a, b: self.next_trading_day(a), range(n), date, ) else: return reduce( lambda a, b: self.previous_trading_day(a), range(abs(n)), date, )
def flatten(s): if isinstance(s, (tuple, list)): return reduce(lambda a, b: a + flatten(b), s, []) elif isinstance(s, set): raise TypeError("parameter flattening cannot order sets") elif isinstance(s, dict): return reduce(lambda a, b: a + flatten(s[b]), sorted(s.keys()), []) elif isinstance(s, BaseParameter): return [s] elif s is None: return [] else: raise TypeError("don't understand type %s for %r" % (type(s), s))
def get_queryset(self): qs = super(SearchableListMixin, self).get_queryset() query = self.get_search_query() if query: w_qs = [] search_pairs = self.get_search_fields_with_filters() for word in self.get_words(query): filters = [Q(**{'%s__%s' % (pair[0], pair[1]): word}) for pair in search_pairs] if self.search_date_fields: dt = self.try_convert_to_date(word) if dt: filters.extend([Q(**{field_name: dt}) for field_name in self.search_date_fields]) w_qs.append(reduce(operator.or_, filters)) qs = qs.filter(reduce(operator.and_, w_qs)) return qs.distinct()
def get_attrs(self, item_list, user): project_id = item_list[0].project_id user_lookups = [] for item in item_list: if item.key != 'sentry:user': continue if ':' not in item.value: continue try: user_lookups.append(Q(**parse_user_tag(item.value))) except ValueError: continue tag_labels = {} if user_lookups: tag_labels.update({ ('sentry:user', euser.tag_value): euser.get_label() for euser in EventUser.objects.filter( reduce(operator.or_, user_lookups), project_id=project_id, ) }) other_lookups = [ Q(key=i.key, value=i.value) for i in item_list if i.key != 'sentry:user' ] if other_lookups: tag_labels.update({ (t.key, t.value): t.get_label() for t in TagValue.objects.filter( reduce(operator.or_, other_lookups), project_id=project_id, ) }) result = {} for item in item_list: try: label = tag_labels[(item.key, item.value)] except KeyError: label = item.value result[item] = { 'name': label, } return result
def to_dict(for_lists, global_vars, data_dict): """ Construct a dict object from a list of ForList object :param for_lists: list of for_list :param global_vars: list of global vars to add :param data_dict: data from an orm-like object (with dot notation) :return: a dict representation of the ForList objects """ res = {} # The first level is a little bit special # Manage global variables for a in global_vars: a_list = a.split('.') tmp = res for i in a_list[:-1]: if not i in tmp: tmp[i] = {} tmp = tmp[i] tmp[a_list[-1]] = reduce(getattr, a_list[1:], data_dict[a_list[0]]) # Then manage for lists recursively for for_list in for_lists: it = for_list.name.split('.') tmp = res for i in it[:-1]: if not i in tmp: tmp[i] = {} tmp = tmp[i] if not it[-1] in tmp: tmp[it[-1]] = [] tmp = tmp[it[-1]] if not it[0] in data_dict: continue if len(it) == 1: loop = enumerate(data_dict[it[0]]) else: loop = enumerate(reduce(getattr, it[-1:], data_dict[it[0]])) for i, val in loop: new_data_dict = {for_list.var_from: val} # We append a new dict only if we need if len(tmp) <= i: tmp.append({}) # Call myself with new context, and get result tmp[i] = ForList.__recur_to_dict( for_list, new_data_dict, tmp[i] ) return res
def find_similar_users(self, user): from sentry.models import (OrganizationMemberTeam, Project) # limit to only teams user has opted into project_ids = list( Project.objects.filter( team__in=OrganizationMemberTeam.objects.filter( organizationmember__user=user, organizationmember__organization__project=self.project_id, is_active=True, ).values('team'), ).values_list('id', flat=True)[:1000] ) if not project_ids: return type(self).objects.none() filters = [] if self.email: filters.append(models.Q(email=self.email)) if self.ip_address: filters.append(models.Q(ip_address=self.ip_address)) if not filters: return type(self).objects.none() return type(self).objects.exclude(id=self.id).filter( reduce(or_, filters), project_id__in=project_ids, )
def _simple_fileslice(fileobj, sliceobj, shape, dtype, offset=0, order='C', heuristic=None): """ Read all data from `fileobj` into array, then slice with `sliceobj` The simplest possible thing; read all the data into the full array, then slice the full array. Parameters ---------- fileobj : file-like object implements ``read`` and ``seek`` sliceobj : object something that can be used to slice an array as in ``arr[sliceobj]`` shape : sequence shape of full array inside `fileobj` dtype : dtype object dtype of array inside `fileobj` offset : int, optional offset of array data within `fileobj` order : {'C', 'F'}, optional memory layout of array in `fileobj` heuristic : optional The routine doesn't use `heuristic`; the parameter is for API compatibility with :func:`fileslice` Returns ------- sliced_arr : array Array in `fileobj` as sliced with `sliceobj` """ fileobj.seek(offset) nbytes = reduce(operator.mul, shape) * dtype.itemsize bytes = fileobj.read(nbytes) new_arr = np.ndarray(shape, dtype, buffer=bytes, order=order) return new_arr[sliceobj]
def get_base_url_from_root(root): if root.base_url: # see :func:`.parse` for why we need to unquote base_url = unquote(root.base_url) else: base_url = root.base_url return reduce(urljoin, base_href(root)[:1], base_url)
def _read_compact_string(self, code): if code >= b'\x30': len_bytes = six.b(chr(ord(code) - 0x30)) + self._read(1) else: len_bytes = b'\x00' + code length = unpack('>H', len_bytes)[0] bytes = [] while length > 0: byte = self._read(1) if b'\x00' <= byte <= b'\x7F': bytes.append(byte) elif b'\xC2' <= byte <= b'\xDF': bytes.append(byte + self._read(1)) elif b'\xE0' <= byte <= b'\xEF': bytes.append(byte + self._read(2)) elif b'\xF0' <= byte <= b'\xF4': bytes.append(byte + self._read(3)) length -= 1 try: return reduce(operator.add, bytes, b'').decode('utf-8') except UnicodeDecodeError: if six.PY2: raise # We're possibly dealing with surrogate pairs in Python 3 return self._decode_byte_array(bytes)
def build_clause(query, attributes): # Iterate through each query segment. clause = None last = None for seg in query.segments: # Get the attribute in question. attribute = attributes[seg.path[0]] # Replace the initial path segment with the expanded # attribute path. seg.path[0:1] = attribute.path.split('.') # Boolean's should use `exact` rather than `iexact`. if attribute.type is bool: op = '__exact' else: op = OPERATOR_MAP[seg.operator] # Build the path from the segment. path = '__'.join(seg.path) + op # Construct a Q-object from the segment. q = reduce(operator.or_, map(lambda x: Q((path, x)), map(attribute.try_clean, seg.values))) # Combine the segment with the last. clause = last.combinator(clause, q) if last is not None else q last = seg # Return the constructed clause. return clause
def import_object(module_name, expr): mod = __import__(module_name) mod = reduce(getattr, module_name.split('.')[1:], mod) globals = builtins if not isinstance(globals, dict): globals = globals.__dict__ return eval(expr, globals, mod.__dict__)
def _read_binary(self, code, length=None): chunks = [] while True: if b'\x20' <= code <= b'\x2F': # binary data length 0-16 length = ord(code) - 0x20 elif b'\x34' <= code <= b'\x37': # binary data length 0-1023 len_b1 = (ord(code) - 0x34) << 8 len_b0 = ord(self._read(1)) length = len_b0 + len_b1 else: len_bytes = self._read(2) length = unpack('>H', len_bytes)[0] if length == 0: break chunks.append(self._read(length)) if code != b'A': break length = None code = self._read(1) return Binary(reduce(operator.add, chunks, b''))
def foreignkey_autocomplete(self, request): """ Searches in the fields of the given related model and returns the result as a simple string to be used by the jQuery Autocomplete plugin """ query = request.GET.get('q', None) app_label = request.GET.get('app_label', None) model_name = request.GET.get('model_name', None) search_fields = request.GET.get('search_fields', None) object_pk = request.GET.get('object_pk', None) try: to_string_function = self.related_string_functions[model_name] except KeyError: if six.PY3: to_string_function = lambda x: x.__str__() else: to_string_function = lambda x: x.__unicode__() if search_fields and app_label and model_name and (query or object_pk): def construct_search(field_name): # use different lookup methods depending on the notation if field_name.startswith('^'): return "%s__istartswith" % field_name[1:] elif field_name.startswith('='): return "%s__iexact" % field_name[1:] elif field_name.startswith('@'): return "%s__search" % field_name[1:] else: return "%s__icontains" % field_name model = apps.get_model(app_label, model_name) queryset = model._default_manager.all() data = '' if query: for bit in query.split(): or_queries = [models.Q(**{construct_search(smart_str(field_name)): smart_str(bit)}) for field_name in search_fields.split(',')] other_qs = QuerySet(model) other_qs.query.select_related = queryset.query.select_related other_qs = other_qs.filter(reduce(operator.or_, or_queries)) queryset = queryset & other_qs additional_filter = self.get_related_filter(model, request) if additional_filter: queryset = queryset.filter(additional_filter) if self.autocomplete_limit: queryset = queryset[:self.autocomplete_limit] data = ''.join([six.u('%s|%s\n') % (to_string_function(f), f.pk) for f in queryset]) elif object_pk: try: obj = queryset.get(pk=object_pk) except Exception: # FIXME: use stricter exception checking pass else: data = to_string_function(obj) return HttpResponse(data, content_type='text/plain') return HttpResponseNotFound()
def __vertex_raw_labels(self, return_name=False): raw_names = { -1: 'Unknown', 0: 'Architecture', 1: 'Computer Network', 2: 'Computer Security', 3: 'Data Mining', 4: 'Theory', 5: 'Graphics' } if self.__vertex_raw_labels_cache is not None: if return_name: return self.__vertex_raw_labels_cache, raw_names else: return self.__vertex_raw_labels_cache # These are conferences that are to merged confdata = [ [ 'ASPLOS|Architectural Support for Programming Languages and Operating Systems', 'FAST|Conference on File and Storage Technologies', 'HPCA|High-Performance Computer Architecture', 'ISCA|Symposium on Computer Architecture', 'MICRO|MICRO', 'USENIX ATC|USENIX Annul Technical Conference', 'PPoPP|Principles and Practice of Parallel Programming' ], [ 'MOBICOM|Mobile Computing and Networking Transactions on Networking', 'SIGCOMM|applications, technologies, architectures, and protocols for computer communication', 'INFOCOM|Computer Communications' ], [ 'CCS|Computer and Communications Security', 'NDSS|Network and Distributed System Security', # 'CRYPTO|International Cryptology Conference', # 'EUROCRYPT|European Cryptology Conference', 'S\&P|Symposium on Security and Privacy', 'USENIX Security|Usenix Security Symposium' ], [ 'SIGMOD|Conference on Management of Data', 'SIGKDD|Knowledge Discovery and Data Mining', 'SIGIR|Research on Development in Information Retrieval', 'VLDB|Very Large Data Bases', 'ICDE|Data Engineering' ], [ 'STOC|ACM Symposium on Theory of Computing', 'FOCS|Symposium on Foundations of Computer Science', 'LICS|Symposium on Logic in Computer Science', 'CAV|Computer Aided Verification' ], [ # 'ACM MM|Multimedia', 'SIGGRAPH|SIGGRAPH Annual Conference', 'IEEE VIS|Visualization Conference', 'VR|Virtual Reality' ], # ['AAAI|AAAI Conference on Artificial Intelligence', # 'CVPR|Computer Vision and Pattern Recognition', # 'ICCV|International Conference on Computer Vision', # 'ICML|International Conference on Machine Learning', # 'IJCAI|International Joint Conference on Artificial Intelligence', # 'NIPS|Annual Conference on Neural Information Processing Systems', # 'ACL|Annual Meeting of the Association for Computational Linguistics'] ] # confdata records the representing conferences for each field confdata = {n: i for i, arr in enumerate(confdata) for n in arr} for k in confdata.keys(): sname, lname = k.split('|') confdata[k] = [ confdata[k], re.compile(sname), re.compile(lname, re.I) ] # conffeat is a list of matrices, each of them is a user-conference matrix for participation information conffeat = [ self.__data['conf_feat'][y] for y in range(self.localunit, self.localunit + self.nunits) ] # names of conferences in the user-conference matrix, in global indices (before filtering) conffeat_names = self.__data['conf_names'] confmap = self.__data['confmap'] # maps conference names from global indices to their original names conffeat_names = [confmap[c] for c in conffeat_names] # we use theory conferences because it is more independent rawlb = [] for i in range(self.nsteps): startunit = self._time2unit(self.step2time(i + self.localstep)) endunit = startunit + self.stepsize relstartunit, relendunit = startunit - self.localunit, endunit - self.localunit print( "generating samples for years from {} to {}, i.e. featidx from {} to {}" .format(startunit, endunit - 1, relstartunit, relendunit - 1)) curconffeat = reduce( lambda x, y: x + y, conffeat[relstartunit:relendunit], np.zeros(conffeat[relstartunit].shape, dtype=conffeat[relstartunit].dtype)).A rawlb.append( self.__label_vertices(curconffeat, conffeat_names, confdata)) print("{}/{} positive samples at step {}".format( np.sum(rawlb[-1] == 1), len(rawlb[-1]), i + self.localstep)) rawlb = np.vstack(rawlb) self.__vertex_raw_labels_cache = rawlb if return_name: return self.__vertex_raw_labels_cache, raw_names else: return self.__vertex_raw_labels_cache
def compile(self, strength=5.0): """Returns the compiled :class:`Model`. This method reduces the degree of the expression if the degree is higher than 2, and convert it into :class:`.Model` which has information about QUBO. Args: strength (float): The strength of the reduction constraint. Insufficient strength can result in the binary quadratic model not having the same minimizations as the polynomial. Returns: :class:`Model`: The model compiled from the :class:`.Express`. Examples: In this example, there are higher order terms :math:`abc` and :math:`abd`. It is decomposed as [[``a*b``, ``c``], ``d``] hierarchically and converted into QUBO. By calling :func:`to_qubo()` of the :obj:`model`, we get the resulting QUBO. >>> from pyqubo import Binary >>> a, b, c, d = Binary("a"), Binary("b"), Binary("c"), Binary("d") >>> model = (a*b*c + a*b*d).compile() >>> pprint(model.to_qubo()) # doctest: +SKIP ({('a', 'a'): 0.0, ('a', 'a*b'): -10.0, ('a', 'b'): 5.0, ('a*b', 'a*b'): 15.0, ('a*b', 'b'): -10.0, ('a*b', 'c'): 1.0, ('a*b', 'd'): 1.0, ('b', 'b'): 0.0, ('c', 'c'): 0, ('d', 'd'): 0}, 0.0) """ def compile_param_if_express(val): if isinstance(val, Express): return val._compile_param() else: return val # Constraint for AND(multiplier, multiplicand) = product def binary_product(multiplier, multiplicand, product, weight): return { BinaryProd({product}): 3.0 * weight, BinaryProd({multiplicand, product}): -2.0 * weight, BinaryProd({multiplier, product}): -2.0 * weight, BinaryProd({multiplier, multiplicand}): weight } # When the label contains PROD_SYM, elements of products should be sorted # such that the resulting label is uniquely determined. def normalize_label(label): s = Express.PROD_SYM if s in label: return s.join(sorted(label.split(s))) else: return label # Expand the expression to polynomial expanded, constraints, penalties = Express._expand(self) expanded = Express._merge_term(expanded, penalties) # Make polynomials quadratic offset = 0.0 pubo = {} for term_key, value in expanded.items(): if term_key.is_constant(): offset = value else: pubo[tuple(term_key.keys)] = value bqm = dimod.make_quadratic(pubo, strength, dimod.BINARY) bqm_qubo, bqm_offset = bqm.to_qubo() # Extracts product constrains product_consts = {} for (a, b), v in bqm.info['reduction'].items(): prod = normalize_label(v['product']) a = normalize_label(a) b = normalize_label(b) product_consts["AND({},{})={}".format(a, b, prod)]\ = binary_product(a, b, prod, strength) # Normalize labels and compile values of the QUBO compiled_qubo = {} for (label1, label2), value in bqm_qubo.items(): norm_label1 = normalize_label(label1) norm_label2 = normalize_label(label2) # Sort the tuple of labels such that the created key is uniquely determined. if norm_label2 > norm_label1: label_key = (norm_label1, norm_label2) else: label_key = (norm_label2, norm_label1) # Compile values of the QUBO compiled_qubo[label_key] = compile_param_if_express(value) compiled_qubo = CompiledQubo( compiled_qubo, compile_param_if_express(offset + bqm_offset)) # compile values of constraint compiled_constraints = {} for label, constraint in Express._merge_dict(constraints, product_consts).items(): compiled_constraints[label] =\ CompiledConstraint({prod: compile_param_if_express(value) for prod, value in constraint.items()}) # Merge structures uniq_variables = Express._unique_vars(self) structure = reduce(Express._merge_dict, [var.structure for var in uniq_variables]) return Model(compiled_qubo, structure, compiled_constraints)
def best(self): grid = self.grid obj_task = self.task_group.tasks[self.objective['name']] obj_model = self.models[self.objective['name']] # If unconstrained if self.numConstraints() == 0: # Compute the GP mean obj_mean, obj_var = obj_model.function_over_hypers( obj_model.predict, grid) # find the min and argmin of the GP mean current_best_location = grid[np.argmin(obj_mean), :][None] best_ind = np.argmin(obj_mean) current_best_value = obj_mean[best_ind] std_at_best = np.sqrt(obj_var[best_ind]) # un-normalize the min of mean to original units unnormalized_best_value = obj_task.unstandardize_mean( obj_task.unstandardize_variance(current_best_value)) unnormalized_std_at_best = obj_task.unstandardize_variance( std_at_best) # Print out the minimum according to the model if not self.quiet: sys.stderr.write( '\nMinimum expected objective value under model ' 'is %.5f (+/- %.5f), at location:\n' % (unnormalized_best_value, unnormalized_std_at_best)) self.task_group.paramify_and_print( self.task_group.from_unit(current_best_location).flatten(), left_indent=16, indent_top_row=True) # Compute the best value seen so far vals = self.task_group.values[self.objective['name']] inps = self.task_group.inputs best_observed_value = np.min(vals) best_observed_location = inps[np.argmin(vals), :][None] # Don't need to un-normalize inputs here because these are the raw inputs if not self.quiet: sys.stderr.write( '\nMinimum of observed values is %f, at location:\n' % best_observed_value) self.task_group.paramify_and_print( best_observed_location.flatten(), left_indent=16, indent_top_row=True) else: mc = self.probabilistic_constraint(grid) if not np.any(mc): # P-con is violated everywhere # Compute the product of the probabilities, and return None for the current best value probs = reduce( lambda x, y: x * y, [self.confidence(c, grid) for c in self.constraints], np.ones(grid.shape[0])) best_probs_ind = np.argmax(probs) best_probs_location = grid[best_probs_ind, :][None] # TODO -- could use BFGS for this (unconstrained) optimization as well -- everytime for min of mean if not self.quiet: sys.stderr.write('\nNo feasible region found (yet).\n') sys.stderr.write( 'Maximum probability of satisfying constraints = %f\n' % np.max(probs)) sys.stderr.write('At location: ') self.task_group.paramify_and_print( self.task_group.from_unit( best_probs_location).flatten(), left_indent=16) return None, best_probs_location # A feasible region has been found # Compute GP mean and find minimum mean, var = obj_model.function_over_hypers(obj_model.predict, grid) valid_mean = mean[mc] valid_var = var[mc] best_ind = np.argmin(valid_mean) current_best_location = (grid[mc])[best_ind, :][None] ind = np.argmin(valid_mean) current_best_value = valid_mean[ind] std_at_best = np.sqrt(valid_var[ind]) unnormalized_best = obj_task.unstandardize_mean( obj_task.unstandardize_variance(current_best_value)) unnormalized_std_at_best = obj_task.unstandardize_variance( std_at_best) # not used -- not quite # right to report this -- i mean there is uncertainty in the constraints too # this is the variance at that location, not the standard deviation of the minimum... # not sure if this distinction is a big deal if not self.quiet: sys.stderr.write( '\nMinimum expected objective value satisfying constraints w/ high prob: %f\n' % unnormalized_best) sys.stderr.write('At location: ') self.task_group.paramify_and_print( self.task_group.from_unit(current_best_location).flatten(), left_indent=16) # Compute the best value seen so far with np.errstate(invalid='ignore'): all_constraints_satisfied = np.all(np.greater( np.array( [x.values for x in self.task_group.tasks.values()]), 0), axis=0) if not np.any(all_constraints_satisfied): if not self.quiet: sys.stderr.write( 'No observed result satisfied all constraints.\n') else: inps = self.task_group.inputs vals = self.task_group.values[self.objective['name']] # get rid of those that violate constraints vals[np.logical_not(all_constraints_satisfied)] = np.max(vals) # get rid of NaNs -- set them to biggest not-nan value, then they won't be the minimum vals[np.isnan(vals)] = np.max(vals[np.logical_not( np.isnan(vals))]) best_observed_value = np.min(vals) best_observed_location = inps[np.argmin(vals), :][None] # Don't need to un-normalize inputs here because these are the raw inputs if not self.quiet: sys.stderr.write( '\nBest observed values satisfying constraints is %f, at location:\n' % best_observed_value) self.task_group.paramify_and_print( best_observed_location.flatten(), left_indent=16, indent_top_row=True) # Return according to model, not observed return current_best_value, current_best_location
def acquisition_function(self, cand, current_best, compute_grad=True): obj_model = self.models[self.objective['name']] # If unconstrained, just compute regular ei if self.numConstraints() == 0: return compute_ei(obj_model, cand, ei_target=current_best, compute_grad=compute_grad) if cand.ndim == 1: cand = cand[None] N_cand = cand.shape[0] ############## ---------------------------------------- ############ ############## ############ ############## Part that depends on the objective ############ ############## ############ ############## ---------------------------------------- ############ if current_best is None: ei = 1. ei_grad = 0. else: target = current_best # Compute the predictive mean and variance if not compute_grad: ei = compute_ei(obj_model, cand, target, compute_grad=compute_grad) else: ei, ei_grad = compute_ei(obj_model, cand, target, compute_grad=compute_grad) ############## ---------------------------------------- ############ ############## ############ ############## Part that depends on the constraints ############ ############## ############ ############## ---------------------------------------- ############ # Compute p(valid) for ALL constraints p_valid, p_grad = list(), list() for c in self.constraints: if compute_grad: pv, pvg = self.models[c].pi(cand, compute_grad=True) p_valid.append(pv) p_grad.append(pvg) else: p_valid.append(self.models[c].pi(cand, compute_grad=False)) p_valid_prod = reduce(np.multiply, p_valid, np.ones(N_cand)) # To compute the gradient, need to do the chain rule for the product of N factors if compute_grad: p_grad_prod = np.zeros(p_grad[0].shape) for i in xrange(self.numConstraints()): pg = p_grad[i] for j in xrange(self.numConstraints()): if j == i: continue pg *= p_valid[j] p_grad_prod += pg # multiply that gradient by all other pv's (this might be numerically disasterous if pv=0...) ############## ---------------------------------------- ############ ############## ############ ############## Combine the two parts (obj and con) ############ ############## ############ ############## ---------------------------------------- ############ acq = ei * p_valid_prod if not compute_grad: return acq else: return acq, ei_grad * p_valid_prod + p_grad_prod * ei
def __hash__(self): return reduce(xor, [hash(term) for term in self.terms])
def __call__(self, sequence): return reduce(lambda x, operation: operation(x), self.operations, sequence)
def decode_solution(self, solution, vartype, feed_dict=None): """Returns decoded solution. Args: solution (list[bit]/dict[label, bit]/dict[index, bit]): The solution returned from solvers. vartype (:class:`dimod.Vartype`/str/set, optional): Variable type of the solution. Accepted input values: * :class:`.Vartype.SPIN`, ``'SPIN'``, ``{-1, 1}`` * :class:`.Vartype.BINARY`, ``'BINARY'``, ``{0, 1}`` feed_dict (dict[str, float]): Specify the placeholder values. Returns: tuple(dict, dict, float): Tuple of the decoded solution, broken constraints and energy. Structure of decoded_solution is defined by :obj:`structure`. """ def put_value_with_keys(dict_body, keys, value): for key in keys[:-1]: if key not in dict_body: dict_body[key] = {} dict_body = dict_body[key] dict_body[keys[-1]] = value decoded_solution = {} dict_bin_solution = self._parse_solution(solution, vartype) for label, bit in dict_bin_solution.items(): if label in self.structure: if vartype == dimod.SPIN: out_value = 2 * bit - 1 elif vartype == dimod.BINARY: out_value = bit put_value_with_keys(decoded_solution, self.structure[label], out_value) # Check satisfaction of constraints broken_const = {} for label, const in self.constraints.items(): energy = const.energy(dict_bin_solution, feed_dict) if energy > 0.0: result_value = { var: dict_bin_solution[var] for var in reduce( or_, [k.keys for k in const.polynomial.keys()]) } broken_const[label] = { "result": result_value, "penalty": energy } elif energy < 0.0: raise ValueError( "The energy of the constraint \"{label}\" is {energy}." "But an energy of constraints should not be negative.". format(label=label, energy=energy)) problem_energy = self.energy(dict_bin_solution, dimod.BINARY, feed_dict) return decoded_solution, broken_const, problem_energy
def __exit__(self, type, value, traceback): """Restore previous value.""" config_obj = reduce(dict.__getitem__, self.nested_keys[:-1], current_app.config) config_obj[self.nested_keys[-1]] = self.prev_value
def cudandarray_to_garray(x, copyif=False): """ take a CudaNdarray and return a gnumpy.garray object. :type x: CudaNdarray :param x: The array to transform to gnumpy.garray. :type copyif: bool :param copyif: If False, raise an error if x is not c contiguous. If it is c contiguous, we return a GPUArray that share the same memory region as x. If True, copy x if it is no c contiguous, so the return won't shape the same memory region. If c contiguous, the return will share the same memory region. We need to do this as GPUArray don't fully support strided memory. :return type: cudamat.CUDAMatrix """ if not isinstance(x, cuda.CudaNdarray): raise ValueError( "We can transfer only CudaNdarray to cudamat.CUDAMatrix") else: # Check if it is c contiguous size = 1 c_contiguous = True for i in range(x.ndim - 1, -1, -1): if x.shape[i] == 1: continue if x._strides[i] != size: c_contiguous = False break size *= x.shape[i] if not c_contiguous: if copyif: x = x.copy() else: raise ValueError( "We where asked to don't copy memory, but the memory is not c contiguous." ) # Now x is always c contiguous. # the next step is to create a CUDAMatrix object. We do so by first creating # a cudamat object with no data_host. cm_mat = cudamat.cudamat() cm_mat.size[0] = reduce(lambda x, y: x * y, x.shape, 1) cm_mat.size[1] = 1 cm_mat.on_host = 0 cm_mat.on_device = 1 cm_mat.is_trans = 0 cm_mat.owns_data = 0 # <-- note: cm_mat dosen't owe the data; x does. So x will delete it. # x.gpudata is a long. We need a pointer to a float. cast. import ctypes cm_mat.data_device = ctypes.cast(x.gpudata, ctypes.POINTER(ctypes.c_float)) px = cudamat.CUDAMatrix(cm_mat) px._base = x # x won't be freed if the cudamat object isn't freed. px.mat_on_host = False # let cudamat know that we don't have a numpy # array attached. # Note how gnumpy tracks its cudamat objects: it moves things to the # _cmsReuseCache when the gnumpy array is deleted, thus the arrays # returned by theano will never be deleted. # However, if the garray thinks that the object is a view, then it won't # move the _base to the _cmsResueCache; so the cudamat object will be deleted, # and we won't overpump the world with memory. _is_alias_of = ___const_garray ans = gnumpy.garray(px, x.shape, _is_alias_of) return ans
def __enter__(self): """Save previous value and swap it with the new.""" config_obj = reduce(dict.__getitem__, self.nested_keys[:-1], current_app.config) self.prev_value = config_obj[self.nested_keys[-1]] config_obj[self.nested_keys[-1]] = self.new_value
def last(seq): return reduce(lambda l, r: r, seq)
def standard_jvp2(jvprules, primitive, primals, tangents, **params): val_out = primitive.bind(*primals, **params) tangents_out = (rule(t, val_out, *primals, **params) for rule, t in zip(jvprules, tangents) if rule is not None and t is not zero) return val_out, reduce(add_tangents, tangents_out, zero)
def cdbhash(s): return reduce(lambda h, c: (((h << 5) + h) ^ ord(c)) & 0xffffffff, s, 5381)
def encode_tuple(self, obj): ref = self.add_ref(obj) if ref: return ref encoded = reduce(operator.add, map(self.encode, obj), b'') return pack('>2cl', b'V', b'l', len(obj)) + encoded + b'z'
def join_components(cls, components): return reduce(posixpath.join, SeqUtils.filter_true(components))
def _jug_reduce(reducer, inputs): from six.moves import reduce reducer = _get_function(reducer) return reduce(reducer, chain(inputs))
def __call__(self, environ, start_response): # specify that we want the parsed dataset environ['x-wsgiorg.want_parsed_response'] = True req = Request(environ) projection, selection = parse_ce(req.query_string) # check if there are any functions calls in the request called = (any(s for s in selection if FUNCTION.match(s)) or any(p for p in projection if isinstance(p, string_types))) # ignore DAS requests and requests without functions path, response = req.path.rsplit('.', 1) if response == 'das' or not called: return self.app(environ, start_response) # apply selection without any function calls req.query_string = '&'.join(s for s in selection if not FUNCTION.match(s)) res = req.get_response(self.app) # get the dataset method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False) if not method: raise ServerError("Unable to call server-side function!") dataset = method(DatasetType) # apply selection containing server-side functions selection = (s for s in selection if FUNCTION.match(s)) for expr in selection: if RELOP.search(expr): call, op, other = RELOP.split(expr) op = { '<': operator.lt, '>': operator.gt, '!=': operator.ne, '=': operator.eq, '>=': operator.ge, '<=': operator.le, '=~': lambda a, b: re.match(b, a), }[op] other = ast.literal_eval(other) else: call, op, other = expr, operator.eq, 1 # evaluate the function call sequence = eval_function(dataset, call, self.functions) # is this an inplace call? for var in walk(dataset, SequenceType): if sequence is var: break else: # get the data from the resulting variable, and use it to # constrain the original dataset child = list(sequence.children())[0] data = np.fromiter(child.data, child.dtype) if data.dtype.char == "S": valid = np.array( list(map(lambda v: op(str(v), str(other)), data)), bool) else: valid = op(data, other) for sequence in walk(dataset, SequenceType): sequence.data = np.rec.fromrecords( [tuple(row) for row in sequence], names=sequence.keys())[valid] # now apply projection if projection: projection = fix_shorthand(projection, dataset) base = [p for p in projection if not isinstance(p, string_types)] func = [p for p in projection if isinstance(p, string_types)] # apply non-function projection out = apply_projection(base, dataset) # apply function projection for call in func: var = eval_function(dataset, call, self.functions) for child in walk(var): parent = reduce(operator.getitem, [out] + child.id.split('.')[:-1]) if child.name not in parent.keys(): parent[child.name] = child break dataset = out # Return the original response (DDS, DAS, etc.) path, response = req.path.rsplit('.', 1) res = BaseHandler.responses[response](dataset) return res(environ, start_response)
def _expand(exp): """Expand the expression hierarchically into dict format. For example, ``2*Binary(a)*Binary(b) + 1`` is represented as dict format ``{BinaryProd(ab): 2.0, CONST_TERM_KEY: 1.0}``. Let's see how this dict is created step by step. First, focus on `2*Binary(a)*Binary(b)` in which each expression is expanded as _expand(Num(2)) # => {CONST_TERM_KEY: 2.0} _expand(Binary("a")) # => {BinaryProd(a): 1.0} _expand(Binary("b")) # => {BinaryProd(b): 1.0} respectively. :class:`Mul` combines the expression in the following way _expand(Mul(Num(2), Binary("a"))) # => {BinaryProd(a): 2.0} _expand(Mul(Binary("b"), Mul(Num(2), Binary("a")))) # => {BinaryProd(ab): 2.0} Finally, :class:`Add` combines the ``{BinaryProd(ab): 2.0}`` and `{CONST_TERM_KEY: 1.0}`, and we get the final form {BinaryProd(ab): 2.0, CONST_TERM_KEY: 1.0} Args: exp (:class:`Express`): Input expression. Returns: tuple(expanded_terms, constraints): tuple of expanded_terms and constrains. Expanded expression takes the form ``dict[:class:`BinaryProd`, value]``. Constraints takes the form of ``dict[label, expanded_terms]``. """ if isinstance(exp, WithPenalty): val, val_const, val_penalty = Express._expand(exp.express) penalty, penalty_const, penalty_penalty = Express._expand( exp.penalty) combined_penalty = Express._merge_term(val_penalty, penalty_penalty) combined_penalty = Express._merge_term(combined_penalty, penalty) return val, Express._merge_dict_update( val_const, penalty_const), combined_penalty elif isinstance(exp, Add): left, left_const, left_p = Express._expand(exp.left) right, right_const, right_p = Express._expand(exp.right) result = Express._merge_term(right, left) return result, Express._merge_dict_update(left_const, right_const),\ Express._merge_term(left_p, right_p) elif isinstance(exp, AddList): expanded, const, penalty = reduce( lambda arg1, arg2: (Express._merge_term(arg1[0], arg2[ 0]), Express._merge_dict_update(arg1[1], arg2[1]), Express._merge_term(arg1[2], arg2[2])), [Express._expand(term) for term in exp.terms]) return expanded, const, penalty elif isinstance(exp, Mul): left, left_const, left_p = Express._expand(exp.left) right, right_const, right_p = Express._expand(exp.right) expanded_terms = defaultdict(float) for k1, v1 in left.items(): for k2, v2 in right.items(): merged_key = BinaryProd.merge_term_key(k1, k2) expanded_terms[merged_key] += v1 * v2 return expanded_terms, Express._merge_dict_update(left_const, right_const),\ Express._merge_term(left_p, right_p) elif isinstance(exp, Placeholder): expanded_terms = defaultdict(float) expanded_terms[Express.CONST_TERM_KEY] = exp return expanded_terms, {}, {} elif isinstance(exp, Constraint): child, child_const, child_p = Express._expand(exp.child) child_const[exp.label] = copy.copy(child) return child, child_const, child_p elif isinstance(exp, Num): terms = defaultdict(float) terms[Express.CONST_TERM_KEY] = exp.value return terms, {}, {} elif isinstance(exp, Binary): terms = defaultdict(float) terms[BinaryProd({exp.label})] = 1.0 return terms, {}, {} elif isinstance(exp, Spin): terms = defaultdict(float) terms[BinaryProd({exp.label})] = 2.0 terms[Express.CONST_TERM_KEY] = -1.0 return terms, {}, {} elif isinstance(exp, UserDefinedExpress): return Express._expand(exp.express) else: raise TypeError("Unexpected input type {}.".format( type(exp))) # pragma: no cover
def get_group_event_ids(self, project_id, group_id, environment_id, tags): # NOTE: `environment_id=None` needs to be filtered differently in this method. # EventTag never has NULL `environment_id` fields (individual Events always have an environment), # and so `environment_id=None` needs to query EventTag for *all* environments (except, ironically # the aggregate environment). if environment_id is None: # filter for all 'real' environments exclude = {'_key__environment_id': AGGREGATE_ENVIRONMENT_ID} env_filter = {} else: exclude = {} env_filter = {'_key__environment_id': environment_id} tagvalue_qs = TagValue.objects.filter( reduce(or_, (Q(_key__key=k, _key__status=TagKeyStatus.VISIBLE, value=v) for k, v in six.iteritems(tags))), project_id=project_id, _key__project_id=project_id, **env_filter ) if exclude: tagvalue_qs = tagvalue_qs.exclude(**exclude) tagvalue_qs = tagvalue_qs.values_list('_key_id', 'id', '_key__key', 'value') tagvalues = defaultdict(list) for key_id, value_id, key, value in tagvalue_qs: tagvalues[(key, value)].append((key_id, value_id)) tagvalues = dict(tagvalues) try: # ensure all key/value pairs were found tag_lookups = [tagvalues[(k, v)] for k, v in six.iteritems(tags)] # [[(key0, value0), (key1, value1)], ...] except KeyError: # one or more tags were invalid, thus the result should be an empty # set return [] # Django doesnt support union, so we limit results and try to find # reasonable matches # get initial matches to start the filter kv_pairs = tag_lookups.pop() matches = list( EventTag.objects.filter( reduce(or_, (Q(key_id=k, value_id=v) for k, v in kv_pairs)), project_id=project_id, group_id=group_id, ).values_list('event_id', flat=True)[:1000] ) # for each remaining tag, find matches contained in our # existing set, pruning it down each iteration for kv_pairs in tag_lookups: matches = list( EventTag.objects.filter( reduce(or_, (Q(key_id=k, value_id=v) for k, v in kv_pairs)), project_id=project_id, group_id=group_id, event_id__in=matches, ).values_list('event_id', flat=True)[:1000] ) if not matches: return [] return matches
def any(cls): return reduce(operator.or_, (x.value for x in cls))
def test_huge_elemwise_fusion(): """ Test the the GpuElemwise fusion work correctly We check that we fuse one node with part of its input in case their is too many inputs and that would make it bust the 256 bytes limits. """ shape = (2, 3, 4, 5, 6) ttype = tensor.tensor(dtype='float32', broadcastable=(False, ) * len(shape)) gpu_ptr_size = theano.sandbox.cuda.opt.get_device_type_sizes( )['gpu_ptr_size'] if gpu_ptr_size == 8: nb_in = 7 len_topo = 10 elif gpu_ptr_size == 4: nb_in = 8 len_topo = 11 else: raise Exception("Unexpected value for gpu_ptr_size", gpu_ptr_size) vars = [tensor.tanh(ttype) for x in range(nb_in)] f = pfunc(vars, [reduce(operator.sub, vars)], mode=mode_with_gpu) topo = f.maker.fgraph.toposort() assert len(topo) == len_topo assert sum([isinstance(node.op, cuda.GpuElemwise) for node in topo]) == 2 assert isinstance(topo[-3].op.scalar_op, theano.scalar.basic.Sub) assert isinstance(topo[-2].op.scalar_op, theano.scalar.basic.Composite) # let debugmode catch errors gen = lambda: theano._asarray(numpy.random.rand(*shape), dtype='float32') f(*[gen() for i in range(nb_in)]) # Test the case where we can't put the computation on the gpu! their is too # many dimensions to the input to have 2 inputs to the op! shape = ( 1, 2, 3, 4, 5, 6, 7, 2, 2, 3, 2, 1, 2, 2, 2, ) ttype = tensor.tensor(dtype='float32', broadcastable=(False, ) * len(shape)) vars = [tensor.tanh(ttype) for x in range(7)] f = pfunc( vars, [vars[0] - vars[1] - vars[2] - vars[3] - vars[4] - vars[5] - vars[6]], mode=mode_with_gpu) topo = f.maker.fgraph.toposort() assert len(topo) == 1 assert sum([isinstance(node.op, cuda.GpuElemwise) for node in topo]) == 0 assert sum([isinstance(node.op, tensor.Elemwise) for node in topo]) == 1 # let debugmode catch errors gen = lambda: theano._asarray(numpy.random.rand(*shape), dtype='float32') f(gen(), gen(), gen(), gen(), gen(), gen(), gen()) def gen(shape): return theano._asarray(numpy.random.rand(*shape), dtype='float32') max_var = 16 # excluded for shape in [ (2, ), (2, 2), (2, 2, 2), (2, 2, 2, 2), (2, 2, 2, 2, 2), # 5d (2, 2, 2, 2, 2, 2), # (2, 2, 2, 2, 2, 2, 2), # (2, 2, 2, 2, 2, 2, 2, 2), # (2, 2, 2, 1, 1, 1, 1, 2, 2), # 9d ]: vals = [cuda.shared_constructor(gen(shape)) for x in range(max_var)] for use_tan in [True, False]: if use_tan: vars = [tensor.tanh(x) for x in vals] else: vars = vals for nb_var in range(1, max_var): out = reduce(lambda x, y: x + y, vars[:nb_var]) if not isinstance(out.type, CudaNdarrayType): out = cuda.gpu_from_host(out) f = pfunc([], [out], mode=mode_with_gpu) topo = f.maker.fgraph.toposort() # print shape, nb_var, use_tan, len(topo) assert (sum( [isinstance(node.op, cuda.GpuElemwise) for node in topo]) == len(topo) or (nb_var == 1 and use_tan is False)) assert sum([ isinstance(node.op, tensor.Elemwise) for node in topo ]) == 0 # let debugmode catch errors f()
def parse_marker_dict(marker_dict): op = marker_dict["op"] lhs = marker_dict["lhs"] rhs = marker_dict["rhs"] # This is where the spec sets for each side land if we have an "or" operator side_spec_list = [] side_markers_list = [] finalized_marker = "" # And if we hit the end of the parse tree we use this format string to make a marker format_string = "{lhs} {op} {rhs}" specset = SpecifierSet() specs = set() # Essentially we will iterate over each side of the parsed marker if either one is # A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset # Union the "and" specs, intersect the "or"s to find the most appropriate range if any(issubclass(type(side), Mapping) for side in (lhs, rhs)): for side in (lhs, rhs): side_specs = set() side_markers = set() if issubclass(type(side), Mapping): merged_side_specs, merged_side_markers = parse_marker_dict(side) side_specs.update(merged_side_specs) side_markers.update(merged_side_markers) else: marker = _ensure_marker(side) marker_parts = getattr(marker, "_markers", []) if marker_parts[0][0].value == "python_version": side_specs |= set(get_specset(marker_parts)) else: side_markers.add(str(marker)) side_spec_list.append(side_specs) side_markers_list.append(side_markers) if op == "and": # When we are "and"-ing things together, it probably makes the most sense # to reduce them here into a single PySpec instance specs = reduce(lambda x, y: set(x) | set(y), side_spec_list) markers = reduce(lambda x, y: set(x) | set(y), side_markers_list) if not specs and not markers: return specset, finalized_marker if markers and isinstance(markers, (tuple, list, Set)): finalized_marker = Marker(" and ".join([m for m in markers if m])) elif markers: finalized_marker = str(markers) specset._specs = frozenset(specs) return specset, finalized_marker # Actually when we "or" things as well we can also just turn them into a reduced # set using this logic now sides = reduce(lambda x, y: set(x) & set(y), side_spec_list) finalized_marker = " or ".join( [normalize_marker_str(m) for m in side_markers_list] ) specset._specs = frozenset(sorted(sides)) return specset, finalized_marker else: # At the tip of the tree we are dealing with strings all around and they just need # to be smashed together specs = set() if lhs == "python_version": format_string = "{lhs}{op}{rhs}" marker = Marker(format_string.format(**marker_dict)) marker_parts = getattr(marker, "_markers", []) _set = get_specset(marker_parts) if _set: specs |= set(_set) specset._specs = frozenset(specs) return specset, finalized_marker
def _jug_map_reduce(reducer, mapper, inputs): from six.moves import reduce reducer = _get_function(reducer) mapper = _get_function(mapper) return reduce(reducer, _jug_map(mapper, inputs))
def tritonast2arybo(e, use_exprs=True, use_esf=False, context=None): ''' Convert a subset of Triton's AST into Arybo's representation Args: e: Triton AST use_esf: use ESFs when creating the final expression context: dictionnary that associates Triton expression ID to arybo expressions Returns: An :class:`arybo.lib.MBAVariable` object ''' children_ = e.getChildren() children = (tritonast2arybo(c, use_exprs, use_esf, context) for c in children_) reversed_children = (tritonast2arybo(c, use_exprs, use_esf, context) for c in reversed(children_)) Ty = e.getType() if Ty == TAstN.ZX: n = next(children) v = next(children) n += v.nbits if n == v.nbits: return v return v.zext(n) if Ty == TAstN.SX: n = next(children) v = next(children) n += v.nbits if n == v.nbits: return v return v.sext(n) if Ty == TAstN.INTEGER: return e.getInteger() if Ty == TAstN.BV: cst = next(children) nbits = next(children) if use_exprs: return EX.ExprCst(cst, nbits) else: return _get_mba(nbits, use_esf).from_cst(cst) if Ty == TAstN.EXTRACT: last = next(children) first = next(children) v = next(children) return v[first:last + 1] if Ty == TAstN.CONCAT: if use_exprs: return EX.ExprConcat(*list(reversed_children)) else: return flatten(reversed_children) if Ty == TAstN.VARIABLE: name = e.getSymbolicVariable().getName() ret = _get_mba(e.getBitvectorSize(), use_esf).var(name) if use_exprs: ret = EX.ExprBV(ret) return ret if Ty == TAstN.REFERENCE: if context is None: raise ValueError( "reference node without context can't be resolved") id_ = e.getSymbolicExpression().getId() ret = context.get(id_, None) if ret is None: raise ValueError("expression id %d not found in context" % id_) return ret if Ty == TAstN.LET: # Alias # djo: "c'est pas utilise osef" raise ValueError("unsupported LET operation") # Logical/arithmetic shifts shifts = { TAstN.BVASHR: lambda a, b: a.ashr(b), TAstN.BVLSHR: lambda a, b: a.lshr(b), TAstN.BVSHL: operator.lshift, TAstN.BVROL: lambda x, n: x.rol(n), TAstN.BVROR: lambda x, n: x.ror(n) } shift = shifts.get(Ty, None) if not shift is None: v = next(children) n = next(children) return shift(v, n) # Unary op unops = { TAstN.BVNOT: lambda x: ~x, TAstN.LNOT: lambda x: ~x, TAstN.BVNEG: operator.neg } unop = unops.get(Ty, None) if not unop is None: return unop(next(children)) binops = { TAstN.BVADD: operator.add, TAstN.BVSUB: operator.sub, TAstN.BVAND: operator.and_, TAstN.BVOR: operator.or_, TAstN.BVXOR: operator.xor, TAstN.BVMUL: operator.mul, TAstN.BVNAND: lambda x, y: ~(x & y), TAstN.BVNOR: lambda x, y: ~(x | y), TAstN.BVXNOR: lambda x, y: ~(x ^ y), TAstN.BVUDIV: lambda x, y: x.udiv(y), TAstN.BVSDIV: lambda x, y: x.sdiv(y), TAstN.BVUREM: lambda x, y: x.urem(y), TAstN.BVSREM: lambda x, y: x.srem(y), TAstN.LAND: operator.and_, TAstN.LOR: operator.or_ } binop = binops.get(Ty, None) if not binop is None: return reduce(binop, children) # Logical op lops = { TAstN.EQUAL: lambda x, y: EX.ExprCmpEq(x, y), TAstN.DISTINCT: lambda x, y: EX.ExprCmpNeq(x, y), TAstN.BVUGE: lambda x, y: EX.ExprCmpGte(x, y, False), TAstN.BVUGT: lambda x, y: EX.ExprCmpGt(x, y, False), TAstN.BVULE: lambda x, y: EX.ExprCmpLte(x, y, False), TAstN.BVULT: lambda x, y: EX.ExprCmpLt(x, y, False), TAstN.BVSGE: lambda x, y: EX.ExprCmpGte(x, y, True), TAstN.BVSGT: lambda x, y: EX.ExprCmpGt(x, y, True), TAstN.BVSLE: lambda x, y: EX.ExprCmpLte(x, y, True), TAstN.BVSLT: lambda x, y: EX.ExprCmpLt(x, y, True) } lop = lops.get(Ty, None) if not lop is None: return reduce(lop, children) # Conditional if Ty != TAstN.ITE: raise ValueError("unsupported node type %s" % str(Ty)) return EX.ExprCond(next(children), next(children), next(children))
def probabilistic_constraint(self, pred): return reduce(np.logical_and, [ self.confidence(c, pred) >= self.task_group.tasks[c].options.get( 'min-confidence', 0.99) for c in self.constraints ], np.ones(pred.shape[0], dtype=bool))
def build_project_breakdown_series(reports): Key = namedtuple("Key", "label url color data") def get_legend_data(report): filtered, rate_limited = report.usage_summary return { "events": sum(sum(value) for timestamp, value in report.series), "filtered": filtered, "rate_limited": rate_limited, } # Find the reports with the most total events. (The number of reports to # keep is the same as the number of colors available to use in the legend.) instances = map( operator.itemgetter(0), sorted( reports.items(), key=lambda instance__report: sum( sum(values) for timestamp, values in instance__report[1][0] ), reverse=True, ), )[: len(colors)] # Starting building the list of items to include in the report chart. This # is a list of [Key, Report] pairs, in *ascending* order of the total sum # of values in the series. (This is so when we render the series, the # largest color blocks are at the bottom and it feels appropriately # weighted.) selections = map( lambda instance__color: ( Key( instance__color[0].slug, instance__color[0].get_absolute_url(), instance__color[1], get_legend_data(reports[instance__color[0]]), ), reports[instance__color[0]], ), zip(instances, colors), )[::-1] # Collect any reports that weren't in the selection set, merge them # together and add it at the top (front) of the stack. overflow = set(reports) - set(instances) if overflow: overflow_report = reduce(merge_reports, [reports[instance] for instance in overflow]) selections.insert( 0, (Key("Other", None, "#f2f0fa", get_legend_data(overflow_report)), overflow_report) ) def summarize(key, points): total = sum(points) return [(key, total)] if total else [] # Collect all of the independent series into a single series to make it # easier to render, resulting in a series where each value is a sequence of # (key, count) pairs. series = reduce( merge_series, [series_map(functools.partial(summarize, key), report[0]) for key, report in selections], ) legend = [key for key, value in reversed(selections)] return { "points": [(to_datetime(timestamp), value) for timestamp, value in series], "maximum": max(sum(count for key, count in value) for timestamp, value in series), "legend": { "rows": legend, "total": Key("Total", None, None, reduce(merge_mappings, [key.data for key in legend])), }, }
def operation(sequence): result = reduce(function, sequence, initializer(sequence)) logger.debug("%r reduced %s items to %s.", function, len(sequence), len(result)) return result
def __pow__(self, order): if int(order) == order and order >= 1: return reduce(lambda a, b: Mul(a, b), order * [self]) else: raise ValueError( "Power of {} th order cannot be done.".format(order))
def get_max_combination_number(prameter_matrix, n): param_len_list = [len(value_list) for value_list in prameter_matrix] return sum([ reduce(lambda x, y: x * y, z) for z in combinations(param_len_list, n) ])
def gen_plan_feas_by_plans(plans_df): """ :param data: :return: """ mode_columns_names = ['mode_feas_{}'.format(i) for i in range(12)] def gen_mode_code(mode_list): ma = np.zeros(12) ma[mode_list] = 1 return ma mode_g = plans_df.groupby('sid')['transport_mode'].apply( gen_mode_code).reset_index() mode_columns = ['sid'] + mode_columns_names mode_data = np.concatenate(mode_g['transport_mode'].values, axis=0).reshape(len(mode_g), 12) sid_data = mode_g['sid'].values.reshape(len(mode_g), 1) mode_df = pd.DataFrame(np.hstack([sid_data, mode_data]), columns=mode_columns) def get_first(x): return x.values[0] def gen_mode_texts(x): tl = ' '.join(['word_{}'.format(mode) for mode in x.values]) return tl agg_fun = { 'transport_mode': [get_first, gen_mode_texts], 'distance': ['max', 'min', 'mean', lambda x: np.std(x)], 'price': ['max', 'min', 'mean', lambda x: np.std(x)], 'eta': ['max', 'min', 'mean', lambda x: np.std(x)] } # std ddof =1 agg_columns = [ 'sid', 'first_mode', 'mode_texts', 'max_dist', 'min_dist', 'mean_dist', 'std_dist', 'max_price', 'min_price', 'mean_price', 'std_price', 'max_eta', 'min_eta', 'mean_eta', 'std_eta' ] agg_df = plans_df.groupby('sid').agg(agg_fun).reset_index() agg_df.columns = agg_columns merge_df = pd.merge(plans_df, agg_df, on=['sid'], how='inner') # 原来版本是 keep='last' max_dist_mode_df = merge_df.loc[merge_df['distance'] == merge_df['max_dist'], ['sid', 'transport_mode']] max_dist_mode_df.columns = ['sid', 'max_dist_mode'] max_dist_mode_df.drop_duplicates(subset='sid', keep='last', inplace=True) min_dist_mode_df = merge_df.loc[merge_df['distance'] == merge_df['min_dist'], ['sid', 'transport_mode']] min_dist_mode_df.columns = ['sid', 'min_dist_mode'] min_dist_mode_df.drop_duplicates(subset='sid', keep='first', inplace=True) max_price_mode_df = merge_df.loc[ merge_df['price'] == merge_df['max_price'], ['sid', 'transport_mode']] max_price_mode_df.columns = ['sid', 'max_price_mode'] max_price_mode_df.drop_duplicates(subset='sid', keep='last', inplace=True) min_price_mode_df = merge_df.loc[ merge_df['price'] == merge_df['min_price'], ['sid', 'transport_mode']] min_price_mode_df.columns = ['sid', 'min_price_mode'] min_price_mode_df.drop_duplicates(subset='sid', keep='first', inplace=True) max_eta_mode_df = merge_df.loc[merge_df['eta'] == merge_df['max_eta'], ['sid', 'transport_mode']] max_eta_mode_df.columns = ['sid', 'max_eta_mode'] max_eta_mode_df.drop_duplicates(subset='sid', keep='last', inplace=True) min_eta_mode_df = merge_df.loc[merge_df['eta'] == merge_df['min_eta'], ['sid', 'transport_mode']] min_eta_mode_df.columns = ['sid', 'min_eta_mode'] min_eta_mode_df.drop_duplicates(subset='sid', keep='first', inplace=True) complex_feature_df = reduce( lambda ldf, rdf: pd.merge(ldf, rdf, on=['sid'], how='inner'), [ max_dist_mode_df, min_dist_mode_df, max_price_mode_df, min_price_mode_df, max_eta_mode_df, min_eta_mode_df ]) plan_feature_df = reduce( lambda ldf, rdf: pd.merge(ldf, rdf, on=['sid'], how='inner'), [mode_df, agg_df, complex_feature_df]) return plan_feature_df