def __init__(self,proj_id):
self.dtype = np.float32
try:
self.vg = projector.volume_geometry(proj_id)
self.pg = projector.projection_geometry(proj_id)
self.data_mod = data2d
self.appendString = ""
if projector.is_cuda(proj_id):
self.appendString += "_CUDA"
except Exception:
self.vg = projector3d.volume_geometry(proj_id)
self.pg = projector3d.projection_geometry(proj_id)
self.data_mod = data3d
self.appendString = "3D"
if projector3d.is_cuda(proj_id):
self.appendString += "_CUDA"
self.vshape = functions.geom_size(self.vg)
self.vsize = reduce(operator.mul,self.vshape)
self.sshape = functions.geom_size(self.pg)
self.ssize = reduce(operator.mul,self.sshape)
self.shape = (self.ssize, self.vsize)
self.proj_id = proj_id
self.T = OpTomoTranspose(self)
def as_list(self):
# on top-level, I have type and depth, for easier parallelism,
# but this is hacky as hell
schema = (
self.schema.items if isinstance(self.schema, ArraySchema)
else self.schema
)
items = [InputAdapter(item, self.evaluator, schema)
for item in self.value]
if not self.prefix:
return reduce(operator.add, [a.arg_list() for a in items], [])
if self.separate and self.item_separator is None:
return reduce(operator.add, [[self.prefix] + a.arg_list()
for a in items], [])
if not self.separate and self.item_separator is None:
return [self.prefix + a.list_item()
for a in items if a.list_item() is not None]
joined = self.item_separator.join(
filter(None, [a.list_item() for a in items])
)
if self.separate and self.item_separator is not None:
return [self.prefix, joined]
return [self.prefix + joined]
def evaluate(self, previousResult, original=None, fast=False):
original = original if original is not None else previousResult
if original._grouped_self and fast:
name = GetName(self)
# TODO: Rewrite this, this is terrible.
go_to_index = len(self._queue)
for idx, item in enumerate(self._queue):
if isinstance(item, OperatorStep):
go_to_index = idx
break
transform_input = lambda x: reduce(
lambda prevResult, f: f.evaluate(prevResult, original, fast=fast),
self._queue[1:go_to_index],
x
)
out = original._grouped_self[name].transform(transform_input)
out = reduce(lambda prevResult, f: f.evaluate(prevResult, original, fast=fast),
self._queue[go_to_index:],
out)
return out
else:
output = reduce(lambda prevResult, f: f.evaluate(prevResult, original),
self._queue,
original)
return output
def generate(operator, *args):
"""Generates SparseInterval instance current and past
results of RecurrentEvent.forward() generators with respect
to 'operator' of given expression."""
operands = []
for arg in args:
if isinstance(arg, dict):
try:
result = next(arg['generator'])
arg['results'] = (
arg['results'].union(SparseInterval(*[result]))
)
except StopIteration:
arg['exhausted'] = True
operands.append(arg['results'])
elif isinstance(arg, SparseInterval):
operands.append(arg)
if operator == AND:
return reduce(lambda m, v: m.intersection(v), operands)
elif operator == OR:
return reduce(lambda m, v: m.union(v), operands)
elif operator == NOT:
union = reduce(lambda m, v: m.union(v), operands)
intervals = it.chain((MIN,),
it.chain.from_iterable(union.intervals),
(MAX,))
return SparseInterval(*zip(intervals, intervals))
else:
raise AssertionError
def __purity(cls, mem, references, macro=True, verbose=False):
clusters = {}
avg = 0
memberships = [x for x in mem]
for (i, x) in memberships:
if x in clusters:
clusters[x].append(i)
else:
clusters[x] = [i]
for docs in clusters.values():
(_, count) = median(reduce(lambda s, x: s + x, [references[x] for x in docs]))
if verbose:
print('# %f = %d/%d, labels %s for cluster %s' % (
float(count) / float(len(docs)),
count, len(docs),
str(reduce(lambda s, x: s + x, [references[x] for x in docs])),
str(docs)))
if macro:
avg += count
else:
avg += float(count) / float(len(docs))
if macro:
return float(avg) / len(memberships)
else:
return float(avg) / len(clusters)
def reshape(self, *shape):
"""Gives a new shape to an array without changing its data."""
# TODO: add more error-checking, perhaps
if not self.flags.forc:
raise RuntimeError("only contiguous arrays may "
"be used as arguments to this operation")
if isinstance(shape[0], tuple) or isinstance(shape[0], list):
shape = tuple(shape[0])
if shape == self.shape:
return self
if -1 in shape:
shape = list(shape)
idx = shape.index(-1)
size = -reduce(lambda x, y: x * y, shape, 1)
shape[idx] = self.size // size
if -1 in shape[idx:]:
raise ValueError("can only specify one unknown dimension")
shape = tuple(shape)
size = reduce(lambda x, y: x * y, shape, 1)
if size != self.size:
raise ValueError("total size of new array must be unchanged")
return GPUArray(
shape=shape,
dtype=self.dtype,
allocator=self.allocator,
base=self,
gpudata=int(self.gpudata))
def get_context_data(self, **kwargs):
context = super(SearchableListMixin, self).get_context_data(**kwargs)
qs = self.model.objects.all()
total_objects = qs.count()
query = self.get_search_query()
is_filter = False
if query:
is_filter = True
w_qs = []
search_pairs = self.get_search_fields_with_filters()
for word in self.get_words(query):
filters = [Q(**{'%s__%s' % (pair[0], pair[1]): word}) for pair in search_pairs]
if self.search_date_fields:
dt = self.try_convert_to_date(word)
if dt:
filters.extend([Q(**{field_name: dt}) for field_name in self.search_date_fields])
w_qs.append(reduce(operator.or_, filters))
qs = qs.filter(reduce(operator.and_, w_qs))
qs = qs.distinct()
total_objects = qs.count()
paginator = Paginator(qs, 10)
page = self.request.GET.get('page')
try:
qs = paginator.page(page)
except PageNotAnInteger:
qs = paginator.page(1)
except EmptyPage:
qs = paginator.page(paginator.num_pages)
context.update({'object_list': qs, 'is_filter': is_filter, 'total_objects': total_objects})
return context
def __init__(self,proj_id):
self.dtype = np.float32
try:
self.vg = projector.volume_geometry(proj_id)
self.pg = projector.projection_geometry(proj_id)
self.data_mod = data2d
self.appendString = ""
if projector.is_cuda(proj_id):
self.appendString += "_CUDA"
except Exception:
self.vg = projector3d.volume_geometry(proj_id)
self.pg = projector3d.projection_geometry(proj_id)
self.data_mod = data3d
self.appendString = "3D"
if projector3d.is_cuda(proj_id):
self.appendString += "_CUDA"
self.vshape = functions.geom_size(self.vg)
self.vsize = reduce(operator.mul,self.vshape)
self.sshape = functions.geom_size(self.pg)
self.ssize = reduce(operator.mul,self.sshape)
self.shape = (self.ssize, self.vsize)
self.proj_id = proj_id
self.transposeOpTomo = OpTomoTranspose(self)
try:
self.T = self.transposeOpTomo
except AttributeError:
# Scipy >= 0.16 defines self.T using self._transpose()
pass
def backtrack(self, node, tasks):
candidates = self.fetch_tracks(node.op)
tracks = []
def filter(node, depth):
new_candidates = []
for candidate in candidates:
track, i, lopt = candidate
if i < depth:
pass
elif track[i-depth] in (None, node.op):
if i == depth:
tasks[node].append(lopt)
else:
tracks.append(candidate)
else:
new_candidates.append(candidate)
return new_candidates
depth = 0
nodes = [node]
while candidates:
for node in nodes:
candidates = list(filter(node, depth))
depth += 1
_nodes = nodes
nodes = reduce(list.__iadd__,
[reduce(list.__iadd__,
[[n for n, i in out.clients if not isinstance(n, string_types)] for out in node.outputs],
[]) for node in nodes],
[])
candidates = tracks
tracks = []
def _get_merged_histograms(cursor, property_name, path, with_processes,
histograms_url, additional_histograms):
if path[0] == "histograms" and len(path) != 2:
raise ValueError("Histogram access requires a histogram name.")
elif path[0] == "keyedHistograms" and len(path) != 3:
raise ValueError("Keyed histogram access requires both a histogram name and a label.")
# Get parent property
parent = _get_ping_property(cursor, path, histograms_url,
additional_histograms)
# Get children properties
if not isinstance(cursor, dict):
children = []
else:
children = [_get_ping_property(cursor["processes"]["content"],
path, histograms_url,
additional_histograms)]
children += [_get_ping_property(child, path, histograms_url, additional_histograms)
for child in cursor.get("childPayloads", [])]
children += [_get_ping_property(cursor["processes"]["gpu"],
path, histograms_url,
additional_histograms)]
children = list(filter(lambda h: h is not None, children))
# Merge parent and children
merged = ([parent] if parent else []) + children
result = {}
if with_processes:
result[property_name + "_parent"] = parent
result[property_name + "_children"] = reduce(add, children) if children else None
result[property_name] = reduce(add, merged) if merged else None
return result
def combineQualities(allowed_qualities, preferred_qualities):
any_quality = 0
best_quality = 0
if allowed_qualities:
any_quality = reduce(operator.or_, allowed_qualities)
if preferred_qualities:
best_quality = reduce(operator.or_, preferred_qualities)
return any_quality | (best_quality << 16)
def get_usage(self, using="short"):
if using not in set(["short", "long", "both"]):
raise ValueError(
"using has to be 'short', 'long' or 'both'; not %r" % using
)
# The usage is generated in three steps:
# 1. A "linked list" is generated with lists as nodes. Each node is a
# list consisting of an optional positional as first item followed by
# zero or more required positionals, optionally followed by a list
# representing the next node.
#
# 2. The usage string is generated by enclosing the with a space joined
# usage strings of the positionals in a given node with brackets.
# The exception is the root which is not enclosed by brackets as the
# first node is always required.
#
# 3. The usage string generated in `2` is prefixed with the short and/
# or the long and either or both joined with a comma are returned.
def step(acc, next):
root, current = acc
if next.optional:
current.append([next])
current = current[-1]
else:
current.append(next)
return root, current
def render(tree, _root=True):
if isinstance(tree, Positional):
return tree.usage
else:
nodes = u(" ").join(render(node, _root=False) for node in tree)
if _root:
return nodes
return u("[{0}]").format(nodes)
usage = render(reduce(step, self.positionals, ([], ) * 2)[0])
if using == "both" and self.short and self.long:
return (
u("{0} {1}").format(self.short, usage).strip() +
u(", ") +
u("{0} {1}").format(self.long, usage).strip()
)
else:
return u("{0} {{0}}").format(
self.short if using == "short" and self.short or
using in set(["long", "both"]) and not self.long
else self.long
).format(usage)
return usage.format(
short=self.short,
long=self.long,
usage=render(reduce(step, self.positionals, ([], ) * 2)[0])
).strip()
def expr(cls):
e = pp.Literal("@").suppress() + v_integer
u = reduce(operator.or_, [pp.Literal(i) for i in human.SIZE_UNITS.keys()]).leaveWhitespace()
e = e + pp.Optional(u, default=None)
s = pp.Literal(",").suppress()
s += reduce(operator.or_, [pp.Literal(i) for i in generators.DATATYPES.keys()])
e += pp.Optional(s, default="bytes")
return e.setParseAction(lambda x: cls(*x))
def lcm( *a ): """Least common multiple. Usage: lcm( [ 3, 4, 5 ] ) or: lcm( 3, 4, 5 ) """ if len( a ) > 1: return reduce( lcm2, a ) if hasattr( a[0], "__iter__" ): return reduce( lcm2, a[0] ) return a[0]
def gcd( *a ): """Greatest common divisor. Usage: gcd( [ 2, 4, 6 ] ) or: gcd( 2, 4, 6 ) """ if len( a ) > 1: return reduce( gcd2, a ) if hasattr( a[0], "__iter__" ): return reduce( gcd2, a[0] ) return a[0]
def __setattr__(self, name, value):
if isinstance(self._parent, Style.Style):
setattr(
reduce(getattr, self._attribute.split('.'), self._parent),
name, value)
else:
for cell in self._parent:
setattr(
reduce(getattr, self._attribute.split('.'), cell),
name, value)
def _get_total_hits(self):
""" Returns the total number of hits across all teams and players.
"""
return sm.reduce(
add,
[
sm.reduce(add, [p.hits for p in t.players], 0)
for t in self.model.teams
],
0,
)
def add_trading_days(self, n, date):
if n > 0:
return reduce(
lambda a, b: self.next_trading_day(a),
range(n),
date,
)
else:
return reduce(
lambda a, b: self.previous_trading_day(a),
range(abs(n)),
date,
)
def flatten(s):
if isinstance(s, (tuple, list)):
return reduce(lambda a, b: a + flatten(b), s, [])
elif isinstance(s, set):
raise TypeError("parameter flattening cannot order sets")
elif isinstance(s, dict):
return reduce(lambda a, b: a + flatten(s[b]), sorted(s.keys()), [])
elif isinstance(s, BaseParameter):
return [s]
elif s is None:
return []
else:
raise TypeError("don't understand type %s for %r" % (type(s), s))
def get_queryset(self):
qs = super(SearchableListMixin, self).get_queryset()
query = self.get_search_query()
if query:
w_qs = []
search_pairs = self.get_search_fields_with_filters()
for word in self.get_words(query):
filters = [Q(**{'%s__%s' % (pair[0], pair[1]): word}) for pair in search_pairs]
if self.search_date_fields:
dt = self.try_convert_to_date(word)
if dt:
filters.extend([Q(**{field_name: dt}) for field_name in self.search_date_fields])
w_qs.append(reduce(operator.or_, filters))
qs = qs.filter(reduce(operator.and_, w_qs))
return qs.distinct()
def get_attrs(self, item_list, user):
project_id = item_list[0].project_id
user_lookups = []
for item in item_list:
if item.key != 'sentry:user':
continue
if ':' not in item.value:
continue
try:
user_lookups.append(Q(**parse_user_tag(item.value)))
except ValueError:
continue
tag_labels = {}
if user_lookups:
tag_labels.update({
('sentry:user', euser.tag_value): euser.get_label()
for euser in EventUser.objects.filter(
reduce(operator.or_, user_lookups),
project_id=project_id,
)
})
other_lookups = [
Q(key=i.key, value=i.value)
for i in item_list
if i.key != 'sentry:user'
]
if other_lookups:
tag_labels.update({
(t.key, t.value): t.get_label()
for t in TagValue.objects.filter(
reduce(operator.or_, other_lookups),
project_id=project_id,
)
})
result = {}
for item in item_list:
try:
label = tag_labels[(item.key, item.value)]
except KeyError:
label = item.value
result[item] = {
'name': label,
}
return result
def to_dict(for_lists, global_vars, data_dict):
""" Construct a dict object from a list of ForList object
:param for_lists: list of for_list
:param global_vars: list of global vars to add
:param data_dict: data from an orm-like object (with dot notation)
:return: a dict representation of the ForList objects
"""
res = {}
# The first level is a little bit special
# Manage global variables
for a in global_vars:
a_list = a.split('.')
tmp = res
for i in a_list[:-1]:
if not i in tmp:
tmp[i] = {}
tmp = tmp[i]
tmp[a_list[-1]] = reduce(getattr, a_list[1:], data_dict[a_list[0]])
# Then manage for lists recursively
for for_list in for_lists:
it = for_list.name.split('.')
tmp = res
for i in it[:-1]:
if not i in tmp:
tmp[i] = {}
tmp = tmp[i]
if not it[-1] in tmp:
tmp[it[-1]] = []
tmp = tmp[it[-1]]
if not it[0] in data_dict:
continue
if len(it) == 1:
loop = enumerate(data_dict[it[0]])
else:
loop = enumerate(reduce(getattr, it[-1:], data_dict[it[0]]))
for i, val in loop:
new_data_dict = {for_list.var_from: val}
# We append a new dict only if we need
if len(tmp) <= i:
tmp.append({})
# Call myself with new context, and get result
tmp[i] = ForList.__recur_to_dict(
for_list, new_data_dict, tmp[i]
)
return res
def find_similar_users(self, user):
from sentry.models import (OrganizationMemberTeam, Project)
# limit to only teams user has opted into
project_ids = list(
Project.objects.filter(
team__in=OrganizationMemberTeam.objects.filter(
organizationmember__user=user,
organizationmember__organization__project=self.project_id,
is_active=True,
).values('team'),
).values_list('id', flat=True)[:1000]
)
if not project_ids:
return type(self).objects.none()
filters = []
if self.email:
filters.append(models.Q(email=self.email))
if self.ip_address:
filters.append(models.Q(ip_address=self.ip_address))
if not filters:
return type(self).objects.none()
return type(self).objects.exclude(id=self.id).filter(
reduce(or_, filters),
project_id__in=project_ids,
)
def _simple_fileslice(fileobj, sliceobj, shape, dtype, offset=0, order='C',
heuristic=None):
""" Read all data from `fileobj` into array, then slice with `sliceobj`
The simplest possible thing; read all the data into the full array, then
slice the full array.
Parameters
----------
fileobj : file-like object
implements ``read`` and ``seek``
sliceobj : object
something that can be used to slice an array as in ``arr[sliceobj]``
shape : sequence
shape of full array inside `fileobj`
dtype : dtype object
dtype of array inside `fileobj`
offset : int, optional
offset of array data within `fileobj`
order : {'C', 'F'}, optional
memory layout of array in `fileobj`
heuristic : optional
The routine doesn't use `heuristic`; the parameter is for API
compatibility with :func:`fileslice`
Returns
-------
sliced_arr : array
Array in `fileobj` as sliced with `sliceobj`
"""
fileobj.seek(offset)
nbytes = reduce(operator.mul, shape) * dtype.itemsize
bytes = fileobj.read(nbytes)
new_arr = np.ndarray(shape, dtype, buffer=bytes, order=order)
return new_arr[sliceobj]
def get_base_url_from_root(root):
if root.base_url:
# see :func:`.parse` for why we need to unquote
base_url = unquote(root.base_url)
else:
base_url = root.base_url
return reduce(urljoin, base_href(root)[:1], base_url)
def _read_compact_string(self, code):
if code >= b'\x30':
len_bytes = six.b(chr(ord(code) - 0x30)) + self._read(1)
else:
len_bytes = b'\x00' + code
length = unpack('>H', len_bytes)[0]
bytes = []
while length > 0:
byte = self._read(1)
if b'\x00' <= byte <= b'\x7F':
bytes.append(byte)
elif b'\xC2' <= byte <= b'\xDF':
bytes.append(byte + self._read(1))
elif b'\xE0' <= byte <= b'\xEF':
bytes.append(byte + self._read(2))
elif b'\xF0' <= byte <= b'\xF4':
bytes.append(byte + self._read(3))
length -= 1
try:
return reduce(operator.add, bytes, b'').decode('utf-8')
except UnicodeDecodeError:
if six.PY2:
raise
# We're possibly dealing with surrogate pairs in Python 3
return self._decode_byte_array(bytes)
def build_clause(query, attributes):
# Iterate through each query segment.
clause = None
last = None
for seg in query.segments:
# Get the attribute in question.
attribute = attributes[seg.path[0]]
# Replace the initial path segment with the expanded
# attribute path.
seg.path[0:1] = attribute.path.split('.')
# Boolean's should use `exact` rather than `iexact`.
if attribute.type is bool:
op = '__exact'
else:
op = OPERATOR_MAP[seg.operator]
# Build the path from the segment.
path = '__'.join(seg.path) + op
# Construct a Q-object from the segment.
q = reduce(operator.or_,
map(lambda x: Q((path, x)),
map(attribute.try_clean, seg.values)))
# Combine the segment with the last.
clause = last.combinator(clause, q) if last is not None else q
last = seg
# Return the constructed clause.
return clause
def import_object(module_name, expr):
mod = __import__(module_name)
mod = reduce(getattr, module_name.split('.')[1:], mod)
globals = builtins
if not isinstance(globals, dict):
globals = globals.__dict__
return eval(expr, globals, mod.__dict__)
def _read_binary(self, code, length=None):
chunks = []
while True:
if b'\x20' <= code <= b'\x2F':
# binary data length 0-16
length = ord(code) - 0x20
elif b'\x34' <= code <= b'\x37':
# binary data length 0-1023
len_b1 = (ord(code) - 0x34) << 8
len_b0 = ord(self._read(1))
length = len_b0 + len_b1
else:
len_bytes = self._read(2)
length = unpack('>H', len_bytes)[0]
if length == 0:
break
chunks.append(self._read(length))
if code != b'A':
break
length = None
code = self._read(1)
return Binary(reduce(operator.add, chunks, b''))
def foreignkey_autocomplete(self, request):
"""
Searches in the fields of the given related model and returns the
result as a simple string to be used by the jQuery Autocomplete plugin
"""
query = request.GET.get('q', None)
app_label = request.GET.get('app_label', None)
model_name = request.GET.get('model_name', None)
search_fields = request.GET.get('search_fields', None)
object_pk = request.GET.get('object_pk', None)
try:
to_string_function = self.related_string_functions[model_name]
except KeyError:
if six.PY3:
to_string_function = lambda x: x.__str__()
else:
to_string_function = lambda x: x.__unicode__()
if search_fields and app_label and model_name and (query or object_pk):
def construct_search(field_name):
# use different lookup methods depending on the notation
if field_name.startswith('^'):
return "%s__istartswith" % field_name[1:]
elif field_name.startswith('='):
return "%s__iexact" % field_name[1:]
elif field_name.startswith('@'):
return "%s__search" % field_name[1:]
else:
return "%s__icontains" % field_name
model = apps.get_model(app_label, model_name)
queryset = model._default_manager.all()
data = ''
if query:
for bit in query.split():
or_queries = [models.Q(**{construct_search(smart_str(field_name)): smart_str(bit)}) for field_name in search_fields.split(',')]
other_qs = QuerySet(model)
other_qs.query.select_related = queryset.query.select_related
other_qs = other_qs.filter(reduce(operator.or_, or_queries))
queryset = queryset & other_qs
additional_filter = self.get_related_filter(model, request)
if additional_filter:
queryset = queryset.filter(additional_filter)
if self.autocomplete_limit:
queryset = queryset[:self.autocomplete_limit]
data = ''.join([six.u('%s|%s\n') % (to_string_function(f), f.pk) for f in queryset])
elif object_pk:
try:
obj = queryset.get(pk=object_pk)
except Exception: # FIXME: use stricter exception checking
pass
else:
data = to_string_function(obj)
return HttpResponse(data, content_type='text/plain')
return HttpResponseNotFound()
def __vertex_raw_labels(self, return_name=False):
raw_names = {
-1: 'Unknown',
0: 'Architecture',
1: 'Computer Network',
2: 'Computer Security',
3: 'Data Mining',
4: 'Theory',
5: 'Graphics'
}
if self.__vertex_raw_labels_cache is not None:
if return_name:
return self.__vertex_raw_labels_cache, raw_names
else:
return self.__vertex_raw_labels_cache
# These are conferences that are to merged
confdata = [
[
'ASPLOS|Architectural Support for Programming Languages and Operating Systems',
'FAST|Conference on File and Storage Technologies',
'HPCA|High-Performance Computer Architecture',
'ISCA|Symposium on Computer Architecture', 'MICRO|MICRO',
'USENIX ATC|USENIX Annul Technical Conference',
'PPoPP|Principles and Practice of Parallel Programming'
],
[
'MOBICOM|Mobile Computing and Networking Transactions on Networking',
'SIGCOMM|applications, technologies, architectures, and protocols for computer communication',
'INFOCOM|Computer Communications'
],
[
'CCS|Computer and Communications Security',
'NDSS|Network and Distributed System Security',
# 'CRYPTO|International Cryptology Conference',
# 'EUROCRYPT|European Cryptology Conference',
'S\&P|Symposium on Security and Privacy',
'USENIX Security|Usenix Security Symposium'
],
[
'SIGMOD|Conference on Management of Data',
'SIGKDD|Knowledge Discovery and Data Mining',
'SIGIR|Research on Development in Information Retrieval',
'VLDB|Very Large Data Bases', 'ICDE|Data Engineering'
],
[
'STOC|ACM Symposium on Theory of Computing',
'FOCS|Symposium on Foundations of Computer Science',
'LICS|Symposium on Logic in Computer Science',
'CAV|Computer Aided Verification'
],
[ # 'ACM MM|Multimedia',
'SIGGRAPH|SIGGRAPH Annual Conference',
'IEEE VIS|Visualization Conference', 'VR|Virtual Reality'
],
# ['AAAI|AAAI Conference on Artificial Intelligence',
# 'CVPR|Computer Vision and Pattern Recognition',
# 'ICCV|International Conference on Computer Vision',
# 'ICML|International Conference on Machine Learning',
# 'IJCAI|International Joint Conference on Artificial Intelligence',
# 'NIPS|Annual Conference on Neural Information Processing Systems',
# 'ACL|Annual Meeting of the Association for Computational Linguistics']
]
# confdata records the representing conferences for each field
confdata = {n: i for i, arr in enumerate(confdata) for n in arr}
for k in confdata.keys():
sname, lname = k.split('|')
confdata[k] = [
confdata[k],
re.compile(sname),
re.compile(lname, re.I)
]
# conffeat is a list of matrices, each of them is a user-conference matrix for participation information
conffeat = [
self.__data['conf_feat'][y]
for y in range(self.localunit, self.localunit + self.nunits)
]
# names of conferences in the user-conference matrix, in global indices (before filtering)
conffeat_names = self.__data['conf_names']
confmap = self.__data['confmap']
# maps conference names from global indices to their original names
conffeat_names = [confmap[c] for c in conffeat_names]
# we use theory conferences because it is more independent
rawlb = []
for i in range(self.nsteps):
startunit = self._time2unit(self.step2time(i + self.localstep))
endunit = startunit + self.stepsize
relstartunit, relendunit = startunit - self.localunit, endunit - self.localunit
print(
"generating samples for years from {} to {}, i.e. featidx from {} to {}"
.format(startunit, endunit - 1, relstartunit, relendunit - 1))
curconffeat = reduce(
lambda x, y: x + y, conffeat[relstartunit:relendunit],
np.zeros(conffeat[relstartunit].shape,
dtype=conffeat[relstartunit].dtype)).A
rawlb.append(
self.__label_vertices(curconffeat, conffeat_names, confdata))
print("{}/{} positive samples at step {}".format(
np.sum(rawlb[-1] == 1), len(rawlb[-1]), i + self.localstep))
rawlb = np.vstack(rawlb)
self.__vertex_raw_labels_cache = rawlb
if return_name:
return self.__vertex_raw_labels_cache, raw_names
else:
return self.__vertex_raw_labels_cache
def compile(self, strength=5.0):
"""Returns the compiled :class:`Model`.
This method reduces the degree of the expression if the degree is higher than 2,
and convert it into :class:`.Model` which has information about QUBO.
Args:
strength (float): The strength of the reduction constraint.
Insufficient strength can result in the binary quadratic model
not having the same minimizations as the polynomial.
Returns:
:class:`Model`: The model compiled from the :class:`.Express`.
Examples:
In this example, there are higher order terms :math:`abc` and :math:`abd`. It is decomposed as
[[``a*b``, ``c``], ``d``] hierarchically and converted into QUBO.
By calling :func:`to_qubo()` of the :obj:`model`, we get the resulting QUBO.
>>> from pyqubo import Binary
>>> a, b, c, d = Binary("a"), Binary("b"), Binary("c"), Binary("d")
>>> model = (a*b*c + a*b*d).compile()
>>> pprint(model.to_qubo()) # doctest: +SKIP
({('a', 'a'): 0.0,
('a', 'a*b'): -10.0,
('a', 'b'): 5.0,
('a*b', 'a*b'): 15.0,
('a*b', 'b'): -10.0,
('a*b', 'c'): 1.0,
('a*b', 'd'): 1.0,
('b', 'b'): 0.0,
('c', 'c'): 0,
('d', 'd'): 0},
0.0)
"""
def compile_param_if_express(val):
if isinstance(val, Express):
return val._compile_param()
else:
return val
# Constraint for AND(multiplier, multiplicand) = product
def binary_product(multiplier, multiplicand, product, weight):
return {
BinaryProd({product}): 3.0 * weight,
BinaryProd({multiplicand, product}): -2.0 * weight,
BinaryProd({multiplier, product}): -2.0 * weight,
BinaryProd({multiplier, multiplicand}): weight
}
# When the label contains PROD_SYM, elements of products should be sorted
# such that the resulting label is uniquely determined.
def normalize_label(label):
s = Express.PROD_SYM
if s in label:
return s.join(sorted(label.split(s)))
else:
return label
# Expand the expression to polynomial
expanded, constraints, penalties = Express._expand(self)
expanded = Express._merge_term(expanded, penalties)
# Make polynomials quadratic
offset = 0.0
pubo = {}
for term_key, value in expanded.items():
if term_key.is_constant():
offset = value
else:
pubo[tuple(term_key.keys)] = value
bqm = dimod.make_quadratic(pubo, strength, dimod.BINARY)
bqm_qubo, bqm_offset = bqm.to_qubo()
# Extracts product constrains
product_consts = {}
for (a, b), v in bqm.info['reduction'].items():
prod = normalize_label(v['product'])
a = normalize_label(a)
b = normalize_label(b)
product_consts["AND({},{})={}".format(a, b, prod)]\
= binary_product(a, b, prod, strength)
# Normalize labels and compile values of the QUBO
compiled_qubo = {}
for (label1, label2), value in bqm_qubo.items():
norm_label1 = normalize_label(label1)
norm_label2 = normalize_label(label2)
# Sort the tuple of labels such that the created key is uniquely determined.
if norm_label2 > norm_label1:
label_key = (norm_label1, norm_label2)
else:
label_key = (norm_label2, norm_label1)
# Compile values of the QUBO
compiled_qubo[label_key] = compile_param_if_express(value)
compiled_qubo = CompiledQubo(
compiled_qubo, compile_param_if_express(offset + bqm_offset))
# compile values of constraint
compiled_constraints = {}
for label, constraint in Express._merge_dict(constraints,
product_consts).items():
compiled_constraints[label] =\
CompiledConstraint({prod: compile_param_if_express(value)
for prod, value in constraint.items()})
# Merge structures
uniq_variables = Express._unique_vars(self)
structure = reduce(Express._merge_dict,
[var.structure for var in uniq_variables])
return Model(compiled_qubo, structure, compiled_constraints)
def best(self):
grid = self.grid
obj_task = self.task_group.tasks[self.objective['name']]
obj_model = self.models[self.objective['name']]
# If unconstrained
if self.numConstraints() == 0:
# Compute the GP mean
obj_mean, obj_var = obj_model.function_over_hypers(
obj_model.predict, grid)
# find the min and argmin of the GP mean
current_best_location = grid[np.argmin(obj_mean), :][None]
best_ind = np.argmin(obj_mean)
current_best_value = obj_mean[best_ind]
std_at_best = np.sqrt(obj_var[best_ind])
# un-normalize the min of mean to original units
unnormalized_best_value = obj_task.unstandardize_mean(
obj_task.unstandardize_variance(current_best_value))
unnormalized_std_at_best = obj_task.unstandardize_variance(
std_at_best)
# Print out the minimum according to the model
if not self.quiet:
sys.stderr.write(
'\nMinimum expected objective value under model '
'is %.5f (+/- %.5f), at location:\n' %
(unnormalized_best_value, unnormalized_std_at_best))
self.task_group.paramify_and_print(
self.task_group.from_unit(current_best_location).flatten(),
left_indent=16,
indent_top_row=True)
# Compute the best value seen so far
vals = self.task_group.values[self.objective['name']]
inps = self.task_group.inputs
best_observed_value = np.min(vals)
best_observed_location = inps[np.argmin(vals), :][None]
# Don't need to un-normalize inputs here because these are the raw inputs
if not self.quiet:
sys.stderr.write(
'\nMinimum of observed values is %f, at location:\n' %
best_observed_value)
self.task_group.paramify_and_print(
best_observed_location.flatten(),
left_indent=16,
indent_top_row=True)
else:
mc = self.probabilistic_constraint(grid)
if not np.any(mc):
# P-con is violated everywhere
# Compute the product of the probabilities, and return None for the current best value
probs = reduce(
lambda x, y: x * y,
[self.confidence(c, grid) for c in self.constraints],
np.ones(grid.shape[0]))
best_probs_ind = np.argmax(probs)
best_probs_location = grid[best_probs_ind, :][None]
# TODO -- could use BFGS for this (unconstrained) optimization as well -- everytime for min of mean
if not self.quiet:
sys.stderr.write('\nNo feasible region found (yet).\n')
sys.stderr.write(
'Maximum probability of satisfying constraints = %f\n'
% np.max(probs))
sys.stderr.write('At location: ')
self.task_group.paramify_and_print(
self.task_group.from_unit(
best_probs_location).flatten(),
left_indent=16)
return None, best_probs_location
# A feasible region has been found
# Compute GP mean and find minimum
mean, var = obj_model.function_over_hypers(obj_model.predict, grid)
valid_mean = mean[mc]
valid_var = var[mc]
best_ind = np.argmin(valid_mean)
current_best_location = (grid[mc])[best_ind, :][None]
ind = np.argmin(valid_mean)
current_best_value = valid_mean[ind]
std_at_best = np.sqrt(valid_var[ind])
unnormalized_best = obj_task.unstandardize_mean(
obj_task.unstandardize_variance(current_best_value))
unnormalized_std_at_best = obj_task.unstandardize_variance(
std_at_best) # not used -- not quite
# right to report this -- i mean there is uncertainty in the constraints too
# this is the variance at that location, not the standard deviation of the minimum...
# not sure if this distinction is a big deal
if not self.quiet:
sys.stderr.write(
'\nMinimum expected objective value satisfying constraints w/ high prob: %f\n'
% unnormalized_best)
sys.stderr.write('At location: ')
self.task_group.paramify_and_print(
self.task_group.from_unit(current_best_location).flatten(),
left_indent=16)
# Compute the best value seen so far
with np.errstate(invalid='ignore'):
all_constraints_satisfied = np.all(np.greater(
np.array(
[x.values for x in self.task_group.tasks.values()]),
0),
axis=0)
if not np.any(all_constraints_satisfied):
if not self.quiet:
sys.stderr.write(
'No observed result satisfied all constraints.\n')
else:
inps = self.task_group.inputs
vals = self.task_group.values[self.objective['name']]
# get rid of those that violate constraints
vals[np.logical_not(all_constraints_satisfied)] = np.max(vals)
# get rid of NaNs -- set them to biggest not-nan value, then they won't be the minimum
vals[np.isnan(vals)] = np.max(vals[np.logical_not(
np.isnan(vals))])
best_observed_value = np.min(vals)
best_observed_location = inps[np.argmin(vals), :][None]
# Don't need to un-normalize inputs here because these are the raw inputs
if not self.quiet:
sys.stderr.write(
'\nBest observed values satisfying constraints is %f, at location:\n'
% best_observed_value)
self.task_group.paramify_and_print(
best_observed_location.flatten(),
left_indent=16,
indent_top_row=True)
# Return according to model, not observed
return current_best_value, current_best_location
def acquisition_function(self, cand, current_best, compute_grad=True):
obj_model = self.models[self.objective['name']]
# If unconstrained, just compute regular ei
if self.numConstraints() == 0:
return compute_ei(obj_model,
cand,
ei_target=current_best,
compute_grad=compute_grad)
if cand.ndim == 1:
cand = cand[None]
N_cand = cand.shape[0]
############## ---------------------------------------- ############
############## ############
############## Part that depends on the objective ############
############## ############
############## ---------------------------------------- ############
if current_best is None:
ei = 1.
ei_grad = 0.
else:
target = current_best
# Compute the predictive mean and variance
if not compute_grad:
ei = compute_ei(obj_model,
cand,
target,
compute_grad=compute_grad)
else:
ei, ei_grad = compute_ei(obj_model,
cand,
target,
compute_grad=compute_grad)
############## ---------------------------------------- ############
############## ############
############## Part that depends on the constraints ############
############## ############
############## ---------------------------------------- ############
# Compute p(valid) for ALL constraints
p_valid, p_grad = list(), list()
for c in self.constraints:
if compute_grad:
pv, pvg = self.models[c].pi(cand, compute_grad=True)
p_valid.append(pv)
p_grad.append(pvg)
else:
p_valid.append(self.models[c].pi(cand, compute_grad=False))
p_valid_prod = reduce(np.multiply, p_valid, np.ones(N_cand))
# To compute the gradient, need to do the chain rule for the product of N factors
if compute_grad:
p_grad_prod = np.zeros(p_grad[0].shape)
for i in xrange(self.numConstraints()):
pg = p_grad[i]
for j in xrange(self.numConstraints()):
if j == i:
continue
pg *= p_valid[j]
p_grad_prod += pg
# multiply that gradient by all other pv's (this might be numerically disasterous if pv=0...)
############## ---------------------------------------- ############
############## ############
############## Combine the two parts (obj and con) ############
############## ############
############## ---------------------------------------- ############
acq = ei * p_valid_prod
if not compute_grad:
return acq
else:
return acq, ei_grad * p_valid_prod + p_grad_prod * ei
def __hash__(self):
return reduce(xor, [hash(term) for term in self.terms])
def __call__(self, sequence):
return reduce(lambda x, operation: operation(x), self.operations,
sequence)
def decode_solution(self, solution, vartype, feed_dict=None):
"""Returns decoded solution.
Args:
solution (list[bit]/dict[label, bit]/dict[index, bit]):
The solution returned from solvers.
vartype (:class:`dimod.Vartype`/str/set, optional):
Variable type of the solution. Accepted input values:
* :class:`.Vartype.SPIN`, ``'SPIN'``, ``{-1, 1}``
* :class:`.Vartype.BINARY`, ``'BINARY'``, ``{0, 1}``
feed_dict (dict[str, float]):
Specify the placeholder values.
Returns:
tuple(dict, dict, float): Tuple of the decoded solution,
broken constraints and energy.
Structure of decoded_solution is defined by :obj:`structure`.
"""
def put_value_with_keys(dict_body, keys, value):
for key in keys[:-1]:
if key not in dict_body:
dict_body[key] = {}
dict_body = dict_body[key]
dict_body[keys[-1]] = value
decoded_solution = {}
dict_bin_solution = self._parse_solution(solution, vartype)
for label, bit in dict_bin_solution.items():
if label in self.structure:
if vartype == dimod.SPIN:
out_value = 2 * bit - 1
elif vartype == dimod.BINARY:
out_value = bit
put_value_with_keys(decoded_solution, self.structure[label],
out_value)
# Check satisfaction of constraints
broken_const = {}
for label, const in self.constraints.items():
energy = const.energy(dict_bin_solution, feed_dict)
if energy > 0.0:
result_value = {
var: dict_bin_solution[var]
for var in reduce(
or_, [k.keys for k in const.polynomial.keys()])
}
broken_const[label] = {
"result": result_value,
"penalty": energy
}
elif energy < 0.0:
raise ValueError(
"The energy of the constraint \"{label}\" is {energy}."
"But an energy of constraints should not be negative.".
format(label=label, energy=energy))
problem_energy = self.energy(dict_bin_solution, dimod.BINARY,
feed_dict)
return decoded_solution, broken_const, problem_energy
def __exit__(self, type, value, traceback):
"""Restore previous value."""
config_obj = reduce(dict.__getitem__, self.nested_keys[:-1],
current_app.config)
config_obj[self.nested_keys[-1]] = self.prev_value
def cudandarray_to_garray(x, copyif=False):
""" take a CudaNdarray and return a gnumpy.garray object.
:type x: CudaNdarray
:param x: The array to transform to gnumpy.garray.
:type copyif: bool
:param copyif: If False, raise an error if x is not c contiguous.
If it is c contiguous, we return a GPUArray that share
the same memory region as x.
If True, copy x if it is no c contiguous, so the return won't
shape the same memory region. If c contiguous, the return
will share the same memory region.
We need to do this as GPUArray don't fully support strided memory.
:return type: cudamat.CUDAMatrix
"""
if not isinstance(x, cuda.CudaNdarray):
raise ValueError(
"We can transfer only CudaNdarray to cudamat.CUDAMatrix")
else:
# Check if it is c contiguous
size = 1
c_contiguous = True
for i in range(x.ndim - 1, -1, -1):
if x.shape[i] == 1:
continue
if x._strides[i] != size:
c_contiguous = False
break
size *= x.shape[i]
if not c_contiguous:
if copyif:
x = x.copy()
else:
raise ValueError(
"We where asked to don't copy memory, but the memory is not c contiguous."
)
# Now x is always c contiguous.
# the next step is to create a CUDAMatrix object. We do so by first creating
# a cudamat object with no data_host.
cm_mat = cudamat.cudamat()
cm_mat.size[0] = reduce(lambda x, y: x * y, x.shape, 1)
cm_mat.size[1] = 1
cm_mat.on_host = 0
cm_mat.on_device = 1
cm_mat.is_trans = 0
cm_mat.owns_data = 0 # <-- note: cm_mat dosen't owe the data; x does. So x will delete it.
# x.gpudata is a long. We need a pointer to a float. cast.
import ctypes
cm_mat.data_device = ctypes.cast(x.gpudata,
ctypes.POINTER(ctypes.c_float))
px = cudamat.CUDAMatrix(cm_mat)
px._base = x # x won't be freed if the cudamat object isn't freed.
px.mat_on_host = False # let cudamat know that we don't have a numpy
# array attached.
# Note how gnumpy tracks its cudamat objects: it moves things to the
# _cmsReuseCache when the gnumpy array is deleted, thus the arrays
# returned by theano will never be deleted.
# However, if the garray thinks that the object is a view, then it won't
# move the _base to the _cmsResueCache; so the cudamat object will be deleted,
# and we won't overpump the world with memory.
_is_alias_of = ___const_garray
ans = gnumpy.garray(px, x.shape, _is_alias_of)
return ans
def __enter__(self):
"""Save previous value and swap it with the new."""
config_obj = reduce(dict.__getitem__, self.nested_keys[:-1],
current_app.config)
self.prev_value = config_obj[self.nested_keys[-1]]
config_obj[self.nested_keys[-1]] = self.new_value
def last(seq):
return reduce(lambda l, r: r, seq)
def standard_jvp2(jvprules, primitive, primals, tangents, **params):
val_out = primitive.bind(*primals, **params)
tangents_out = (rule(t, val_out, *primals, **params) for rule, t in zip(jvprules, tangents)
if rule is not None and t is not zero)
return val_out, reduce(add_tangents, tangents_out, zero)
def cdbhash(s):
return reduce(lambda h, c: (((h << 5) + h) ^ ord(c)) & 0xffffffff, s, 5381)
def encode_tuple(self, obj):
ref = self.add_ref(obj)
if ref:
return ref
encoded = reduce(operator.add, map(self.encode, obj), b'')
return pack('>2cl', b'V', b'l', len(obj)) + encoded + b'z'
def join_components(cls, components):
return reduce(posixpath.join, SeqUtils.filter_true(components))
def _jug_reduce(reducer, inputs):
from six.moves import reduce
reducer = _get_function(reducer)
return reduce(reducer, chain(inputs))
def __call__(self, environ, start_response):
# specify that we want the parsed dataset
environ['x-wsgiorg.want_parsed_response'] = True
req = Request(environ)
projection, selection = parse_ce(req.query_string)
# check if there are any functions calls in the request
called = (any(s for s in selection if FUNCTION.match(s))
or any(p for p in projection if isinstance(p, string_types)))
# ignore DAS requests and requests without functions
path, response = req.path.rsplit('.', 1)
if response == 'das' or not called:
return self.app(environ, start_response)
# apply selection without any function calls
req.query_string = '&'.join(s for s in selection
if not FUNCTION.match(s))
res = req.get_response(self.app)
# get the dataset
method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
if not method:
raise ServerError("Unable to call server-side function!")
dataset = method(DatasetType)
# apply selection containing server-side functions
selection = (s for s in selection if FUNCTION.match(s))
for expr in selection:
if RELOP.search(expr):
call, op, other = RELOP.split(expr)
op = {
'<': operator.lt,
'>': operator.gt,
'!=': operator.ne,
'=': operator.eq,
'>=': operator.ge,
'<=': operator.le,
'=~': lambda a, b: re.match(b, a),
}[op]
other = ast.literal_eval(other)
else:
call, op, other = expr, operator.eq, 1
# evaluate the function call
sequence = eval_function(dataset, call, self.functions)
# is this an inplace call?
for var in walk(dataset, SequenceType):
if sequence is var:
break
else:
# get the data from the resulting variable, and use it to
# constrain the original dataset
child = list(sequence.children())[0]
data = np.fromiter(child.data, child.dtype)
if data.dtype.char == "S":
valid = np.array(
list(map(lambda v: op(str(v), str(other)), data)),
bool)
else:
valid = op(data, other)
for sequence in walk(dataset, SequenceType):
sequence.data = np.rec.fromrecords(
[tuple(row) for row in sequence],
names=sequence.keys())[valid]
# now apply projection
if projection:
projection = fix_shorthand(projection, dataset)
base = [p for p in projection if not isinstance(p, string_types)]
func = [p for p in projection if isinstance(p, string_types)]
# apply non-function projection
out = apply_projection(base, dataset)
# apply function projection
for call in func:
var = eval_function(dataset, call, self.functions)
for child in walk(var):
parent = reduce(operator.getitem,
[out] + child.id.split('.')[:-1])
if child.name not in parent.keys():
parent[child.name] = child
break
dataset = out
# Return the original response (DDS, DAS, etc.)
path, response = req.path.rsplit('.', 1)
res = BaseHandler.responses[response](dataset)
return res(environ, start_response)
def _expand(exp):
"""Expand the expression hierarchically into dict format.
For example, ``2*Binary(a)*Binary(b) + 1`` is represented as
dict format ``{BinaryProd(ab): 2.0, CONST_TERM_KEY: 1.0}``.
Let's see how this dict is created step by step.
First, focus on `2*Binary(a)*Binary(b)` in which each expression is expanded as
_expand(Num(2)) # => {CONST_TERM_KEY: 2.0}
_expand(Binary("a")) # => {BinaryProd(a): 1.0}
_expand(Binary("b")) # => {BinaryProd(b): 1.0}
respectively.
:class:`Mul` combines the expression in the following way
_expand(Mul(Num(2), Binary("a"))) # => {BinaryProd(a): 2.0}
_expand(Mul(Binary("b"), Mul(Num(2), Binary("a")))) # => {BinaryProd(ab): 2.0}
Finally, :class:`Add` combines the ``{BinaryProd(ab): 2.0}`` and `{CONST_TERM_KEY: 1.0}`,
and we get the final form {BinaryProd(ab): 2.0, CONST_TERM_KEY: 1.0}
Args:
exp (:class:`Express`): Input expression.
Returns:
tuple(expanded_terms, constraints):
tuple of expanded_terms and constrains. Expanded expression takes the form
``dict[:class:`BinaryProd`, value]``.
Constraints takes the form of ``dict[label, expanded_terms]``.
"""
if isinstance(exp, WithPenalty):
val, val_const, val_penalty = Express._expand(exp.express)
penalty, penalty_const, penalty_penalty = Express._expand(
exp.penalty)
combined_penalty = Express._merge_term(val_penalty,
penalty_penalty)
combined_penalty = Express._merge_term(combined_penalty, penalty)
return val, Express._merge_dict_update(
val_const, penalty_const), combined_penalty
elif isinstance(exp, Add):
left, left_const, left_p = Express._expand(exp.left)
right, right_const, right_p = Express._expand(exp.right)
result = Express._merge_term(right, left)
return result, Express._merge_dict_update(left_const, right_const),\
Express._merge_term(left_p, right_p)
elif isinstance(exp, AddList):
expanded, const, penalty = reduce(
lambda arg1, arg2: (Express._merge_term(arg1[0], arg2[
0]), Express._merge_dict_update(arg1[1], arg2[1]),
Express._merge_term(arg1[2], arg2[2])),
[Express._expand(term) for term in exp.terms])
return expanded, const, penalty
elif isinstance(exp, Mul):
left, left_const, left_p = Express._expand(exp.left)
right, right_const, right_p = Express._expand(exp.right)
expanded_terms = defaultdict(float)
for k1, v1 in left.items():
for k2, v2 in right.items():
merged_key = BinaryProd.merge_term_key(k1, k2)
expanded_terms[merged_key] += v1 * v2
return expanded_terms, Express._merge_dict_update(left_const, right_const),\
Express._merge_term(left_p, right_p)
elif isinstance(exp, Placeholder):
expanded_terms = defaultdict(float)
expanded_terms[Express.CONST_TERM_KEY] = exp
return expanded_terms, {}, {}
elif isinstance(exp, Constraint):
child, child_const, child_p = Express._expand(exp.child)
child_const[exp.label] = copy.copy(child)
return child, child_const, child_p
elif isinstance(exp, Num):
terms = defaultdict(float)
terms[Express.CONST_TERM_KEY] = exp.value
return terms, {}, {}
elif isinstance(exp, Binary):
terms = defaultdict(float)
terms[BinaryProd({exp.label})] = 1.0
return terms, {}, {}
elif isinstance(exp, Spin):
terms = defaultdict(float)
terms[BinaryProd({exp.label})] = 2.0
terms[Express.CONST_TERM_KEY] = -1.0
return terms, {}, {}
elif isinstance(exp, UserDefinedExpress):
return Express._expand(exp.express)
else:
raise TypeError("Unexpected input type {}.".format(
type(exp))) # pragma: no cover
def get_group_event_ids(self, project_id, group_id, environment_id, tags):
# NOTE: `environment_id=None` needs to be filtered differently in this method.
# EventTag never has NULL `environment_id` fields (individual Events always have an environment),
# and so `environment_id=None` needs to query EventTag for *all* environments (except, ironically
# the aggregate environment).
if environment_id is None:
# filter for all 'real' environments
exclude = {'_key__environment_id': AGGREGATE_ENVIRONMENT_ID}
env_filter = {}
else:
exclude = {}
env_filter = {'_key__environment_id': environment_id}
tagvalue_qs = TagValue.objects.filter(
reduce(or_, (Q(_key__key=k, _key__status=TagKeyStatus.VISIBLE, value=v)
for k, v in six.iteritems(tags))),
project_id=project_id,
_key__project_id=project_id,
**env_filter
)
if exclude:
tagvalue_qs = tagvalue_qs.exclude(**exclude)
tagvalue_qs = tagvalue_qs.values_list('_key_id', 'id', '_key__key', 'value')
tagvalues = defaultdict(list)
for key_id, value_id, key, value in tagvalue_qs:
tagvalues[(key, value)].append((key_id, value_id))
tagvalues = dict(tagvalues)
try:
# ensure all key/value pairs were found
tag_lookups = [tagvalues[(k, v)] for k, v in six.iteritems(tags)]
# [[(key0, value0), (key1, value1)], ...]
except KeyError:
# one or more tags were invalid, thus the result should be an empty
# set
return []
# Django doesnt support union, so we limit results and try to find
# reasonable matches
# get initial matches to start the filter
kv_pairs = tag_lookups.pop()
matches = list(
EventTag.objects.filter(
reduce(or_, (Q(key_id=k, value_id=v)
for k, v in kv_pairs)),
project_id=project_id,
group_id=group_id,
).values_list('event_id', flat=True)[:1000]
)
# for each remaining tag, find matches contained in our
# existing set, pruning it down each iteration
for kv_pairs in tag_lookups:
matches = list(
EventTag.objects.filter(
reduce(or_, (Q(key_id=k, value_id=v)
for k, v in kv_pairs)),
project_id=project_id,
group_id=group_id,
event_id__in=matches,
).values_list('event_id', flat=True)[:1000]
)
if not matches:
return []
return matches
def any(cls):
return reduce(operator.or_, (x.value for x in cls))
def test_huge_elemwise_fusion():
""" Test the the GpuElemwise fusion work correctly
We check that we fuse one node with part of its input
in case their is too many inputs and that would make it bust the 256
bytes limits.
"""
shape = (2, 3, 4, 5, 6)
ttype = tensor.tensor(dtype='float32',
broadcastable=(False, ) * len(shape))
gpu_ptr_size = theano.sandbox.cuda.opt.get_device_type_sizes(
)['gpu_ptr_size']
if gpu_ptr_size == 8:
nb_in = 7
len_topo = 10
elif gpu_ptr_size == 4:
nb_in = 8
len_topo = 11
else:
raise Exception("Unexpected value for gpu_ptr_size", gpu_ptr_size)
vars = [tensor.tanh(ttype) for x in range(nb_in)]
f = pfunc(vars, [reduce(operator.sub, vars)], mode=mode_with_gpu)
topo = f.maker.fgraph.toposort()
assert len(topo) == len_topo
assert sum([isinstance(node.op, cuda.GpuElemwise) for node in topo]) == 2
assert isinstance(topo[-3].op.scalar_op, theano.scalar.basic.Sub)
assert isinstance(topo[-2].op.scalar_op, theano.scalar.basic.Composite)
# let debugmode catch errors
gen = lambda: theano._asarray(numpy.random.rand(*shape), dtype='float32')
f(*[gen() for i in range(nb_in)])
# Test the case where we can't put the computation on the gpu! their is too
# many dimensions to the input to have 2 inputs to the op!
shape = (
1,
2,
3,
4,
5,
6,
7,
2,
2,
3,
2,
1,
2,
2,
2,
)
ttype = tensor.tensor(dtype='float32',
broadcastable=(False, ) * len(shape))
vars = [tensor.tanh(ttype) for x in range(7)]
f = pfunc(
vars,
[vars[0] - vars[1] - vars[2] - vars[3] - vars[4] - vars[5] - vars[6]],
mode=mode_with_gpu)
topo = f.maker.fgraph.toposort()
assert len(topo) == 1
assert sum([isinstance(node.op, cuda.GpuElemwise) for node in topo]) == 0
assert sum([isinstance(node.op, tensor.Elemwise) for node in topo]) == 1
# let debugmode catch errors
gen = lambda: theano._asarray(numpy.random.rand(*shape), dtype='float32')
f(gen(), gen(), gen(), gen(), gen(), gen(), gen())
def gen(shape):
return theano._asarray(numpy.random.rand(*shape), dtype='float32')
max_var = 16 # excluded
for shape in [
(2, ),
(2, 2),
(2, 2, 2),
(2, 2, 2, 2),
(2, 2, 2, 2, 2), # 5d
(2, 2, 2, 2, 2, 2),
# (2, 2, 2, 2, 2, 2, 2),
# (2, 2, 2, 2, 2, 2, 2, 2),
# (2, 2, 2, 1, 1, 1, 1, 2, 2), # 9d
]:
vals = [cuda.shared_constructor(gen(shape)) for x in range(max_var)]
for use_tan in [True, False]:
if use_tan:
vars = [tensor.tanh(x) for x in vals]
else:
vars = vals
for nb_var in range(1, max_var):
out = reduce(lambda x, y: x + y, vars[:nb_var])
if not isinstance(out.type, CudaNdarrayType):
out = cuda.gpu_from_host(out)
f = pfunc([], [out], mode=mode_with_gpu)
topo = f.maker.fgraph.toposort()
# print shape, nb_var, use_tan, len(topo)
assert (sum(
[isinstance(node.op, cuda.GpuElemwise)
for node in topo]) == len(topo)
or (nb_var == 1 and use_tan is False))
assert sum([
isinstance(node.op, tensor.Elemwise) for node in topo
]) == 0
# let debugmode catch errors
f()
def parse_marker_dict(marker_dict):
op = marker_dict["op"]
lhs = marker_dict["lhs"]
rhs = marker_dict["rhs"]
# This is where the spec sets for each side land if we have an "or" operator
side_spec_list = []
side_markers_list = []
finalized_marker = ""
# And if we hit the end of the parse tree we use this format string to make a marker
format_string = "{lhs} {op} {rhs}"
specset = SpecifierSet()
specs = set()
# Essentially we will iterate over each side of the parsed marker if either one is
# A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset
# Union the "and" specs, intersect the "or"s to find the most appropriate range
if any(issubclass(type(side), Mapping) for side in (lhs, rhs)):
for side in (lhs, rhs):
side_specs = set()
side_markers = set()
if issubclass(type(side), Mapping):
merged_side_specs, merged_side_markers = parse_marker_dict(side)
side_specs.update(merged_side_specs)
side_markers.update(merged_side_markers)
else:
marker = _ensure_marker(side)
marker_parts = getattr(marker, "_markers", [])
if marker_parts[0][0].value == "python_version":
side_specs |= set(get_specset(marker_parts))
else:
side_markers.add(str(marker))
side_spec_list.append(side_specs)
side_markers_list.append(side_markers)
if op == "and":
# When we are "and"-ing things together, it probably makes the most sense
# to reduce them here into a single PySpec instance
specs = reduce(lambda x, y: set(x) | set(y), side_spec_list)
markers = reduce(lambda x, y: set(x) | set(y), side_markers_list)
if not specs and not markers:
return specset, finalized_marker
if markers and isinstance(markers, (tuple, list, Set)):
finalized_marker = Marker(" and ".join([m for m in markers if m]))
elif markers:
finalized_marker = str(markers)
specset._specs = frozenset(specs)
return specset, finalized_marker
# Actually when we "or" things as well we can also just turn them into a reduced
# set using this logic now
sides = reduce(lambda x, y: set(x) & set(y), side_spec_list)
finalized_marker = " or ".join(
[normalize_marker_str(m) for m in side_markers_list]
)
specset._specs = frozenset(sorted(sides))
return specset, finalized_marker
else:
# At the tip of the tree we are dealing with strings all around and they just need
# to be smashed together
specs = set()
if lhs == "python_version":
format_string = "{lhs}{op}{rhs}"
marker = Marker(format_string.format(**marker_dict))
marker_parts = getattr(marker, "_markers", [])
_set = get_specset(marker_parts)
if _set:
specs |= set(_set)
specset._specs = frozenset(specs)
return specset, finalized_marker
def _jug_map_reduce(reducer, mapper, inputs):
from six.moves import reduce
reducer = _get_function(reducer)
mapper = _get_function(mapper)
return reduce(reducer, _jug_map(mapper, inputs))
def tritonast2arybo(e, use_exprs=True, use_esf=False, context=None):
''' Convert a subset of Triton's AST into Arybo's representation
Args:
e: Triton AST
use_esf: use ESFs when creating the final expression
context: dictionnary that associates Triton expression ID to arybo expressions
Returns:
An :class:`arybo.lib.MBAVariable` object
'''
children_ = e.getChildren()
children = (tritonast2arybo(c, use_exprs, use_esf, context)
for c in children_)
reversed_children = (tritonast2arybo(c, use_exprs, use_esf, context)
for c in reversed(children_))
Ty = e.getType()
if Ty == TAstN.ZX:
n = next(children)
v = next(children)
n += v.nbits
if n == v.nbits:
return v
return v.zext(n)
if Ty == TAstN.SX:
n = next(children)
v = next(children)
n += v.nbits
if n == v.nbits:
return v
return v.sext(n)
if Ty == TAstN.INTEGER:
return e.getInteger()
if Ty == TAstN.BV:
cst = next(children)
nbits = next(children)
if use_exprs:
return EX.ExprCst(cst, nbits)
else:
return _get_mba(nbits, use_esf).from_cst(cst)
if Ty == TAstN.EXTRACT:
last = next(children)
first = next(children)
v = next(children)
return v[first:last + 1]
if Ty == TAstN.CONCAT:
if use_exprs:
return EX.ExprConcat(*list(reversed_children))
else:
return flatten(reversed_children)
if Ty == TAstN.VARIABLE:
name = e.getSymbolicVariable().getName()
ret = _get_mba(e.getBitvectorSize(), use_esf).var(name)
if use_exprs:
ret = EX.ExprBV(ret)
return ret
if Ty == TAstN.REFERENCE:
if context is None:
raise ValueError(
"reference node without context can't be resolved")
id_ = e.getSymbolicExpression().getId()
ret = context.get(id_, None)
if ret is None:
raise ValueError("expression id %d not found in context" % id_)
return ret
if Ty == TAstN.LET:
# Alias
# djo: "c'est pas utilise osef"
raise ValueError("unsupported LET operation")
# Logical/arithmetic shifts
shifts = {
TAstN.BVASHR: lambda a, b: a.ashr(b),
TAstN.BVLSHR: lambda a, b: a.lshr(b),
TAstN.BVSHL: operator.lshift,
TAstN.BVROL: lambda x, n: x.rol(n),
TAstN.BVROR: lambda x, n: x.ror(n)
}
shift = shifts.get(Ty, None)
if not shift is None:
v = next(children)
n = next(children)
return shift(v, n)
# Unary op
unops = {
TAstN.BVNOT: lambda x: ~x,
TAstN.LNOT: lambda x: ~x,
TAstN.BVNEG: operator.neg
}
unop = unops.get(Ty, None)
if not unop is None:
return unop(next(children))
binops = {
TAstN.BVADD: operator.add,
TAstN.BVSUB: operator.sub,
TAstN.BVAND: operator.and_,
TAstN.BVOR: operator.or_,
TAstN.BVXOR: operator.xor,
TAstN.BVMUL: operator.mul,
TAstN.BVNAND: lambda x, y: ~(x & y),
TAstN.BVNOR: lambda x, y: ~(x | y),
TAstN.BVXNOR: lambda x, y: ~(x ^ y),
TAstN.BVUDIV: lambda x, y: x.udiv(y),
TAstN.BVSDIV: lambda x, y: x.sdiv(y),
TAstN.BVUREM: lambda x, y: x.urem(y),
TAstN.BVSREM: lambda x, y: x.srem(y),
TAstN.LAND: operator.and_,
TAstN.LOR: operator.or_
}
binop = binops.get(Ty, None)
if not binop is None:
return reduce(binop, children)
# Logical op
lops = {
TAstN.EQUAL: lambda x, y: EX.ExprCmpEq(x, y),
TAstN.DISTINCT: lambda x, y: EX.ExprCmpNeq(x, y),
TAstN.BVUGE: lambda x, y: EX.ExprCmpGte(x, y, False),
TAstN.BVUGT: lambda x, y: EX.ExprCmpGt(x, y, False),
TAstN.BVULE: lambda x, y: EX.ExprCmpLte(x, y, False),
TAstN.BVULT: lambda x, y: EX.ExprCmpLt(x, y, False),
TAstN.BVSGE: lambda x, y: EX.ExprCmpGte(x, y, True),
TAstN.BVSGT: lambda x, y: EX.ExprCmpGt(x, y, True),
TAstN.BVSLE: lambda x, y: EX.ExprCmpLte(x, y, True),
TAstN.BVSLT: lambda x, y: EX.ExprCmpLt(x, y, True)
}
lop = lops.get(Ty, None)
if not lop is None:
return reduce(lop, children)
# Conditional
if Ty != TAstN.ITE:
raise ValueError("unsupported node type %s" % str(Ty))
return EX.ExprCond(next(children), next(children), next(children))
def probabilistic_constraint(self, pred):
return reduce(np.logical_and, [
self.confidence(c, pred) >= self.task_group.tasks[c].options.get(
'min-confidence', 0.99) for c in self.constraints
], np.ones(pred.shape[0], dtype=bool))
def build_project_breakdown_series(reports):
Key = namedtuple("Key", "label url color data")
def get_legend_data(report):
filtered, rate_limited = report.usage_summary
return {
"events": sum(sum(value) for timestamp, value in report.series),
"filtered": filtered,
"rate_limited": rate_limited,
}
# Find the reports with the most total events. (The number of reports to
# keep is the same as the number of colors available to use in the legend.)
instances = map(
operator.itemgetter(0),
sorted(
reports.items(),
key=lambda instance__report: sum(
sum(values) for timestamp, values in instance__report[1][0]
),
reverse=True,
),
)[: len(colors)]
# Starting building the list of items to include in the report chart. This
# is a list of [Key, Report] pairs, in *ascending* order of the total sum
# of values in the series. (This is so when we render the series, the
# largest color blocks are at the bottom and it feels appropriately
# weighted.)
selections = map(
lambda instance__color: (
Key(
instance__color[0].slug,
instance__color[0].get_absolute_url(),
instance__color[1],
get_legend_data(reports[instance__color[0]]),
),
reports[instance__color[0]],
),
zip(instances, colors),
)[::-1]
# Collect any reports that weren't in the selection set, merge them
# together and add it at the top (front) of the stack.
overflow = set(reports) - set(instances)
if overflow:
overflow_report = reduce(merge_reports, [reports[instance] for instance in overflow])
selections.insert(
0, (Key("Other", None, "#f2f0fa", get_legend_data(overflow_report)), overflow_report)
)
def summarize(key, points):
total = sum(points)
return [(key, total)] if total else []
# Collect all of the independent series into a single series to make it
# easier to render, resulting in a series where each value is a sequence of
# (key, count) pairs.
series = reduce(
merge_series,
[series_map(functools.partial(summarize, key), report[0]) for key, report in selections],
)
legend = [key for key, value in reversed(selections)]
return {
"points": [(to_datetime(timestamp), value) for timestamp, value in series],
"maximum": max(sum(count for key, count in value) for timestamp, value in series),
"legend": {
"rows": legend,
"total": Key("Total", None, None, reduce(merge_mappings, [key.data for key in legend])),
},
}
def operation(sequence):
result = reduce(function, sequence, initializer(sequence))
logger.debug("%r reduced %s items to %s.", function, len(sequence),
len(result))
return result
def __pow__(self, order):
if int(order) == order and order >= 1:
return reduce(lambda a, b: Mul(a, b), order * [self])
else:
raise ValueError(
"Power of {} th order cannot be done.".format(order))
def get_max_combination_number(prameter_matrix, n):
param_len_list = [len(value_list) for value_list in prameter_matrix]
return sum([
reduce(lambda x, y: x * y, z) for z in combinations(param_len_list, n)
])
def gen_plan_feas_by_plans(plans_df):
"""
:param data:
:return:
"""
mode_columns_names = ['mode_feas_{}'.format(i) for i in range(12)]
def gen_mode_code(mode_list):
ma = np.zeros(12)
ma[mode_list] = 1
return ma
mode_g = plans_df.groupby('sid')['transport_mode'].apply(
gen_mode_code).reset_index()
mode_columns = ['sid'] + mode_columns_names
mode_data = np.concatenate(mode_g['transport_mode'].values,
axis=0).reshape(len(mode_g), 12)
sid_data = mode_g['sid'].values.reshape(len(mode_g), 1)
mode_df = pd.DataFrame(np.hstack([sid_data, mode_data]),
columns=mode_columns)
def get_first(x):
return x.values[0]
def gen_mode_texts(x):
tl = ' '.join(['word_{}'.format(mode) for mode in x.values])
return tl
agg_fun = {
'transport_mode': [get_first, gen_mode_texts],
'distance': ['max', 'min', 'mean', lambda x: np.std(x)],
'price': ['max', 'min', 'mean', lambda x: np.std(x)],
'eta': ['max', 'min', 'mean', lambda x: np.std(x)]
}
# std ddof =1
agg_columns = [
'sid', 'first_mode', 'mode_texts', 'max_dist', 'min_dist', 'mean_dist',
'std_dist', 'max_price', 'min_price', 'mean_price', 'std_price',
'max_eta', 'min_eta', 'mean_eta', 'std_eta'
]
agg_df = plans_df.groupby('sid').agg(agg_fun).reset_index()
agg_df.columns = agg_columns
merge_df = pd.merge(plans_df, agg_df, on=['sid'], how='inner')
# 原来版本是 keep='last'
max_dist_mode_df = merge_df.loc[merge_df['distance'] ==
merge_df['max_dist'],
['sid', 'transport_mode']]
max_dist_mode_df.columns = ['sid', 'max_dist_mode']
max_dist_mode_df.drop_duplicates(subset='sid', keep='last', inplace=True)
min_dist_mode_df = merge_df.loc[merge_df['distance'] ==
merge_df['min_dist'],
['sid', 'transport_mode']]
min_dist_mode_df.columns = ['sid', 'min_dist_mode']
min_dist_mode_df.drop_duplicates(subset='sid', keep='first', inplace=True)
max_price_mode_df = merge_df.loc[
merge_df['price'] == merge_df['max_price'], ['sid', 'transport_mode']]
max_price_mode_df.columns = ['sid', 'max_price_mode']
max_price_mode_df.drop_duplicates(subset='sid', keep='last', inplace=True)
min_price_mode_df = merge_df.loc[
merge_df['price'] == merge_df['min_price'], ['sid', 'transport_mode']]
min_price_mode_df.columns = ['sid', 'min_price_mode']
min_price_mode_df.drop_duplicates(subset='sid', keep='first', inplace=True)
max_eta_mode_df = merge_df.loc[merge_df['eta'] == merge_df['max_eta'],
['sid', 'transport_mode']]
max_eta_mode_df.columns = ['sid', 'max_eta_mode']
max_eta_mode_df.drop_duplicates(subset='sid', keep='last', inplace=True)
min_eta_mode_df = merge_df.loc[merge_df['eta'] == merge_df['min_eta'],
['sid', 'transport_mode']]
min_eta_mode_df.columns = ['sid', 'min_eta_mode']
min_eta_mode_df.drop_duplicates(subset='sid', keep='first', inplace=True)
complex_feature_df = reduce(
lambda ldf, rdf: pd.merge(ldf, rdf, on=['sid'], how='inner'), [
max_dist_mode_df, min_dist_mode_df, max_price_mode_df,
min_price_mode_df, max_eta_mode_df, min_eta_mode_df
])
plan_feature_df = reduce(
lambda ldf, rdf: pd.merge(ldf, rdf, on=['sid'], how='inner'),
[mode_df, agg_df, complex_feature_df])
return plan_feature_df
