def flatten_message(self, message):
# TODO Check if messages are nested/contain lists
def snake_case(name):
return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
return funcy.walk_keys(snake_case, message)
def create_rps_game(n=1, eps=0):
assert n >= 1
k = n - 1
delta = 60 / (6 * n)
# TODO
context = {
'xRock': create_rock(k=k, p1=True, delta=delta, eps=eps),
'xScissors': create_scissors(k=k, p1=True, delta=delta, eps=eps),
'xPaper': create_paper(k=k, p1=True, delta=delta, eps=eps),
'yRock': create_rock(k=k, p1=False, delta=delta),
'yScissors': create_scissors(k=k, p1=False, delta=delta),
'yPaper': create_paper(k=k, p1=False, delta=delta),
}
context = fn.walk_keys(stl.parse, context)
context.update(CONTEXT)
spec = G.Specs(
obj=stl.parse('G(Rules)', H=H).inline_context(context),
learned=stl.TOP,
init=stl.parse("Init").inline_context(context),
)
return G.Game(specs=spec, model=MODEL)
def _get_data(self):
qs = SampleAnnotation.objects.values(*self.KEYS).prefetch_related(
'sample',
'sample__platform',
'series_annotation__tag',
).filter(series_annotation_id__in=self.metadata.get('ids', []))
return [walk_keys(self.KEYS, annotation) for annotation in qs.iterator()]
def unroll_keys(inputs):
nonlocal time
time += 1
def unroll_key(key):
return f"{key}##time_{time}"
return fn.walk_keys(unroll_key, inputs)
def get_vertical_cages(rows):
transpose_coordinates = lambda t: (t[1], t[0])
res = get_horizontal_cages(transpose(rows))
res = walk_keys(transpose_coordinates, res)
transpose_coordinates_lst = compose(list,
partial(map, transpose_coordinates))
res = walk_values(transpose_coordinates_lst, res)
return res
def get_annotation():
return JsonResponse(
[walk_keys(self.KEYS, annotation) for annotation in
SampleAnnotation.objects.values(*self.KEYS).prefetch_related(
'sample',
'sample__platform',
'series_annotation__tag',
).iterator()],
safe=False)
def _update():
bar = next(self.iteral_buffer)
bar = fy.walk_keys(lambda x: x.lower(), bar)
bar["date"] = self.__set_dtformat(bar)
for i in bar:
try:
bar[i] = float(bar[i]) # 将数值转化为float
except ValueError:
pass
return bar
def __update():
new_bar = next(self._iteral_data)
new_bar = fy.walk_keys(lambda x: x.lower(), new_bar)
new_bar["date"] = self.__set_dtformat(new_bar)
for i in new_bar:
try:
new_bar[i] = float(new_bar[i]) # 将数值转化为float
except ValueError:
pass
return new_bar
def _update():
bar = next(self.iteral_data2)
bar = fy.walk_keys(lambda x: x.lower(), bar)
bar['date'] = self.set_dtformat(bar)
for i in list(bar.keys()):
try:
bar[i] = float(bar[i]) # 将数值转化为float
except ValueError:
pass
return bar
def add(self, config, strip_app_name=False, filter_by_app_name=False,
key_normalisation_func=default_key_normalisation_func):
"""
Add a dict of config data. Values from later dicts will take precedence
over those added earlier, so the order data is added matters.
Note: Double underscores can be used to indicate dict key name
boundaries. i.e. if we have a dict like:
{
'logging': {
'level': INFO
...
}
}
we could pass an environment variable LOGGING__LEVEL=DEBUG to override
the log level.
Note: Key names will be normalised by recursively applying the
key_normalisation_func function. By default this will:
1) Convert keys to lowercase
2) Replace hyphens with underscores
3) Strip leading underscores
This allows key names from different sources (e.g. CLI args, env vars,
etc.) to be able to override each other.
:param config dict: config data
:param strip_app_name boolean: If True, the configured app_name will
stripped from the start of top-level input keys if present.
:param filter_by_app_name boolean: If True, keys that don't begin with
the app name will be discarded.
:return:
"""
config = walk_recursive(key_normalisation_func, OrderedDict(config))
if filter_by_app_name:
config = funcy.compact(funcy.select_keys(
lambda k: k.startswith(self._app_name), config))
if strip_app_name:
strip_app_name_regex = re.compile("^%s" % self._app_name)
config = funcy.walk_keys(
lambda k: re.sub(strip_app_name_regex, '', k), config)
self._sources.append(config)
return self # enables a fluent interface
def flatten(self, actions: Actions) -> Bits:
"""
Converts structured sequence of (sys, env) actions to a
sequence of bits that this concrete specification recognizes.
"""
timed_actions = {}
bmap = self.imap + self.emap
for t, action in enumerate(actions):
old2new = {k: f'{k}##time_{t}' for k in bmap.keys()}
bmap_t = bmap.relabel(old2new)
action_t = fn.walk_keys(old2new.get, action)
timed_actions.update(bmap_t.blast(action_t))
idx2key = bidict(self.bexpr.bdd.vars).inv
return [timed_actions[idx2key[i]] for i in range(len(idx2key))]
def _update_gamelog_keys(self, gamelog):
"""
Removes Player key and switches MainPlayerCode and OppPlayerCode
keys if the MainPlayerCode is not player_code.
:param gamelog: A dict of stats.
:returns: Gamelog with proper keys placement.
"""
# @TODO This is so sad.
gamelog.pop('Player', None)
if self.player_one_code != gamelog['MainPlayerCode']:
replacer = partial(multiple_replace,
adict={'Main': 'Opp', 'Opp': 'Main'})
return walk_keys(replacer, gamelog)
return gamelog
def __getitem__(self, others):
if not isinstance(others, tuple):
return super().__getitem__(others)
kind, relabels = others
if kind not in {'i', 'o', 'l'}:
raise NotImplementedError
attr_name = {
'i': 'input_map',
'o': 'output_map',
'l': 'latch_map',
}.get(kind)
attr_value = fn.walk_keys(lambda x: relabels.get(x, x),
getattr(self, attr_name))
return attr.evolve(self, **{attr_name: attr_value})
def to_bdd(circ_or_expr, output=None, manager=None, renamer=None, levels=None):
if renamer is None:
_count = 0
def renamer(*_):
nonlocal _count
_count += 1
return f"x{_count}"
if not isinstance(circ_or_expr, aiger.BoolExpr):
circ = aiger.to_aig(circ_or_expr, allow_lazy=True)
assert len(circ.latches) == 0
if output is None:
assert len(circ.outputs) == 1
output = fn.first(circ.outputs)
expr = aiger.BoolExpr(circ)
else:
expr = circ_or_expr
manager = BDD() if manager is None else manager
input_refs_to_var = {
ref: renamer(i, ref)
for i, ref in enumerate(expr.inputs)
}
manager.declare(*input_refs_to_var.values())
if levels is not None:
assert set(manager.vars.keys()) <= set(levels.keys())
levels = fn.project(levels, manager.vars.keys())
levels = fn.walk_keys(input_refs_to_var.get, levels)
manager.reorder(levels)
manager.configure(reordering=False)
def lift(obj):
if isinstance(obj, bool):
return manager.true if obj else manager.false
return obj
inputs = {i: manager.var(input_refs_to_var[i]) for i in expr.inputs}
out = expr(inputs, lift=lift)
return out, out.bdd, bidict(input_refs_to_var)
def export(self):
""" This method returns a dictionary that is type-safe to store as JSON or in a database.
"""
followers = self.get_followers()
following = self.get_following()
return {
**self.get_props(),
"profile": self.profile,
"sp": self.sp,
"rep": self.rep,
"balances": walk_keys(str.upper, self.get_balances()),
"followers": followers,
"followers_count": len(followers),
"following": following,
"following_count": len(following),
"curation_stats": self.curation_stats(),
"withdrawal_routes": self.get_withdraw_routes(),
"conversion_requests": self.get_conversion_requests(),
"account_votes": self.get_account_votes(),
}
def _normalize_options(self, query, options):
"""
Here we parse some search tokens diffrently to enable filtering:
GSE\d+ and GPL\d+ filter by specific serie or platform
tag=\w+ filters by tag
valid selects validated annotations
"""
options = DatatableOptions._normalize_options(self, query, options)
# Try normally named field
if not options['search']:
options['search'] = query.get('search', '').strip()
filters = group_by(r'^(GSE|GPL|[Tt]ag=|valid|novalid)',
options['search'].split())
options['search'] = ' '.join(filters.pop(None, []))
filters = walk_keys(str.lower, filters)
filters['tag'] = lmap(r'^[Tt]ag=(.*)', filters.pop('tag=', []))
options['filters'] = filters
return options
def _introspect(func, seen):
seen.add(func)
if inspect.isbuiltin(func) or not hasattr(func, '__module__') or not hasattr(func, '__name__'):
return {}
if isinstance(func, type):
methods = inspect.getmembers(func, predicate=inspect.ismethod)
return join(_introspect(meth, seen) for _, meth in methods if meth not in seen) or {}
if not hasattr(func, '__defaults__') or not hasattr(func, '__code__'):
return {}
func_name = _full_name(func)
consts = merge(get_defaults(func), get_assignments(func))
consts_spec = walk_keys(lambda k: '%s.%s' % (func_name, k), consts)
consts_spec.update({'%s.%s' % (func.__module__, name): value
for name, value in get_closure(func).items()})
# Recurse
callables = filter(callable, consts_spec.values())
recurse_specs = (_introspect(f, seen) for f in callables if f not in seen)
return merge(join(recurse_specs) or {}, consts_spec)
def clean(self):
data = super(AnnotateForm, self).clean()
# Check that platform and series match
if data.get('series') and data.get('platform') \
and data['platform'].gpl_name not in data['series'].platforms:
raise ValidationError({
'series':
"Series %s doesn't contain platform %s" %
(data['series'].gse_name, data['platform'].gpl_name)
})
# Remap sample annotations gsm -> id and check that they correspond to series/platform
if data.get('annotations') and data.get('series') and data.get(
'platform'):
samples_qs = Sample.objects.filter(series=data['series'],
platform=data['platform'])
gsm_to_id = dict(samples_qs.values_list('gsm_name', 'id'))
all_samples = set(gsm_to_id)
tagged_samples = set(data['annotations'])
if all_samples - tagged_samples:
self.add_error(
'annotations',
"These samples are missing from annotations: %s" %
', '.join(sorted(all_samples - tagged_samples)))
if tagged_samples - all_samples:
self.add_error(
'annotations',
"These samples doesn't belong to series/platform: %s" %
', '.join(sorted(tagged_samples - all_samples)))
if data.get('annotations'):
data['annotations'] = walk_keys(gsm_to_id, data['annotations'])
return data
def walk_recursive(f, data):
"""
Recursively apply a function to all dicts in a nested dictionary
:param f: Function to apply
:param data: Dictionary (possibly nested) to recursively apply
function to
:return:
"""
results = {}
if isinstance(data, list):
return [walk_recursive(f, d) for d in data]
elif isinstance(data, dict):
results = funcy.walk_keys(f, data)
for k, v in data.iteritems():
if isinstance(v, dict):
results[f(k)] = walk_recursive(f, v)
elif isinstance(v, list):
results[f(k)] = [walk_recursive(f, d) for d in v]
else:
return f(data)
return results
def validate(self, data):
gsm_to_id = dict(data['series'].samples.filter(
platform=data['platform']).values_list('gsm_name', 'id'))
all_samples = set(gsm_to_id)
tagged_samples = set(data['annotations'])
missing_annotations = all_samples - tagged_samples
if missing_annotations:
raise serializers.ValidationError([
"There are samples with ids {0} which are missing their annotation"
.format(missing_annotations)
])
extra_annotations = tagged_samples - all_samples
if extra_annotations:
raise serializers.ValidationError([
"There is samples with id {0} which doesn't belongs to series {1}"
.format(extra_annotations, data['series'].id)
])
data['annotations'] = walk_keys(gsm_to_id, data['annotations'])
return data
def read_names():
with open('dict.txt') as f:
names = {l.strip(): l.strip() for l in f.readlines()}
return {**walk_keys(ocr_normalize, names), **names}
def walk_keys(func, mapping):
return fn.walk_keys(func, dict(mapping))
def __matmul__(self, other):
return self.evolve(data=fn.merge(
self.data, fn.walk_keys(lambda t: t + self.end, other.data)),
end=self.end + (other.end - other.start))
def spec_mle(mdp, demos, specs, top=100, parallel=False, psat=None):
"""
Searches for the most likely specification in specs given
demonstations, demos, from an agent operating in mdp.
"""
horizon = len(demos[0][0])
specs = list(specs)
print("encoding traces")
tree = prefix_tree(mdp, demos)
print("done encoding traces")
@fn.memoize
def score(spec):
start_time = time.time()
times = {}
print("concretizing spec")
cspec = concretize(spec, mdp, horizon)
print("done spec")
times["build spec"] = time.time() - start_time
if psat is None:
sat_prob = tree.psat(cspec)
else:
sat_prob = psat
start_time = time.time()
print("fitting policy")
ctrl = fit(cspec, sat_prob, bv=True)
print("done fitting")
times["fit"] = time.time() - start_time
start_time = time.time()
print("compute log likelihood of demos")
lprob = tree.log_likelihood(ctrl, actions_only=True)
times["surprise"] = time.time() - start_time
print("\n----------------------------\n")
print(f"BDD size: {cspec.bexpr.dag_size}")
print(f"Controller Size: {ctrl.size}")
print(f"log_prob: {lprob}")
print("\n".join(f"{key}: {val:.2}s" for key, val in times.items()))
print("\n----------------------------\n")
print(times)
return lprob
if parallel:
_specs = list(enumerate(specs))
def score2(spec):
i, spec = spec
return i, score(spec)
spec2score = dict(Pool().map(score2, _specs))
spec2score = fn.walk_keys(lambda idx: specs[idx], spec2score)
best_spec = max(specs, key=spec2score.get)
else:
best_spec = max(specs, key=score)
spec2score = fn.walk_keys(lambda x: x[0], score.memory)
return best_spec, spec2score
def validate(self, data):
data['encrypted_keys'] = walk_keys(int, data['encrypted_keys'])
return data
def __rshift__(self, delta):
return self.evolve(
data=fn.walk_keys(lambda t: t + delta, self.data),
start=self.start + delta,
end=self.end + delta,
)
def keys_to_lower(dic):
return walk_keys(lambda k: k.lower(), dic)
def _retag(val):
return fn.walk_keys(lambda k: mapping.get(k, k), val)
def distribute_series_and_sample_annotations(qs):
series_annotations = distribute_by_created_on(qs)
values = qs.values_list('created_on', 'samples')
group = group_values(walk_keys(ceil_date, values.iterator()))
return series_annotations, accumulate(walk_values(sum, group))
