def _extract_data(topic_model, corpus, dictionary):
doc_lengths = [sum([t[1] for t in doc]) for doc in corpus]
term_freqs_dict = fp.merge_with(sum, *corpus)
vocab = fp.map(dictionary, term_freqs_dict.keys())
term_freqs = term_freqs_dict.values()
gamma, _ = topic_model.inference(corpus)
doc_topic_dists = np.array([r / sum(r) for r in gamma])
topics = topic_model.show_topics(formatted=False,
num_words=len(vocab),
num_topics=topic_model.num_topics)
topics_df = pd.DataFrame(
[dict((y, x) for x, y in tuples) for tuples in topics])[vocab]
topic_term_dists = topics_df.values
return {
'topic_term_dists': topic_term_dists,
'doc_topic_dists': doc_topic_dists,
'doc_lengths': doc_lengths,
'vocab': vocab,
'term_frequency': term_freqs
}
def SagCWL2GCN_Binary_quick_max(
dsm,
cwl2_local_act="sigmoid",
cwl2_layer_widths=[64],
cwl2_layer_depths=[3],
cwl2_stack_tfs=["keep_input"]):
in_dim = dsm.dim_wl2_features()
hidden = [
[b] * l
for b, l in cart(cwl2_layer_widths, cwl2_layer_depths)]
hps = cart(
conv_layer_dims=[[in_dim, *h, 1] for h in hidden],
conv_act=["sigmoid"],
conv_local_act=[cwl2_local_act],
conv_stack_tf=cwl2_stack_tfs,
conv_bias=[True],
learning_rate=[0.001],
squeeze_output=[True]
)
duplicate_settings = {
"conv_layer_dims": ["att_conv_layer_dims"],
"conv_act": ["att_conv_act"],
"conv_local_act": ["att_conv_local_act"],
"conv_stack_tf": ["att_conv_stack_tf"],
"conv_bias": ["att_conv_bias"],
}
return fy.map(entry_duplicator(duplicate_settings), hps)
def contributor_logins(self, repos):
contributors = funcy.map(self.github.rx_contributors, repos)
return rx.Observable.concat(contributors) \
.buffer_with_count(len(repos)) \
.map(funcy.flatten) \
.map(funcy.partial(funcy.pluck, 'login'))
def __init__(self, character_paths, master=None):
super().__init__(master)
self.character_images = tuple(
map(
lambda character_path: PhotoImage(file=path.join(
character_path, 'image.ppm')), character_paths))
self.character_small_images = tuple(
map(
lambda character_path: PhotoImage(file=path.join(
character_path, 'small_image.ppm')), character_paths))
self.master.title('ダメ絶対（？）音感')
self.create_widgets()
self.pack()
def get_decay_curve(self, stepping: timedelta) -> Iterable[float]:
def calc_decay(t: int) -> float:
hl_steps = self.drug.half_life.total_seconds(
) / stepping.total_seconds()
factor = 2**(float(t) / hl_steps)
return self.amount * factor
return map(calc_decay, count())
def get_statistical_data(self, drug: str) -> Optional[Tuple[float, float]]:
# print(list(self.blood_level_factors.keys()))
# print(drug)
if drug not in self.blood_level_factors:
return None
blood_levels = list(
drop(
7 * 24,
take(
self.real_duration,
map(
lambda x: x[0] * x[1][0],
zip(self.drugs_timeline[drug],
self.factor_timeline[drug])))))
levels_avg = sum(blood_levels) / len(blood_levels)
sq_delta = list(map(lambda x: (x - levels_avg)**2, blood_levels))
levels_std_dev = math.sqrt(sum(sq_delta) / len(blood_levels))
return levels_avg, levels_std_dev
def draw_wave(self, wave):
min_ys, max_ys = zip(
*map(juxt(np.min, np.max), np.array_split(wave * 128 + 128, 256)))
for object_id in self.wave_canvas.find_all():
self.wave_canvas.delete(object_id)
for x, min_y, max_y in zip(count(), min_ys, max_ys):
self.wave_canvas.create_line(x, min_y, x, max_y)
def get_plot_lab_levels(
self,
use_date: bool = False
) -> Dict[str, Tuple[List[Union[int, datetime]], List[float]]]:
lab_levels = {}
for drug, ll_data in self.lab_levels.items():
start_time = datetime.combine(self.starting_date, time())
if use_date:
times = list(map(funcy.first, ll_data))
else:
times = list(
map(
lambda x: ((x[0] - start_time).total_seconds() /
(3600 * 24)), ll_data))
lab_levels[self.drugs[drug].name_blood] = (times,
list(
map(
lambda x: x[1],
ll_data)))
return lab_levels
def _eval(x):
tmp = f(x)
assert b >= a
if b > a:
# Force valuation at pivot points
if a < b < OO:
ts = fn.map(lambda t: interp_all(tmp, t - b - a + dt, tmp.end),
tmp.times())
tmp = reduce(op.__or__, ts, tmp)[tmp.start:tmp.end]
return tmp.rolling(a, b).map(_min, tag=phi)
return tmp.retag({phi.arg: phi})
def calculate_running_statistics(in_data: Tuple[Sequence[int], List[float], int]) \
-> Tuple[List[float], List[float]]:
steps, list_avg, i = in_data
sized_pot = SizedPot(steps[i])
group_sum = GroupSum()
group_sum.counts_needed(list(steps))
group_sum.set_data(list_avg)
average = lmap(
lambda s: s[0] / max(min(steps[i], s[1]), 1),
zip(map(group_sum.sum_getter(steps[i]), range(len(list_avg))),
range(len(list_avg))))
std_dev = lmap(lambda s: sized_pot.running_std_dev(*s),
zip(list_avg, average))
return average, std_dev
def main():
command = sys.argv[1]
kwargs, args = split(r'^--', sys.argv[2:])
kwargs = dict(map(r'^--(\w+)=(.+)', kwargs))
from .ext import django
django.register(args, kwargs)
files = FileSet(args, base=kwargs.get('base'), ignore=kwargs.get('ignore'))
if command == 'global':
global_usage(files)
elif command == 'local':
local_usage(files)
else:
print 'Unknown command', command
def get_infs(ip):
def _inf(record):
name = re_find(r'interface\s+(X?Gigabit\S+)', record)
desc = re_find(r'description\s+(\S+ *\S*)', record)
group = re_find(r'link-aggregation\s+(group\s+\d+)', record)
return dict(name=name, desc=desc, group=group)
try:
child = telnet(ip)
rslt = do_some(child, 'disp cu interface')
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
rslt1 = filter(r'X?GigabitEthernet', rslt.split('#'))
rslt2 = map(_inf, rslt1)
return ('success', rslt2, ip)
def get_infs(ip):
def _inf(record):
name = re_find(r'interface\s+(X?Gigabit\S+)', record)
desc = re_find(r'description\s+(\S+ *\S*)', record)
group = re_find(r'link-aggregation\s+(group\s+\d+)', record)
return dict(name=name, desc=desc, group=group)
try:
child = telnet(ip)
rslt = do_some(child, 'disp cu interface')
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
rslt1 = filter(r'X?GigabitEthernet', rslt.split('#'))
rslt2 = map(_inf, rslt1)
return ('success', rslt2, ip)
def get_infs(ip):
def _get_info(record):
name = re_find(r'interface\s(x?gigabitethernet\S+)',
record, flags=re.I)
desc = re_find(r'description\s(\S+ *\S*)', record)
group = re_find(r'(eth-trunk\s\d+)', record)
return dict(name=name, desc=desc, group=group)
try:
child = telnet(ip)
rslt = do_some(child, 'disp cu interface')
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
rslt1 = select(r'GigabitEthernet', rslt.split('#'))
rslt2 = map(_get_info, rslt1)
return ('success', rslt2, ip)
def object_name(submission, extension):
name_parts = F.flatten((
# Parts
submission['parts'],
# Names
['.'.join(s.get('name', '').split()) for s in submission['singers']],
# Location
F.keep(
submission.get('location', {}).get, ('city', 'state', 'country'))))
return '/'.join(
F.map(
sanitize_filename,
(
submission['singing'],
submission['song'],
# attach a uuid to the file name so that we never clobber uploads
'_'.join(filter(None, name_parts)) + '.' + str(uuid.uuid4()) +
extension)))
def main():
command = sys.argv[1]
opts, args = split(r'^--', sys.argv[2:])
opts = dict(map(r'^--([\w-]+)(?:=(.+))?', opts))
# Run ipdb on exception
if 'ipdb' in opts:
import ipdb, traceback
def info(type, value, tb):
traceback.print_exception(type, value, tb)
print()
# Insert look-around helpers into the frame
import inspect, ast
from .asttools import to_source
frame = inspect.getinnerframes(tb)[-1][0]
frame.f_globals.setdefault('ast', ast)
frame.f_globals.setdefault('to_source', to_source)
# Run debugger
ipdb.pm()
sys.excepthook = info
# Register plugins
from .ext import django
django.register(args, opts)
# Do the job
files = FileSet(args,
base=opts.get('base'),
ignore=opts.get('ignore'),
entry_points=opts.get('entry-points'))
if command == 'global':
global_usage(files)
elif command == 'local':
local_usage(files)
elif command == 'scope':
assert len(files) == 1
(filename, file), = files.items()
print(file.scope)
else:
print('Unknown command', command)
def get_groups(ip):
def _get_info(record):
name = re_find(r'interface\s(eth-trunk)(\d+)', record, flags=re.I)
name = ' '.join(name).lower()
mode = re_find(r'mode\s(\S+)', record)
if mode is None:
mode = 'manual'
elif 'lacp' in mode:
mode = 'yes'
desc = re_find(r'description\s(\S+ *\S*)', record)
return dict(name=name, mode=mode, desc=desc)
try:
child = telnet(ip)
rslt = do_some(child, 'disp cu int eth-trunk')
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
rslt1 = select(r'interface', rslt.split('#'))
rslt2 = map(_get_info, rslt1)
return ('success', rslt2, ip)
def bingfa_check():
funcs = {
'ME60': ME60.get_bingfa,
'ME60-X16': ME60.get_bingfa,
'M6000': M6k.get_bingfa,
'M6000-S': M6k.get_bingfa,
}
_get_bf = partial(_model, funcs)
clear()
nodes = graph.find('Bras')
bras = [(x['ip'], x['model']) for x in nodes]
rslt = map(_get_bf, bras)
with open(logFile, 'w') as flog, open(infoFile, 'w') as frslt:
for mark, record, ip in rslt:
flog.write('{ip}:{mark}\n'.format(ip=ip, mark=mark))
for slot, user, date in record:
frslt.write('{ip},{slot},{user},{date}\n'.format(ip=ip,
slot=slot,
user=user,
date=date))
def add_itv_online():
devices = {'ME60': ME60, 'ME60-X16': ME60, 'M6000-S': M6k, 'M6000': M6k}
action = 'get_itv_online'
nodes = graph.find('Bras')
bras = [(x['ip'], x['model']) for x in nodes]
funcs = map(lambda x: partial(getattr(devices.get(x[1]), action), x[0]),
bras)
with Pool(16) as p:
rslt = p.map(lambda f: f(), funcs)
rslt = filter(lambda x: x[0] == 'success', rslt)
update_time = datetime.now().strftime('%Y-%m-%d %H:%M')
cmd = """
match (b:Bras {ip:{ip}})
merge (b)-[:HAS]->(i:ITV)
set i.count={count},i.updateTime={time}
"""
tx = graph.cypher.begin()
lmap(lambda y: tx.append(cmd, ip=y[2], count=y[1], time=update_time), rslt)
tx.process()
tx.commit()
def main():
command = sys.argv[1]
opts, args = split(r'^--', sys.argv[2:])
opts = dict(map(r'^--([\w-]+)(?:=(.+))?', opts))
# Run ipdb on exception
if 'ipdb' in opts:
import ipdb, traceback
def info(type, value, tb):
traceback.print_exception(type, value, tb)
print()
# Insert look-around helpers into the frame
import inspect, ast
from .asttools import to_source
frame = inspect.getinnerframes(tb)[-1][0]
frame.f_globals.setdefault('ast', ast)
frame.f_globals.setdefault('to_source', to_source)
# Run debugger
ipdb.pm()
sys.excepthook = info
# Register plugins
from .ext import django
django.register(args, opts)
# Do the job
files = FileSet(args, base=opts.get('base'), ignore=opts.get('ignore'),
entry_points=opts.get('entry-points'))
if command == 'global':
global_usage(files)
elif command == 'local':
local_usage(files)
elif command == 'scope':
assert len(files) == 1
(filename, file), = files.items()
print(file.scope)
else:
print('Unknown command', command)
def SagK2GNN_FC_Binary(
dsm,
cwl2_local_act="sigmoid",
cwl2_layer_widths=[32, 64],
cwl2_layer_depths=[2, 4],
cwl2_stack_tfs=[None, "keep_input"]):
in_dim = dsm.dim_wl2_features()
hidden = [
([b] * l, [b, b])
for b, l in cart(cwl2_layer_widths, cwl2_layer_depths)]
hidden_hp = [dict(
conv_layer_dims=[in_dim, *ch],
fc_layer_dims=[*fh, 1]
) for ch, fh in hidden]
base_hp = cart(
conv_act=["sigmoid"],
conv_local_act=[cwl2_local_act],
conv_stack_tf=cwl2_stack_tfs,
conv_bias=[True],
fc_bias=[True],
learning_rate=[0.01, 0.001, 0.0001],
squeeze_output=[True]
)
hps = cart_merge(base_hp, hidden_hp)
duplicate_settings = {
"conv_layer_dims": ["att_conv_layer_dims"],
"conv_act": ["att_conv_act"],
"conv_local_act": ["att_conv_local_act"],
"conv_stack_tf": ["att_conv_stack_tf"],
"conv_bias": ["att_conv_bias"],
}
return fy.map(entry_duplicator(duplicate_settings), hps)
def assert_valid_bst(mode, ixy_map,
ixy_arr, tree, n_inserted, n_node):
''' tree is bst '''
key = prop(mode)
# Num of leaves ixy ref = num of inserted ixys
# Parent must be positive value except root.
for i,node in enumerate(tree[1:n_inserted+1]):
assert node.parent >= 0, (n_inserted, i, pyobj(node))
# Get ixy idxes from tree structure
ixy_idxes = all_ixy_idxes(
#tup_tree(tree[:n_inserted+50]))
tup_tree(tree[:n_node+100]))
if DBG: print(f' after[{n_node}]',#tup_tree(tree[:n_node+10]))
[f'{p} {l} {r}' for _,p,l,r in
tup_tree(tree[:n_node+10])])###########
if DBG: print('iidxes', ixy_idxes)
if DBG: print('n_node =',n_node)
# Inserted number of ixys preserved?
no0idxes = F.compact([abs(i) for i in ixy_idxes])
assert n_inserted == len(no0idxes), \
'ixy_idxes = {}, tup_tree = {}'.format(
ixy_idxes, tup_tree(tree[:n_inserted+4]))
# All ixy have unique index.
assert len(set(no0idxes)) == n_inserted,\
f'{len(set(no0idxes))} == {n_inserted}'
# All leaves point ixy(neg idx), not inode.
assert all(idx <= 0 for idx in ixy_idxes), \
'ixy_idxes = {}, tree = {}'.format(
ixy_idxes, tup_tree(tree[:n_inserted+4]))
# Inserted ixys are sorted in ascending order.
inserted_ixys = F.lmap(
lambda i: ixy_arr[abs(i)], ixy_idxes)
for ixy1, ixy2 in F.pairwise(inserted_ixys):
assert key(ixy1) <= key(ixy2), 'tree = {}' \
.format(tup_tree(tree[:n_inserted+4]))
# All leaves: l <= r
leaves = F.lfilter(is_leaf, tree[:n_inserted+4])
for leaf in leaves:
l = leaf.left; r = leaf.right
if l and r:
l_val = key(ixy_map[abs(l)])
r_val = key(ixy_map[abs(r)])
assert l_val <= r_val
# All inodes must be sorted in ascending order.
inodes = all_inodes(tup_tree(tree[:n_node+100]))
for n1, n2 in F.pairwise(inodes):
k1 = n1[0]; k2 = n2[0]
assert k1 <= k2
# Inserted ixys are sorted in ascending order.
neg_idxeseq = F.mapcat(tup(
lambda k,p,l,r:
((l,) if l < 0 else ()) + ((r,) if r < 0 else ())),
inodes)
ixy_idxes = F.map(abs, neg_idxeseq)
saved_ixys = F.map(lambda i: pyobj(ixy_arr[i]), ixy_idxes)
keys = F.lmap(key, saved_ixys)
for k1,k2 in F.pairwise(keys):
assert k1 <= k2
def map(f, *seq):
return F.map(f,*seq) if seq \
else lambda *xs: F.map(f,*xs)
def calc_std_dev(self, average: float) -> float:
sqsum = sum(map(lambda x: (x - average)**2, self.data))
if len(self.data) <= 1:
return 0.0
size_factor = 1 / (len(self.data) - 1.5)
return sqrt(size_factor * sqsum)
def _parse_query(q):
tags, words = split(r'^tag:', q.split())
tags = map(r'^tag:(.*)', tags)
return ' '.join(words), tags
def tmap(f,*seq):
return tuple(F.map(f,*seq)) if seq \
else lambda *xs: tuple(F.map(f,*xs))
def cached_as(*samples,
timeout=None,
extra=None,
lock=None,
keep_fresh=False,
key_func=func_cache_key):
"""
Caches results of a function and invalidates them same way as given queryset(s).
NOTE: Ignores queryset cached ops settings, always caches.
If keep_fresh is True, this will prevent caching if the given querysets are
invalidated during the function call. This prevents prolonged caching of
stale data.
"""
if not samples:
raise TypeError('Pass a queryset, a model or an object to cache like')
# If we unexpectedly get list instead of queryset return identity decorator.
# Paginator could do this when page.object_list is empty.
if len(samples) == 1 and isinstance(samples[0], list):
return lambda func: func
def _get_queryset(sample):
if isinstance(sample, models.Model):
queryset = sample.__class__.objects.filter(pk=sample.pk)
elif isinstance(sample, type) and issubclass(sample, models.Model):
queryset = sample.objects.all()
else:
queryset = sample
queryset._require_cacheprofile()
return queryset
querysets = lmap(_get_queryset, samples)
dbs = list({qs.db for qs in querysets})
cond_dnfs = join_with(lcat, map(dnfs, querysets))
key_extra = [qs._cache_key(prefix=False) for qs in querysets]
key_extra.append(extra)
if timeout is None:
timeout = min(qs._cacheprofile['timeout'] for qs in querysets)
if lock is None:
lock = any(qs._cacheprofile['lock'] for qs in querysets)
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
if not settings.CACHEOPS_ENABLED or transaction_states.is_dirty(
dbs):
return func(*args, **kwargs)
prefix = get_prefix(func=func, _cond_dnfs=cond_dnfs, dbs=dbs)
cache_key = prefix + 'as:' + key_func(func, args, kwargs,
key_extra)
with redis_client.getting(cache_key, lock=lock) as cache_data:
cache_read.send(sender=None,
func=func,
hit=cache_data is not None)
if cache_data is not None:
return pickle.loads(cache_data)
else:
if keep_fresh:
# We call this "asp" for "as precall" because this key is
# cached before the actual function is called. We randomize
# the key to prevent falsely thinking the key was not
# invalidated when in fact it was invalidated and the
# function was called again in another process.
suffix = key_func(func, args, kwargs,
key_extra + [random()])
precall_key = prefix + 'asp:' + suffix
# Cache a precall_key to watch for invalidation during
# the function call. Its value does not matter. If and
# only if it remains valid before, during, and after the
# call, the result can be cached and returned.
cache_thing(prefix,
precall_key,
'PRECALL',
cond_dnfs,
timeout,
dbs=dbs)
else:
precall_key = ''
result = func(*args, **kwargs)
cache_thing(prefix,
cache_key,
result,
cond_dnfs,
timeout,
dbs=dbs,
precall_key=precall_key)
return result
return wrapper
return decorator
def store_graphs_as_tu_data(graphs,
targets,
dim_node_features=1,
dim_edge_features=1,
num_node_labels=1,
num_edge_labels=1,
name=None,
raw_dir=None):
if raw_dir is None or name is None:
raise Exception("Missing dataset name or destination raw_dir.")
base_dir = raw_dir / name
indicator_path = base_dir / f'{name}_graph_indicator.txt'
edges_path = base_dir / f'{name}_A.txt'
graph_labels_path = base_dir / f'{name}_graph_labels.txt'
if num_node_labels > 0:
node_labels_path = base_dir / f'{name}_node_labels.txt'
else:
node_labels_path = os.devnull
if num_edge_labels > 0:
edge_labels_path = base_dir / f'{name}_edge_labels.txt'
else:
edge_labels_path = os.devnull
if dim_node_features > 0:
node_attrs_path = base_dir / f'{name}_node_attributes.txt'
else:
node_attrs_path = os.devnull
if dim_edge_features > 0:
edge_attrs_path = base_dir / f'{name}_edge_attributes.txt'
else:
edge_attrs_path = os.devnull
if not base_dir.exists():
os.makedirs(base_dir)
with\
open(indicator_path, "w") as indicator,\
open(edges_path, "w") as edges,\
open(graph_labels_path, "w") as graph_labels,\
open(node_labels_path, "w") as node_labels,\
open(edge_labels_path, "w") as edge_labels,\
open(node_attrs_path, "w") as node_attrs,\
open(edge_attrs_path, "w") as edge_attrs:
v_offset = 1
for i, (g, target) in enumerate(zip(graphs, targets), 1):
v_lookup = {}
graph_labels.write(f"{target}\n")
for n, data in g.nodes(data=True):
indicator.write(f"{i}\n")
feat = ",".join(fy.map(str, data.get("features", [])))
node_attrs.write(f"{feat}\n")
label = data.get("label")
if label is not None:
node_labels.write(f"{label}\n")
v_lookup[n] = v_offset
v_offset += 1
for u, v, data in g.edges(data=True):
edges.write(f"{v_lookup[u]},{v_lookup[v]}\n")
feat = ",".join(fy.map(str, data.get("features", [])))
edge_attrs.write(f"{feat}\n")
label = data.get("label")
if label is not None:
edge_labels.write(f"{label}\n")
def summarize_evaluation(eval_dir,
selection_metric="val_accuracy",
ignore_worst=0):
if not eval_dir.exists():
print(f"No evalutation '{eval_dir}' found.")
return
with open(eval_dir / "config.json") as f:
config = json.load(f)
with open(eval_dir / "hyperparams.json") as f:
hps = json.load(f)
results_dir = eval_dir / "results"
assert results_dir.exists(), f"No results found for '{eval_dir}'."
summary_dir = eval_dir / "summary"
if not summary_dir.exists():
os.makedirs(summary_dir)
result_files = [(list(fy.map(int, f[:-5].split("-"))), results_dir / f)
for f in os.listdir(results_dir)]
fold_files = fy.group_by(lambda f: f[0][0], result_files)
fold_param_files = {
fold: fy.group_by(lambda f: f[0][1], files)
for fold, files in fold_files.items()
}
folds = list(fold_param_files.items())
folds.sort(key=fy.first)
best_goal = selection_metrics[selection_metric]
results = []
all_hps = True
for fold_i, param_files in folds:
best_res = None
param_file_items = list(param_files.items())
all_hps = all_hps and len(param_files) == len(hps)
for hp_i, files in param_file_items:
hp_train_results = defaultdict(list)
hp_test_results = defaultdict(list)
selection_vals = []
all_selection_vals = []
for (_, _, i), file in files:
with open(file, "r") as f:
result = json.load(f)
selection_val = result["train"][selection_metric][-1]
all_selection_vals.append(selection_val)
if i < config["repeat"]:
selection_vals.append(selection_val)
for metric, val in result["train"].items():
hp_train_results[metric].append(val[-1])
for metric, val in result["test"].items():
hp_test_results[metric].append(val)
top_idxs = np.argsort(np.array(all_selection_vals))
if len(all_selection_vals) > ignore_worst:
if best_goal == "max":
top_idxs = top_idxs[ignore_worst:]
elif best_goal == "min":
top_idxs = top_idxs[:-ignore_worst]
top_statistics = fy.compose(statistics,
lambda l: np.array(l)[top_idxs])
hp_res = dict(fold_idx=fold_i,
train=dict_map(top_statistics, hp_train_results),
test=dict_map(top_statistics, hp_test_results),
select=np.mean(selection_vals),
hp_i=hp_i,
hp=hps[hp_i],
select_repeats=len(selection_vals),
eval_repeats=len(files))
if (best_res is None or
(best_goal == "max" and best_res["select"] < hp_res["select"])
or
(best_goal == "min" and best_res["select"] > hp_res["select"])
or
(best_res["select"] == hp_res["select"]
and best_res["eval_repeats"] < hp_res["eval_repeats"])):
best_res = hp_res
if best_res is not None:
results.append(best_res)
else:
print(f"No results for {fold_i}.")
combined_train = dict_map(
statistics,
fy.merge_with(
np.array,
*map(lambda res: dict_map(lambda t: t["mean"], res["train"]),
results)))
combined_test = dict_map(
statistics,
fy.merge_with(
np.array,
*map(lambda res: dict_map(lambda t: t["mean"], res["test"]),
results)))
results_summary = {
"folds": results,
"combined_train": combined_train,
"combined_test": combined_test,
"args": {
"ignore_worst": ignore_worst
},
"done": all_hps and len(folds) == 10
}
with open(summary_dir / "results.json", "w") as f:
json.dump(results_summary, f, cls=NumpyEncoder, indent="\t")
return results_summary
def interp_all(sig, t, end=OO):
v = fn.map(lambda u: signal([(t, interp(sig, t, u))], t, end, u), sig.tags)
return reduce(op.__or__, v)
def contributors_sorted_by_repos_contributed_in(self, contributors):
users_by_repos = funcy.count_by(None, contributors).items()
sorted_users = sorted(users_by_repos, key=itemgetter(1), reverse=True)
return funcy.map(0, sorted_users)
def test_list_iter():
assert is_list(py2.map(None, []))
assert is_iter(py3.map(None, []))
assert is_list(funcy.map(None, [])) == PY2
assert is_iter(funcy.map(None, [])) == PY3
def estimate_blood_levels(self, corrected_std_dev: bool = True):
self.lab_events = {}
for lab_data in self.labs_list:
for d in lab_data.labs.keys():
self.lab_levels[d] = []
self.lab_events[d] = []
for lab_data in self.labs_list:
for d, val in lab_data.labs.items():
self.lab_levels[d].append((lab_data.time, val))
if len(self.events) > 0:
for x in range(len(self.events) + 1):
self.lab_events[d].append([])
for n, event in enumerate(self.events):
if n == 0 and lab_data.time < datetime.combine(
event[0], time()):
self.lab_events[d][0].append((lab_data.time, val))
break
if lab_data.time > datetime.combine(event[0], time()):
self.lab_events[d][n + 1].append(
(lab_data.time, val))
break
else:
if len(self.lab_events[d]) == 0:
self.lab_events[d].append([])
self.lab_events[d][0].append((lab_data.time, val))
drugs = set(self.lab_levels.keys())
for d in drugs:
if d in self.lab_levels:
if d not in self.blood_level_factors:
self.blood_level_factors[d] = []
for _ in range(len(self.events) + 1):
self.blood_level_factors[d].append((0.0, 0.0))
for event_num in range(len(self.events) + 1):
level_matcher = list(
map(
lambda x:
(x[1], self.get_drug_at_timepoint(d, x[0])),
self.lab_events[d][event_num]))
estimates = list(
map(lambda x: (x[0] / x[1]), level_matcher))
average_est = sum(
map(lambda x: x / len(estimates), estimates))
levels = list(
map(lambda x: (x[0], x[1] * average_est),
level_matcher))
if len(estimates) > 1 and corrected_std_dev:
std_dev = math.sqrt(
sum(map(lambda x: (x[0] - x[1])**2, levels)) /
(len(levels) - 1.5))
# std_dev = math.sqrt(sum(map(lambda x: (x - average_est)**2, estimates)) / (len(estimates)-1))
else:
std_dev = math.sqrt(
sum(map(lambda x:
(x[0] - x[1])**2, levels)) / len(levels))
# std_dev = math.sqrt(sum(map(lambda x: (x - average_est)**2, estimates)) / len(estimates))
self.blood_level_factors[d][event_num] = (average_est,
std_dev)
def test_list_iter():
assert is_list(py2.map(None, []))
assert is_iter(py3.map(None, []))
assert is_list(funcy.map(None, [])) == PY2
assert is_iter(funcy.map(None, [])) == PY3
def get_plot_data(self,
plot_delta: timedelta = timedelta(days=1),
adjusted: bool = False,
stddev_multiplier: float = 1.0,
offset: float = 0.0,
color: bool = False,
use_x_date: bool = False) -> \
Tuple[np.ndarray, Dict[str, plot_data_type]]:
t_arr = take(
self.duration,
map(
lambda x: x * (self.step.total_seconds() / plot_delta.
total_seconds()) + offset, count()))
if use_x_date:
t_arr = map(
lambda x: datetime.combine(self.starting_date, time()) +
plot_delta * x, t_arr)
t_arr = np.array(list(t_arr))
# print(f't_arr.size()={len(t_arr)}')
out = {}
drugs = sorted(list(self.drugs_timeline.keys()),
key=lambda x: self.drugs[x].name)
step_time_d = int(self.step.total_seconds())
steps = (
int(
math.ceil(
int(timedelta(days=self.step_days[0]).total_seconds()) /
step_time_d)),
int(
math.ceil(
int(timedelta(days=self.step_days[1]).total_seconds()) /
step_time_d)),
int(
math.ceil(
int(timedelta(days=self.step_days[2]).total_seconds()) /
step_time_d)))
self.running_average = {}
self.running_stddev = {}
for n, drug in enumerate(drugs):
# print(f"{n}: {drug}")
timeline = self.drugs_timeline[drug]
drug_name = self.drugs[drug].name_blood
# print(f"{steps}/{len(timeline)}")
if self.drugs[drug].factor != 1.0:
drug_name += f" (x{self.drugs[drug].factor})"
if adjusted and drug in self.blood_level_factors and len(
self.blood_level_factors[drug]) > 0:
# print(self.blood_level_factors)
factor_timeline = []
ev_num = 0
for t in range(len(timeline)):
t_time = datetime.combine(self.starting_date,
time()) + t * self.step
if len(self.events) > 0:
if len(self.blood_level_factors[drug]) > ev_num + 1:
if datetime.combine(self.events[ev_num][0], time())+self.events[ev_num][1] > \
t_time > datetime.combine(self.events[ev_num][0], time()):
factor: timedelta = t_time - datetime.combine(
self.events[ev_num][0], time())
factor: float = factor / self.events[ev_num][1]
factor_timeline.append((
self.blood_level_factors[drug][ev_num +
1][0] *
factor +
self.blood_level_factors[drug][ev_num][0] *
(1 - factor),
self.blood_level_factors[drug][ev_num +
1][1] *
factor +
self.blood_level_factors[drug][ev_num][1] *
(1 - factor)))
elif datetime.combine(
self.events[ev_num][0],
time()) + self.events[ev_num][1] <= t_time:
ev_num += 1
factor_timeline.append(
self.blood_level_factors[drug][ev_num])
else:
factor_timeline.append(
self.blood_level_factors[drug][ev_num])
else:
factor_timeline.append(
self.blood_level_factors[drug][ev_num])
else:
factor_timeline.append(
self.blood_level_factors[drug][0])
self.factor_timeline[drug] = factor_timeline
list_avg = lmap(lambda x: x[0] * x[1][0],
zip(timeline, factor_timeline))
arr_avg = np.array(list_avg)
arr_min = np.array(
lmap(
lambda x: x[0] * x[1][0] - x[1][1] * stddev_multiplier,
zip(timeline, factor_timeline)))
arr_max = np.array(
lmap(
lambda x: x[0] * x[1][0] + x[1][1] * stddev_multiplier,
zip(timeline, factor_timeline)))
mp_ctx = mp.get_context('fork')
statistics_data: List[Tuple[Sequence[int], List[float], int]]
statistics_data = [(steps, list_avg, i) for i in range(3)]
with mp_ctx.Pool(3) as mp_pool:
statistics_results = mp_pool.map(
calculate_running_statistics, statistics_data)
running_average, running_std_dev = tuple(
map(list, list(zip(*statistics_results))))
self.running_average[drug_name] = tuple(
map(np.array, running_average))
self.running_stddev[drug_name] = tuple(
map(np.array, running_std_dev))
if color:
# print(f"{drug}: {n} => {get_color(n)}")
out[drug_name] = (arr_avg, arr_min, arr_max, get_color(n))
else:
out[drug_name] = (arr_avg, arr_min, arr_max)
else:
arr = np.array(timeline) * self.drugs[drug].factor
if color:
out[drug_name] = (arr, arr, arr, get_color(n))
else:
out[drug_name] = (arr, arr, arr)
# print(f't_arr.size({drug.name})={len(out[drug.name])}')
return t_arr, out
