def test_kfold():
output = load_train_df(
dataset_dir='/store/tellus/train',
output='/store/tmp/train.pqt'
)
df = pd.read_parquet(output)
sets = kfold(df, n_splits=10)
for s in sets:
assert pipe(
s['train_pos'],
take(100),
map(lambda x: x['label']),
filter(lambda x: x == 0),
list,
len
) == 0
assert pipe(
s['val_pos'],
take(100),
map(lambda x: x['label']),
filter(lambda x: x == 0),
list,
len
) == 0
assert pipe(
s['train_neg'],
take(100),
map(lambda x: x['label']),
filter(lambda x: x == 1),
list,
len
) == 0
assert pipe(
s['val_neg'],
take(100),
map(lambda x: x['label']),
filter(lambda x: x == 1),
list,
len
) == 0
assert len(s) == 4
def __init__(self,
feature_size=8,
depth=3,
):
super().__init__()
self.down_layers = nn.ModuleList([
DownSample(1, feature_size * 2 ** depth),
*pipe(
range(depth),
reversed,
map(lambda x: DownSample(
feature_size * (2 ** (x + 1)),
feature_size * (2 ** x),
)),
list,
)
])
self.center = DownSample(
in_ch=feature_size,
out_ch=feature_size,
)
self.up_layers = nn.ModuleList([
*pipe(
self.down_layers,
reversed,
map(lambda x: x.out_ch),
take(depth),
map(lambda x: UpSample(
feature_size,
feature_size,
x,
)),
list,
),
UpSample(
feature_size,
feature_size,
feature_size * 2 ** depth,
),
])
self._output = nn.Conv2d(
feature_size,
2,
kernel_size=3
)
def construct_group_dict(group_path, config):
"""
Given a config and a path that points to a data group, compute the data group's updated parameters.
The group_path is a list of keys and indices e.g. ['train', 'datasets', 1, 'groups', 0]
that can be followed to reach a group's config.
"""
# Find (almost) all prefixes of the group path
all_paths = list(
map(compose(list, tz.take(seq=group_path)),
range(1, len(group_path))))
# Filter to exclude paths that point to lists
paths_to_merge = list(
filter(lambda p: isinstance(last(p[1]), str), pairwise(all_paths)))
# Find all the (mid-level) dicts that the filtered paths point to
mid_level_dicts = list(
map(
lambda p: tz.keyfilter(lambda k: k != last(p[1]),
tz.get_in(p[0], config)),
paths_to_merge))
# Merge parameters at all levels to get a single parameter set for the group
def dmerge(*args):
if all(is_mapping, *args):
return Munch(tz.merge(*args))
else:
return tz.last(*args)
group_dict = tz.merge_with(
dmerge,
tz.keyfilter(lambda k: k not in ['train', 'val', 'test'],
config), # top-level dict
*mid_level_dicts, # mid-level dicts
tz.get_in(group_path, config) # bottom-level dict
)
return group_dict
def test_take():
assert list(take(2)([1, 2, 3])) == [1, 2]
def test_take():
assert list(take(2)([1, 2, 3])) == [1, 2]
step_fn = next(step_fns)
rotation_break_seq = set()
while True:
if is_odd_square(n - 1):
step_fn = next(step_fns)
side_length += 2
delta_seq = [
side_length - 2, side_length - 2 + 1, side_length - 2 + 1
]
rotation_break_seq = cc.pipe(it.accumulate([n] + delta_seq),
cc.drop(1), set)
elif n in rotation_break_seq:
step_fn = next(step_fns)
sum_dict[pos] = neighbors_sum(pos, sum_dict)
yield (pos, sum_dict[pos])
pos = step_fn(pos)
n += 1
answer = cc.pipe(sum_path(),
cc.curry(it.dropwhile)(lambda x: x[1] <= int(sys.argv[1])),
cc.take(1), list, lambda x: x[0][1])
pp(answer)
import itertools as itls
# In[75]:
import cytoolz as tlz
from cytoolz import curried as tlzc
from text2math import raw2text as r2t
from text2math import text2tokens as t2t
from gensim import corpora
# In[62]:
head = lambda l: list(tlzc.take(4, l))
# ---
# # Loading Data
# In[4]:
def read_student_title():
with open("../student_title.csv") as f:
return tlz.drop(1, [line for line in csv.reader(f)])
# ---
