def __str__(self):
if self.has_complex_coeffs():
return ' + '.join(
str(c) + '×X^' + str(i) for i, c in enumerate(self._coeffs))
terms = []
signs = []
for i, c in enumerate(self):
term = ''
if c == 0:
continue
if c > 0:
signs += ['+']
elif c < 0:
signs += ['-']
else:
signs += ['']
if c not in (1, -1) or i == 0:
term += str(abs(c))
if i >= 1:
term += 'x'
if i >= 2:
term += '^' + str(i)
if term != '':
terms += [term]
terms, signs = list(reversed(terms)), list(reversed(signs))
first_sign, signs = signs[0], (' ' + s + ' ' for s in signs[1:])
if first_sign == '+':
first_sign = ''
return first_sign + ''.join(toolz.interleave([terms, signs]))
def retrieve_newsgroup_articles(max_documents=-1):
# List of documents
documents = []
news_groups_all = fetch_20newsgroups(subset='all')
train_news_groups = fetch_20newsgroups(subset='train')
train_news_groups_zipped = zip(train_news_groups.target,
train_news_groups.data,
train_news_groups.filenames)
test_news_groups = fetch_20newsgroups(subset='test')
test_news_groups_zipped = zip(test_news_groups.target,
test_news_groups.data,
test_news_groups.filenames)
news_groups = interleave(
[train_news_groups_zipped, test_news_groups_zipped])
for news_group_id, article, file_name in news_groups:
if max_documents > -1 and len(documents) >= max_documents:
return documents
words = tokenize(article)
document = {
'words': words,
'categories': [news_groups_all.target_names[news_group_id]],
'is_training_example': True if 'train/' in file_name else False,
'is_test_example': True if 'test/' in file_name else False,
'words_filtered': do_filter_words(words),
'file_id': file_name
}
if len(document['words_filtered']) < 30:
continue
documents.append(document)
return documents
def __init__(self, activations, dims, bn_init=None, **kwargs):
if bn_init is None:
bn_init = IsotropicGaussian(0.1)
self.bn_init = bn_init
self.activations = activations
self.batch_normnalizations = [
BatchNormalization(name='bn_{}'.format(i),
beta_init=bn_init,
gamma_init=bn_init)
for i in range(len(activations))
]
self.linear_transformations = [
Linear(name='linear_{}'.format(i)) for i in range(len(activations))
]
# Interleave the transformations and activations
application_methods = []
for entity in interleave([
self.linear_transformations, self.batch_normnalizations,
activations
]):
if entity is None:
continue
if isinstance(entity, Brick):
application_methods.append(entity.apply)
else:
application_methods.append(entity)
if not dims:
dims = [None] * (len(activations) + 1)
self.dims = dims
super(BatchNormalizedMLP, self).__init__(application_methods, **kwargs)
def __init__(self, input_dim0, input_dim1, other_dims, activations,
batch_size, **kwargs):
self.activations = activations
self.input_dim0 = input_dim0
self.input_dim1 = input_dim1
self.other_dims = other_dims
self.batch_size = batch_size
self.linear_transformations = []
self.linear_transformations.append(
TensorLinear(input_dim0=self.input_dim0,
input_dim1=self.input_dim1,
output_dim=self.other_dims[0],
batch_size=batch_size))
self.linear_transformations.extend([
Linear(name='linear_{}'.format(i),
input_dim=other_dims[i],
output_dim=other_dims[i + 1])
for i in range(len(other_dims) - 1)
])
self.linear_transformations.append(
TensorLinear_inverse(input_dim=self.other_dims[-1],
output_dim0=self.input_dim0,
output_dim1=self.input_dim1,
batch_size=batch_size))
application_methods = []
for entity in interleave([self.linear_transformations, activations]):
if entity is None:
continue
if isinstance(entity, Brick):
application_methods.append(entity.apply)
else:
application_methods.append(entity)
super(tabula_NADE, self).__init__(application_methods, **kwargs)
def __init__(self, input_dim0, input_dim1, other_dims, activations, batch_size,
**kwargs):
self.activations = activations
self.input_dim0 = input_dim0
self.input_dim1 = input_dim1
self.other_dims = other_dims
self.batch_size = batch_size
self.linear_transformations = []
self.linear_transformations.append(TensorLinear(input_dim0=self.input_dim0,
input_dim1=self.input_dim1,
output_dim=self.other_dims[0],
batch_size=batch_size)
)
self.linear_transformations.extend([Linear(name='linear_{}'.format(i),
input_dim=other_dims[i],
output_dim=other_dims[i + 1])
for i in range(len(other_dims) - 1)])
self.linear_transformations.append(TensorLinear_inverse(input_dim=self.other_dims[-1],
output_dim0=self.input_dim0,
output_dim1=self.input_dim1,
batch_size=batch_size))
application_methods = []
for entity in interleave([self.linear_transformations, activations]):
if entity is None:
continue
if isinstance(entity, Brick):
application_methods.append(entity.apply)
else:
application_methods.append(entity)
super(tabula_NADE, self).__init__(application_methods, **kwargs)
def __getattr__(name):
# Highly obsfuscated code. This is the puzzle. If you solve it you can use
# forbidden powers in production code. You can, but you shouldn't. Really,
# don't do it!
import toolz
import inspect
import os
if name != "forbidden_powers":
raise AttributeError(name)
evil = os.environ.get("EVIL", "").lower() == "true"
try:
fn = globals()["_forbidden_powers"]
except KeyError:
pass
else:
if evil:
data = inspect.getsource(fn)
print("Forbidden power spells")
print(f"SPELL_1 = {data[::2]!r}")
print(f"SPELL_2 = {data[1::2]!r}")
print("\n\n")
return fn
code = "".join(toolz.interleave([SPELL_1, SPELL_2]))
if evil:
print("Forbidden_powers code\n\n")
print(code)
exec(code, globals())
return globals()["_forbidden_powers"]
def getCubeIndex(self, ch, listoftimestamps, listofidxs, resolution, neariso=False):
"""Retrieve the indexes of inserted cubes"""
index_store = self.getIndexStore()
index_store_temp = index_store+'&temp'
index_list_size = len(listofidxs)*len(listoftimestamps)
try:
index_list = list(interleave([[1]*index_list_size, self.getIndexList(ch, listoftimestamps, listofidxs, resolution, neariso)]))
self.client.zadd(index_store_temp, *index_list)
# if listoftimestamps:
# # TODO KL Test this
# index_list = list(interleave([[1]*len(listofidxs), self.getIndexList(ch, resolution, listofidxs)]))
# self.client.zadd(index_store_temp, *index_list)
# else:
# index_list = list(interleave([[1]*len(listofidxs), self.getIndexList(ch, resolution, listofidxs)]))
# self.client.zadd(index_store_temp, *index_list)
# self.client.zinterstore(index_store_temp, [index_store_temp, index_store] )
self.client.zunionstore(index_store_temp, {index_store_temp : 1, index_store : 0}, 'MIN')
ids_to_fetch = self.client.zrevrangebyscore(index_store_temp, '+inf', 1)
self.client.delete(index_store_temp)
except Exception, e:
logger.error("Error retrieving cube indexes into the database. {}".format(e))
raise SpatialDBError("Error retrieving cube indexes into the database. {}".format(e))
def putCubeIndex(self,
ch,
listoftimestamps,
listofidxs,
resolution,
neariso=False):
"""Add the listofidxs to the store"""
try:
cachedtime_list = [time.time()
] * len(listofidxs) * len(listoftimestamps)
index_list = list(
interleave([
cachedtime_list,
self.getIndexList(ch, listoftimestamps, listofidxs,
resolution, neariso)
]))
self.client.zadd(self.getIndexStore(), *index_list)
# if listoftimestamps:
# # TODO KL Test this
# cachedtime_list = [time.time()]*len(listofidxs)
# index_list = list(interleave([cachedtime_list, self.getIndexList(ch, resolution, listofidxs)]))
# self.client.zadd(self.getIndexStore(), *index_list)
# else:
# cachedtime_list = [time.time()]*len(listofidxs)
# index_list = list(interleave([cachedtime_list, self.getIndexList(ch, resolution, listofidxs)]))
# self.client.zadd(self.getIndexStore(), *index_list)
except Exception, e:
logger.error(
"Error inserting cube indexes into the database. {}".format(e))
raise SpatialDBError(
"Error inserting cube indexes into the database. {}".format(e))
def retrieve_reuters_documents(max_documents=-1, filter_words=True):
# List of documents
documents = []
training_files = [
file_id for file_id in reuters.fileids() if 'training/' in file_id
]
test_files = [
file_id for file_id in reuters.fileids() if 'test/' in file_id
]
for file_id in interleave([training_files, test_files]):
if max_documents > -1 and len(documents) >= max_documents:
return documents
words = list(reuters.words(fileids=file_id))
words_filtered = do_filter_words(words)
document = {
'words': words,
'title': reuters.raw(fileids=file_id).split("\n")[0],
'categories': reuters.categories(fileids=file_id),
'is_training_example': True if 'training/' in file_id else False,
'is_test_example': True if 'test/' in file_id else False,
'words_filtered': words_filtered if filter_words else words,
'file_id': file_id
}
if len(words_filtered) < 30:
continue
documents.append(document)
return documents
def apply(self, input_):
out = input_
for brick in interleave([self.linear_transformations, self.batch_norms, self.activations]):
if brick is None:
continue
if isinstance(brick, Brick):
out = brick.apply(out)
return out
def apply(self, input_):
out = input_
for brick in interleave(
[self.linear_transformations, self.batch_norms, self.activations]):
if brick is None:
continue
if isinstance(brick, Brick):
out = brick.apply(out)
return out
def compute_stats(df):
means = df.groupby(["model"],
sort=False).mean().add_prefix("Average ").reset_index()
std = df.groupby(["model"],
sort=False).std().add_suffix(" (STD)").reset_index()
model = means["model"]
means = means.drop(columns=["model"])
std = std.drop(columns=["model"])
stats = pd.concat([means, std], axis=1)[list(interleave([means, std]))]
stats.insert(0, 'model', model)
return stats
def encode(message, rails):
encoded = []
for i in range(rails):
if i == 0 or i == rails - 1:
encoded.append(message[i::2 * rails - 2])
else:
l1 = message[i::2 * rails - 2]
l2 = message[2 * rails - (i + 2)::2 * rails - 2]
encoded.append("".join(toolz.interleave([l1, l2])))
return "".join(encoded)
def __init__(self, activations, dims, **kwargs):
self.activations = activations
self.linear_transformations = [Linear(name='linear_{}'.format(i))
for i in range(len(activations))]
# Interleave the transformations and activations
application_methods = [brick.apply for brick in interleave(
[self.linear_transformations, activations]) if brick is not None]
if not dims:
dims = [None] * (len(activations) + 1)
self.dims = dims
super(MLP, self).__init__(application_methods, **kwargs)
def insert(arr, obj, values, axis):
# axis is a required argument here to avoid needing to deal with the numpy
# default case (which reshapes the array to make it flat)
if not -arr.ndim <= axis < arr.ndim:
raise IndexError('axis %r is out of bounds for an array of dimension '
'%s' % (axis, arr.ndim))
if axis < 0:
axis += arr.ndim
if isinstance(obj, slice):
obj = np.arange(*obj.indices(arr.shape[axis]))
obj = np.asarray(obj)
scalar_obj = obj.ndim == 0
if scalar_obj:
obj = np.atleast_1d(obj)
obj = np.where(obj < 0, obj + arr.shape[axis], obj)
if (np.diff(obj) < 0).any():
raise NotImplementedError(
'da.insert only implemented for monotonic ``obj`` argument')
split_arr = split_at_breaks(arr, np.unique(obj), axis)
if getattr(values, 'ndim', 0) == 0:
# we need to turn values into a dask array
name = 'values-' + tokenize(values)
dtype = getattr(values, 'dtype', type(values))
values = Array({(name,): values}, name, chunks=(), dtype=dtype)
values_shape = tuple(len(obj) if axis == n else s
for n, s in enumerate(arr.shape))
values = broadcast_to(values, values_shape)
elif scalar_obj:
values = values[(slice(None),) * axis + (None,)]
values_chunks = tuple(values_bd if axis == n else arr_bd
for n, (arr_bd, values_bd)
in enumerate(zip(arr.chunks,
values.chunks)))
values = values.rechunk(values_chunks)
counts = np.bincount(obj)[:-1]
values_breaks = np.cumsum(counts[counts > 0])
split_values = split_at_breaks(values, values_breaks, axis)
interleaved = list(interleave([split_arr, split_values]))
interleaved = [i for i in interleaved if i.nbytes]
return concatenate(interleaved, axis=axis)
def insert(arr, obj, values, axis):
# axis is a required argument here to avoid needing to deal with the numpy
# default case (which reshapes the array to make it flat)
if not -arr.ndim <= axis < arr.ndim:
raise IndexError('axis %r is out of bounds for an array of dimension '
'%s' % (axis, arr.ndim))
if axis < 0:
axis += arr.ndim
if isinstance(obj, slice):
obj = np.arange(*obj.indices(arr.shape[axis]))
obj = np.asarray(obj)
scalar_obj = obj.ndim == 0
if scalar_obj:
obj = np.atleast_1d(obj)
obj = np.where(obj < 0, obj + arr.shape[axis], obj)
if (np.diff(obj) < 0).any():
raise NotImplementedError(
'da.insert only implemented for monotonic ``obj`` argument')
split_arr = split_at_breaks(arr, np.unique(obj), axis)
if getattr(values, 'ndim', 0) == 0:
# we need to turn values into a dask array
name = 'values-' + tokenize(values)
dtype = getattr(values, 'dtype', type(values))
values = Array({(name,): values}, name, chunks=(), dtype=dtype)
values_shape = tuple(len(obj) if axis == n else s
for n, s in enumerate(arr.shape))
values = broadcast_to(values, values_shape)
elif scalar_obj:
values = values[(slice(None),) * axis + (None,)]
values_chunks = tuple(values_bd if axis == n else arr_bd
for n, (arr_bd, values_bd)
in enumerate(zip(arr.chunks,
values.chunks)))
values = values.rechunk(values_chunks)
counts = np.bincount(obj)[:-1]
values_breaks = np.cumsum(counts[counts > 0])
split_values = split_at_breaks(values, values_breaks, axis)
interleaved = list(interleave([split_arr, split_values]))
interleaved = [i for i in interleaved if i.nbytes]
return concatenate(interleaved, axis=axis)
def compile(self):
self._extract_subqueries()
extracted = self.format_subqueries()
buf = []
if extracted:
buf.append(f'WITH {extracted}')
buf.extend(
toolz.interleave((
map(self.format_relation, self.tables),
self._get_keyword_list(),
)))
return '\n'.join(buf)
def __init__(self, activations, dims, prototype=None, **kwargs):
self.activations = activations
self.prototype = Linear() if prototype is None else prototype
self.linear_transformations = []
for i in range(len(activations)):
linear = copy.deepcopy(self.prototype)
name = self.prototype.__class__.__name__.lower()
linear.name = '{}_{}'.format(name, i)
self.linear_transformations.append(linear)
if not dims:
dims = [None] * (len(activations) + 1)
self.dims = dims
# Interleave the transformations and activations
applications = [a for a in interleave([self.linear_transformations,
activations]) if a is not None]
super(MLP, self).__init__(applications, **kwargs)
def solve_matrix(lhs_mat, rhs_mat):
df = toolz.merge(lhs_mat, rhs_mat)
df = pd.DataFrame(df)
df[list(rhs_mat.keys())] *= -1
# df.replace("nan", 0)
df = df.fillna(value=0)
matrix = sp.Matrix(df.values.astype(int))
consts = matrix.nullspace()
headings = list(df.columns)
consts = consts[0].values()
consts = [float(x) for x in consts]
consts = [Fraction(x).limit_denominator() for x in consts]
solns = list(toolz.interleave([headings, consts]))
solns = list(toolz.partition(2, solns))
return solns
def solve_matrix(lhs_mat, rhs_mat):
df = toolz.merge(lhs_mat, rhs_mat)
df = pd.DataFrame(df)
df[list(rhs_mat.keys())] *= -1
# df.replace("nan", 0)
df = df.fillna(value=0)
matrix = sp.Matrix(df.values.astype(int))
consts = matrix.nullspace()
headings = list(df.columns)
consts = consts[0].values()
consts = [float(x) for x in consts]
consts = [Fraction(x).limit_denominator() for x in consts]
solns = list(toolz.interleave([headings, consts]))
solns = list(toolz.partition(2, solns))
return solns
def adjust_links(s: str):
matches = list(link_pattern.finditer(s))
if len(matches) == 0:
return s
abs_save_paths = [
Path(load_config().save_path, 'assets', m.group(1)) for m in matches
]
first_segment = s[:matches[0].start(1)]
segments = [
s[m1.end(1):m2.start(1)] for m1, m2 in t.sliding_window(2, matches)
]
last_segment = s[matches[-1].end(1):]
return "".join(
t.interleave([[first_segment] + segments,
map(str, abs_save_paths)])) + last_segment
def __init__(self, activations, dims, **kwargs):
self.activations = activations
self.linear_transformations = [Linear(name='linear_{}'.format(i))
for i in range(len(activations))]
# Interleave the transformations and activations
application_methods = []
for entity in interleave([self.linear_transformations, activations]):
if entity is None:
continue
if isinstance(entity, Brick):
application_methods.append(entity.apply)
else:
application_methods.append(entity)
if not dims:
dims = [None] * (len(activations) + 1)
self.dims = dims
super(MLP, self).__init__(application_methods, **kwargs)
def __init__(self, activations, dims, prototype=None, **kwargs):
self.activations = activations
self.prototype = Linear() if prototype is None else prototype
self.linear_transformations = []
for i in range(len(activations)):
linear = copy.deepcopy(self.prototype)
name = self.prototype.__class__.__name__.lower()
linear.name = '{}_{}'.format(name, i)
self.linear_transformations.append(linear)
if not dims:
dims = [None] * (len(activations) + 1)
self.dims = dims
# Interleave the transformations and activations
applications = [
a for a in interleave([self.linear_transformations, activations])
if a is not None
]
super(MLP, self).__init__(applications, **kwargs)
def __init__(self, activations, dims, **kwargs):
self.activations = activations
self.linear_transformations = [Linear(name='linear_{}'.format(i))
for i in range(len(activations))]
# Interleave the transformations and activations
application_methods = []
for entity in interleave([self.linear_transformations, activations]):
if entity is None:
continue
if isinstance(entity, Brick):
application_methods.append(entity.apply)
else:
application_methods.append(entity)
if not dims:
dims = [None] * (len(activations) + 1)
self.dims = dims
super(MLP, self).__init__(application_methods, **kwargs)
def putCubeIndex(self, ch, listoftimestamps, listofidxs, resolution, neariso=False):
"""Add the listofidxs to the store"""
try:
cachedtime_list = [time.time()]*len(listofidxs)*len(listoftimestamps)
index_list = list(interleave([cachedtime_list, self.getIndexList(ch, listoftimestamps, listofidxs, resolution, neariso)]))
self.client.zadd(self.getIndexStore(), *index_list)
# if listoftimestamps:
# # TODO KL Test this
# cachedtime_list = [time.time()]*len(listofidxs)
# index_list = list(interleave([cachedtime_list, self.getIndexList(ch, resolution, listofidxs)]))
# self.client.zadd(self.getIndexStore(), *index_list)
# else:
# cachedtime_list = [time.time()]*len(listofidxs)
# index_list = list(interleave([cachedtime_list, self.getIndexList(ch, resolution, listofidxs)]))
# self.client.zadd(self.getIndexStore(), *index_list)
except Exception, e:
logger.error("Error inserting cube indexes into the database. {}".format(e))
raise SpatialDBError("Error inserting cube indexes into the database. {}".format(e))
def getCubeIndex(self,
ch,
listoftimestamps,
listofidxs,
resolution,
neariso=False):
"""Retrieve the indexes of inserted cubes"""
index_store = self.getIndexStore()
index_store_temp = index_store + '&temp'
index_list_size = len(listofidxs) * len(listoftimestamps)
try:
index_list = list(
interleave([[1] * index_list_size,
self.getIndexList(ch, listoftimestamps, listofidxs,
resolution, neariso)]))
self.client.zadd(index_store_temp, *index_list)
# if listoftimestamps:
# # TODO KL Test this
# index_list = list(interleave([[1]*len(listofidxs), self.getIndexList(ch, resolution, listofidxs)]))
# self.client.zadd(index_store_temp, *index_list)
# else:
# index_list = list(interleave([[1]*len(listofidxs), self.getIndexList(ch, resolution, listofidxs)]))
# self.client.zadd(index_store_temp, *index_list)
# self.client.zinterstore(index_store_temp, [index_store_temp, index_store] )
self.client.zunionstore(index_store_temp, {
index_store_temp: 1,
index_store: 0
}, 'MIN')
ids_to_fetch = self.client.zrevrangebyscore(
index_store_temp, '+inf', 1)
self.client.delete(index_store_temp)
except Exception, e:
logger.error(
"Error retrieving cube indexes into the database. {}".format(
e))
raise SpatialDBError(
"Error retrieving cube indexes into the database. {}".format(
e))
def __init__(self, activations, dims, prototype=None, **kwargs):
self.activations = activations
self.prototype = Linear() if prototype is None else prototype
self.linear_transformations = []
for i in range(len(activations)):
linear = copy.deepcopy(self.prototype)
name = self.prototype.__class__.__name__.lower()
linear.name = '{}_{}'.format(name, i)
self.linear_transformations.append(linear)
# Interleave the transformations and activations
application_methods = []
for entity in interleave([self.linear_transformations, activations]):
if entity is None:
continue
if isinstance(entity, Brick):
application_methods.append(entity.apply)
else:
application_methods.append(entity)
if not dims:
dims = [None] * (len(activations) + 1)
self.dims = dims
super(MLP, self).__init__(application_methods, **kwargs)
def __init__(self, activations, dims, prototype=None, **kwargs):
self.activations = activations
self.prototype = Linear() if prototype is None else prototype
self.linear_transformations = []
for i in range(len(activations)):
linear = copy.deepcopy(self.prototype)
name = self.prototype.__class__.__name__.lower()
linear.name = '{}_{}'.format(name, i)
self.linear_transformations.append(linear)
# Interleave the transformations and activations
application_methods = []
for entity in interleave([self.linear_transformations, activations]):
if entity is None:
continue
if isinstance(entity, Brick):
application_methods.append(entity.apply)
else:
application_methods.append(entity)
if not dims:
dims = [None] * (len(activations) + 1)
self.dims = dims
super(MLP, self).__init__(application_methods, **kwargs)
def retrieve_imdb_movie_reviews(max_documents=-1):
# List of documents
documents = []
file_names = interleave([
list_files('/data-sets/aclImdb/test/neg'),
list_files('/data-sets/aclImdb/test/pos'),
list_files('/data-sets/aclImdb/train/neg'),
list_files('/data-sets/aclImdb/train/pos'),
list_files('/data-sets/aclImdb/train/unsup')
])
for file_name in file_names:
if max_documents > -1 and len(documents) >= max_documents:
return documents
text_file = open(file_name, "r", encoding='utf8')
lines = text_file.readlines()
text_file.close()
words = tokenize("\n".join(lines))
document = {
'words':
words,
'categories': ['neg'] if '/neg/' in file_name else
['pos'] if '/pos/' in file_name else [],
'is_training_example':
True if '/train/' in file_name else False,
'is_test_example':
True if '/test/' in file_name else False,
'words_filtered':
do_filter_words(words),
'file_id':
file_name
}
if len(document['words_filtered']) < 30:
continue
documents.append(document)
return documents
question.possible_answers
)
def add_last_answer(question: Question, text: str) -> Question:
return Question(
question.id,
question.text,
question.possible_answers + [Answer(question.id, text)]
)
def is_appropriate(question: Question) -> bool:
return question.text.startswith('bb')
business_rules_questions = [
Question(10, 'bizness?', [Answer(10, 'biz'), Answer(10, 'ness')]),
Question(20, 'nezbis?', [Answer(20, 'nez'), Answer(20, 'bis')])
]
questions_to_ask = filter(
is_appropriate,
toolz.interleave((
toolz.take(1, business_rules_questions),
map(toolz.compose(reformulate, partial(add_last_answer, text='non')),
algorithmic_generated_questions)
)))
print(list(questions_to_ask))
def doubleEveryOther(ints: PVector[int]) -> PVector[int]:
return pvector(
interleave([
ints[::-1][0::2],
pvector(map(lambda x: x * 2, ints[::-1][1::2]))
]))
def ldisj_seq_goal(S):
nonlocal goals
goals, _goals = tee(goals)
yield from interleave(g(S) for g in _goals)
#### Two dataframes
df_delays = df[[ 'start_delay_ref', 'ones', 'duration_delay']]
df_delays.columns=['start', 'index' , 'duration']
df_delays['condition']='delay'
df_inter_delays = df[[ 'starts_inter_delay', 'zeros', 'durations_inter']]
df_inter_delays.columns=['start', 'index' , 'duration']
df_inter_delays['condition']='baseline';
##### Interleave
#concat_df = pd.concat([df_inter_delays, df_delays ]).sort_index().reset_index(drop=True)
#concat_df['intercept'] = 1
#
concat_df = pd.DataFrame(interleave([df_inter_delays.values, df_delays.values]))
concat_df.columns=['start', 'index_c' , 'duration', 'condition']
concat_df['intercept'] = 1
concat_df['index_c'] = concat_df['index_c'].replace([0,1], [1,2]);
#Matrix = pd.DataFrame(concat_df.values)
#Matrix= Matrix.round(2)
#Matrix.to_csv('dm_fs.par', header=False, index=False, sep='\t', mode='a')
#
#
#
#
#
#concat_df.values.to_csv('dm_fs.par', header=False, index=False, sep='\t', mode='a')
#
def bind(z, g):
"""Apply a goal to a state stream and then combine the resulting state streams."""
# We could also use `chain`, but `interleave` preserves the old behavior.
# return chain.from_iterable(map(g, z))
return interleave(map(g, z))
def merge_dataframes_outer(self, dfs_outer, dfs_outer_outer,
year_output_folder, name):
result = pd.concat(dfs_outer, axis=1)[list(interleave(dfs_outer))]
result.to_csv(year_output_folder + name, index=True)
dfs_outer_outer.append(result)
def make_yaml(path, title=None):
yaml = f"""---
title: {title}
image_dir: {path}
"""
items = []
directory = f'static/img/{path}'
for img_fname in os.listdir(directory):
i = Image.open(f'{directory}/{img_fname}')
w, h = i.size
item_string = f"""
- name: {img_fname}
width: {w}
height: {h}
caption:
description:
"""
# displaying in a column on the page, the apparent height is relative to width
items.append({'rel_height': h / w, 'yaml': item_string})
#
# Need to balance out the columns of images for the gallery
# The columns are templated according to the modulus of the index of the item in a list
# so, the kth column will include every image whose index % k == 0
#
#
# In order for a good even bucketing, start with a *reverse* sorted array
#
items_reverse_sorted = sorted(items,
key=lambda x: x['rel_height'],
reverse=True)
# utility function
def sum_group(xs):
return sum([x['rel_height'] for x in xs])
groups = [list() for _ in range(COLUMNS)]
for item in items_reverse_sorted:
min_group = groups[0]
min_sum = sum_group(min_group)
for g in groups:
current_sum = sum_group(g)
if current_sum < min_sum:
min_group = g
min_group.append(item)
#
# shuffle for looks, so that we don't get a blocky grid
# we want to leave the "bottom" of the columns where they're at
# bc they are the smallest images, and shuffling a large image into the end
# can potentially cause lopsided columns if the total number of images are not evenly divisible
min_length = min([len(g) for g in groups])
for g in groups:
subset = g[:min_length]
random.shuffle(subset)
g[:min_length] = subset
random.shuffle(groups)
# now stagger them, so they will be doled out into columns
items = toolz.interleave(groups)
# finally, cook up the yaml
yaml_strings = map(lambda x: x['yaml'], items)
yaml = f"""{yaml}
images:
{"".join(yaml_strings)}
---
"""
return yaml
def pointVectorFill(points, size):
numc = len(points)
if numc == size:
return points
# filling if less than size
if numc < size:
nmiss = size - numc
#inc_rate = 2
while nmiss > 0:
pos = 0
newpoints = []
while (pos < (len(points) - 1)) and (nmiss > 0):
newpoints.append(((points[pos][0] + points[pos + 1][0]) / 2,
(points[pos][1] + points[pos + 1][1]) / 2))
nmiss -= 1
pos += 1
#inc_rate += 1
points = list(interleave([points, newpoints]))
#plt.plot(x,y,'ro')
#for p in points:
# plt.plot(p[0], p[1], 'bx')
#plt.show()
return points
#...........................................................................
# selecting significant points..............................................
else:
x = []
y = []
for p in points:
x.append(p[0])
y.append(p[1])
plt.plot(x, y, 'ro')
#plt.show()
#...................first derivative....................................
slopes = [0]
ch_points = [(0, 0)]
for i in range(1, len(x)):
xdif = x[i] - x[i - 1]
ydif = y[i] - y[i - 1]
slope = ydif / (xdif + 1e-10)
slopes.append(slope)
ch_points.append((i, slope))
#plt.plot(range(len(x)),slopes)
#plt.show()
#.......................second derivative...............................
ddx = [0]
for i in range(1, len(slopes)):
ddx.append(abs(slopes[i - 1] - slopes[i]))
#plt.plot(range(len(ddx)), ddx)
c = sorted(range(len(ddx)), key=lambda k: ddx[k])
c.reverse()
c = [0] + c[:(size - 1)]
if not (numc - 1) in c:
c[size - 1] = (numc - 1)
c = sorted(c)
new_points = []
for i in c:
new_points.append(points[i])
#plt.plot(x,y,'ro')
#for i in range(size):
# plt.plot(x[c[i]], y[c[i]], 'bx')
#plt.show()
return new_points
def _generate_stencil(self):
init = InitialSimplexPoint(self)
point = SimplexPoint(self.point, init, 0)
seen = {point}
seen_add = seen.add
first_seen = {point.point_key}
first_seen_add = first_seen.add
stencil_points_append = self._stencil_points.append
for p in point.get_points():
stencil_points_append(p)
yield p
first_seen_add(p.point_key)
seen_add(p)
self_reflect = []
mirror_reflect = []
reflect = []
self_contract = [point]
contract = []
while True:
next_self_reflect = []
next_mirror_reflect = []
next_reflect = []
next_self_contract = []
next_contract = []
for p in concatv(
interleave(x.get_reflections() for x in self_reflect),
interleave(x.get_reflections() for x in mirror_reflect),
interleave(x.get_reflections() for x in reflect),
interleave(x.get_reflections() for x in self_contract),
interleave(x.get_reflections() for x in contract),
):
if p.point_key not in first_seen:
stencil_points_append(p)
yield p
first_seen_add(p.point_key)
seen_add(p)
next_reflect.append(p)
elif p not in seen:
seen_add(p)
if p.index == 0:
next_self_reflect.append(p)
elif p.index == 1:
next_mirror_reflect.append(p)
else:
next_reflect.append(p)
for p in concatv(
interleave(x.get_contractions() for x in self_reflect),
interleave(x.get_contractions() for x in mirror_reflect),
interleave(x.get_contractions() for x in reflect),
interleave(x.get_contractions() for x in self_contract),
interleave(x.get_contractions() for x in contract),
):
if p.point_key not in first_seen:
stencil_points_append(p)
yield p
first_seen_add(p.point_key)
seen_add(p)
next_contract.append(p)
elif p not in seen:
seen_add(p)
if p.index == 0:
next_self_contract.append(p)
else:
next_contract.append(p)
self_reflect = next_self_reflect
mirror_reflect = next_mirror_reflect
reflect = next_reflect
self_contract = next_self_contract
contract = next_contract
# Filename: Of_CSVtoCSV.py
# Author: Michael L. Smith
#
# Takes in a directory of CSVs generated from OpenFace 2.0 FeatureExtractor's function and
# converts them to the necessary format needed for my animations.
import os
import numpy as np
import pandas as pd
from toolz import interleave
directory = os.fsencode('/Users/MichaelSmith/Desktop/CSVs/OF_CSVs')
for file in os.listdir(directory):
filename = os.fsdecode(file)
savename = filename[:10]
print(filename)
if '.csv' in filename:
#print(filename)
csv_pd = pd.read_csv(os.path.join(os.fsdecode(directory), filename))
X = csv_pd.loc[:, ' x_0':' x_67']
Y = csv_pd.loc[:, ' y_0':' y_67']
XY = pd.DataFrame(interleave([X.values, Y.values])).T
XY.to_csv('/Users/MichaelSmith/Desktop/CSVs/ANIM_CSVs/' + savename + '.csv', index=False, header=False)
