def write_tables(fname, models, year):
"""
Write all tables injected into `models` to a pandas.HDFStore file.
If year is not None it will be used to prefix the table names so that
multiple years can go in the same file.
Parameters
----------
fname : str
File name for HDFStore. Will be opened in append mode and closed
at the end of this function.
models : list of str
Models from which to gather injected tables for saving.
year : int or None
If an integer, used as a prefix along with table names for
labeling DataFrames in the HDFStore.
"""
models = (get_model(m) for m in toolz.unique(models))
table_names = toolz.unique(toolz.concat(m._tables_used() for m in models))
tables = (get_table(t) for t in table_names)
key_template = '{}/{{}}'.format(year) if year is not None else '{}'
with pd.get_store(fname, mode='a') as store:
for t in tables:
store[key_template.format(t.name)] = t.to_frame()
def render_tabular(api, options=None):
"""Entry point for the tabular reporter interface."""
# determine separator
separator = options.get('report.separator', '\t')
human = options.get('report.human')
panel = options.get('report.panel')
samples = options.get('report.samples')
group = options.get('report.group')
# read gene panel file if it has been set
if panel:
superblock_ids = [line.rstrip() for line in panel]
else:
superblock_ids = None
# get sample ID, group and cutoff from metadata
sample_query = limit_query(api.samples(), group=group, samples=samples)
metadata = ((sample.id, sample.group_id, sample.cutoff)
for sample in sample_query)
# get the data
base_query = limit_query(api.average_metrics(superblock_ids=superblock_ids),
group=group,
samples=samples)
queries = [metadata,
base_query,
api.diagnostic_yield(superblock_ids=superblock_ids,
group_id=group, sample_ids=samples),
api.sex_checker(group_id=group, sample_ids=samples)]
# group multiple queries by sample ID (first column)
key_metrics = groupby(get(0), concat(queries))
# get the column names dynamically from the query
headers = concatv(['sample_id', 'group_id', 'cutoff'],
(column['name'] for column
in base_query.column_descriptions),
['diagnostic yield', 'gender'])
unique_headers = unique(headers)
# iterate over all values, concat different query results, and keep
# only the unique values (excluding second sample_id)
data = (unique(concat(values)) for values in itervalues(key_metrics))
if human:
# export key_metrics in a more human friendly format
return tabulate(data, unique_headers)
# yield headers
return '\n'.join(cons('#' + separator.join(unique_headers),
stringify_list(data, separator=separator)))
def compute_up(expr, args, **kwargs):
from_objs = list(unique(concat(map(get_all_froms, args))))
if len(from_objs) > 1:
# TODO: how do you do this in sql? please send help
raise ValueError('only columns from the same table can be merged')
cols = list(unique(concat(map(get_unsafe_inner_columns, args, expr.args))))
sel = sa.select(cols, from_obj=from_objs[0])
where = unify_wheres(args)
if where is not None:
sel = sel.where(where)
return sel
def compile_components(summary, schema):
"""Given a ``Summary`` object and a table schema, returning 5 sub-functions.
Parameters
----------
summary : Summary
The expression describing the aggregations to be computed.
Returns
-------
A tuple of the following functions:
``create(shape)``
Takes the aggregate shape, and returns a tuple of initialized numpy
arrays.
``info(df)``
Takes a dataframe, and returns preprocessed 1D numpy arrays of the
needed columns.
``append(i, x, y, *aggs_and_cols)``
Appends the ``i``th row of the table to the ``(x, y)`` bin, given the
base arrays and columns in ``aggs_and_cols``. This does the bulk of the
work.
``combine(base_tuples)``
Combine a list of base tuples into a single base tuple. This forms the
reducing step in a reduction tree.
``finalize(aggs)``
Given a tuple of base numpy arrays, returns the finalized
``dynd`` array.
"""
paths, reds = zip(*preorder_traversal(summary))
# List of base reductions (actually computed)
bases = list(unique(concat(r._bases for r in reds)))
dshapes = [b.out_dshape(schema) for b in bases]
# List of tuples of (append, base, input columns, temps)
calls = [_get_call_tuples(b, d) for (b, d) in zip(bases, dshapes)]
# List of unique column names needed
cols = list(unique(concat(pluck(2, calls))))
# List of temps needed
temps = list(pluck(3, calls))
create = make_create(bases, dshapes)
info = make_info(cols)
append = make_append(bases, cols, calls)
combine = make_combine(bases, dshapes, temps)
finalize = make_finalize(bases, summary, schema)
return create, info, append, combine, finalize
def columns_in_filters(filters):
"""
Returns a list of the columns used in a set of query filters.
Parameters
----------
filters : list of str or str
List of the filters as passed passed to ``apply_filter_query``.
Returns
-------
columns : list of str
List of all the strings mentioned in the filters.
"""
if not filters:
return []
if not isinstance(filters, str):
filters = ' '.join(filters)
columns = []
reserved = {'and', 'or', 'in', 'not'}
for toknum, tokval, _, _, _ in generate_tokens(StringIO(filters).readline):
if toknum == NAME and tokval not in reserved:
columns.append(tokval)
return list(toolz.unique(columns))
def schema(self):
group = self.grouper.schema[0].parameters[0]
reduction_name = type(self.apply).__name__
apply = self.apply.dshape[0].parameters[0]
params = unique(group + apply, key=lambda x: x[0])
return dshape(Record(list(params)))
def test_multi_column_join():
metadata = sa.MetaData()
lhs = sa.Table('aaa', metadata,
sa.Column('x', sa.Integer),
sa.Column('y', sa.Integer),
sa.Column('z', sa.Integer))
rhs = sa.Table('bbb', metadata,
sa.Column('w', sa.Integer),
sa.Column('x', sa.Integer),
sa.Column('y', sa.Integer))
L = symbol('L', 'var * {x: int, y: int, z: int}')
R = symbol('R', 'var * {w: int, x: int, y: int}')
joined = join(L, R, ['x', 'y'])
expected = lhs.join(rhs, (lhs.c.x == rhs.c.x)
& (lhs.c.y == rhs.c.y))
expected = select(list(unique(expected.columns, key=lambda c:
c.name))).select_from(expected)
result = compute(joined, {L: lhs, R: rhs})
assert str(result) == str(expected)
assert str(select(result)) == str(select(expected))
# Schemas match
print(result.c.keys())
print(joined.fields)
assert list(result.c.keys()) == list(joined.fields)
def compute_up(expr, data, **kwargs):
if not valid_grouper(expr):
raise TypeError("Grouper must have a non-nested record or one "
"dimensional collection datashape, "
"got %s of type %r with dshape %s" %
(expr.grouper, type(expr.grouper).__name__, expr.dshape))
s = alias_it(data)
if valid_reducer(expr.apply):
reduction = compute(expr.apply, s, post_compute=False)
else:
raise TypeError('apply must be a Summary expression')
grouper = get_inner_columns(compute(expr.grouper, s, post_compute=False))
reduction_columns = pipe(reduction.inner_columns,
map(get_inner_columns),
concat)
columns = list(unique(chain(grouper, reduction_columns)))
if (not isinstance(s, sa.sql.selectable.Alias) or
(hasattr(s, 'froms') and isinstance(s.froms[0],
sa.sql.selectable.Join))):
assert len(s.froms) == 1, 'only a single FROM clause supported for now'
from_obj, = s.froms
else:
from_obj = None
return reconstruct_select(columns,
getattr(s, 'element', s),
from_obj=from_obj,
group_by=grouper)
def find_names(node):
"""Return the unique :class:`ast.Name` instances in an AST.
Parameters
----------
node : ast.AST
Returns
-------
unique_names : List[ast.Name]
Examples
--------
>>> import ast
>>> node = ast.parse('a + b')
>>> names = find_names(node)
>>> names # doctest: +ELLIPSIS
[<_ast.Name object at 0x...>, <_ast.Name object at 0x...>]
>>> names[0].id
'a'
>>> names[1].id
'b'
"""
return list(
toolz.unique(
filter(None, NameFinder().find(node)),
key=lambda node: (node.id, type(node.ctx)),
)
)
def compute_up(expr, data, scope=None, **kwargs):
data = lower_column(data)
grouper = compute(
expr.grouper,
scope,
post_compute=False,
return_type='native',
**kwargs
)
app = expr.apply
reductions = [
compute(
val,
data,
post_compute=None,
return_type='native',
).label(name)
for val, name in zip(app.values, app.fields)
]
froms = list(unique(chain(get_all_froms(grouper),
concat(map(get_all_froms, reductions)))))
inner_cols = list(getattr(grouper, 'inner_columns', [grouper]))
grouper_cols = inner_cols[:]
inner_cols.extend(concat(
getattr(getattr(r, 'element', None), 'inner_columns', [r])
for r in reductions
))
wheres = unify_wheres([grouper] + reductions)
sel = unify_froms(sa.select(inner_cols, whereclause=wheres), froms)
return sel.group_by(*grouper_cols)
def _get_variables(self):
"""Collect variables, updates and auxiliary variables.
In addition collects all :class:`.Scan` ops and recurses in the
respective inner Theano graphs.
"""
updates = OrderedDict()
shared_outputs = [o for o in self.outputs if is_shared_variable(o)]
usual_outputs = [o for o in self.outputs if not is_shared_variable(o)]
variables = shared_outputs
if usual_outputs:
# Sort apply nodes topologically, get variables and remove
# duplicates
inputs = graph.inputs(self.outputs)
sorted_apply_nodes = graph.io_toposort(inputs, usual_outputs)
self.scans = list(unique([node.op for node in sorted_apply_nodes
if isinstance(node.op, Scan)],
key=lambda op: id(op)))
self._scan_graphs = [ComputationGraph(scan.outputs)
for scan in self.scans]
seen = set()
main_vars = (
[var for var in list(chain(
*[apply_node.inputs for apply_node in sorted_apply_nodes]))
if not (var in seen or seen.add(var))] +
[var for var in self.outputs if var not in seen])
# While preserving order add auxiliary variables, and collect
# updates
seen = set()
# Intermediate variables could be auxiliary
seen_avs = set(main_vars)
variables = []
for var in main_vars:
variables.append(var)
for annotation in getattr(var.tag, 'annotations', []):
if annotation not in seen:
seen.add(annotation)
new_avs = [
av for av in annotation.auxiliary_variables
if not (av in seen_avs or seen_avs.add(av))]
variables.extend(new_avs)
updates = dict_union(updates, annotation.updates)
# If shared_variables is assigned default_update (cloned), we cannot eval()
# it to get the real numpy array value, hence, try to trace back
# original shared variable
def shared_variable_filter(var):
if is_shared_variable(var) and hasattr(var, 'default_update'):
for annotation in var.tag.annotations:
if hasattr(annotation, var.name) and \
is_shared_variable(getattr(annotation, var.name)):
return getattr(annotation, var.name)
return var
self.variables = map(shared_variable_filter, variables)
self.updates = updates
def all_subaccounts(self):
"""
Returns an iterator of all subaccounts that have a recorded transaction
with the account.
"""
return toolz.unique(t.subaccount for t in self.transactions)
def choosers_columns_used(self):
"""
Columns from the choosers table that are used for filtering.
"""
return list(toolz.unique(toolz.concat(
m.choosers_columns_used() for m in self.models.values())))
def diagnostic_yield(self, metric='completeness', cutoff=1,
superblock_ids=None, group_id=None, sample_ids=None):
"""Calculate diagnostic yield."""
# extract column to filter on
metric_column = getattr(BlockData, metric)
# set up the base query for all blocks
total_query = self.total_count(BlockData)
if superblock_ids:
# apply the superblock filter on the Block class level
total_query = total_query.join(BlockData.parent)\
.filter(Block.superblock_id.in_(superblock_ids))
# extend base query to include only passed blocks
pass_query = total_query.filter(metric_column >= cutoff)
# optionally limit query
queries = [limit_query(query, group=group_id, samples=sample_ids)
for query in (total_query, pass_query)]
# group multiple queries by sample ID (first column)
metrics = groupby(get(0), concat(queries))
# iterate over all values, concat different query results, and keep
# only the unique values (excluding second sample_id)
combined = (unique(concat(values)) for values in itervalues(metrics))
# calculate diagnostic yield by simple division
for sample_id, group_id, total, covered in combined:
yield sample_id, group_id, (covered / total)
def gpon_svlan(ip='', username='', password='', slots=None):
ports = product(slots, range(1, 9))
cmds = map(
lambda x: "show service-port interface gpon-olt_1/{0}/{1}".format(x[0], x[1]), ports)
try:
svlan = []
child = telnet(ip, username, password)
for cmd in cmds:
result = []
child.sendline(cmd)
while True:
index = child.expect(
[zte_prompt, zte_pager], timeout=120)
if index == 0:
result.append(child.before)
break
else:
result.append(child.before)
child.send(' ')
continue
r = ''.join(result).split('\r\n')[1:-1]
v = [x.replace('\x08', '').strip().split()[1]
for x in r if 'OK' in x and 'YES' in x]
v1 = [x for x in v if x.isdigit()]
p = re.findall(r'\d/\d{1,2}/\d', cmd)
svlan += product(p, unique(v1))
child.sendline('exit')
child.close()
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ['fail', None, ip]
return ['success', svlan, ip]
def alts_columns_used(self):
"""
Columns from the alternatives table that are used for filtering.
"""
return list(toolz.unique(toolz.concat(
m.alts_columns_used() for m in self.models.values())))
def __repr__(self) -> str:
return '{}({})'.format(
self.name,
', '.join(
'{}={!r}'.format(slot, getattr(self, slot))
for slot in toolz.unique(self.__slots__ + ('nullable',))
),
)
def columns_used(self):
"""
Columns from any table used in the model. May come from either
the choosers or alternatives tables.
"""
return list(toolz.unique(toolz.concat(
m.columns_used() for m in self.models.values())))
def choosers_columns_used(self):
"""
Columns from the choosers table that are used for filtering.
"""
return list(toolz.unique(toolz.concatv(
util.columns_in_filters(self.choosers_predict_filters),
util.columns_in_filters(self.choosers_fit_filters))))
def columns_used(self):
"""
Returns all the columns used across all models in the group
for filtering and in the model expression.
"""
return list(toolz.unique(toolz.concat(
m.columns_used() for m in self.models.values())))
def interaction_columns_used(self):
"""
Columns from the interaction dataset used for filtering and in
the model. These may come originally from either the choosers or
alternatives tables.
"""
return list(toolz.unique(toolz.concat(
m.interaction_columns_used() for m in self.models.values())))
def test_EqualityHashKey_index_key():
d1 = {'firstname': 'Alice', 'age': 21, 'data': {}}
d2 = {'firstname': 'Alice', 'age': 34, 'data': {}}
d3a = {'firstname': 'Bob', 'age': 56, 'data': {}}
d3b = {'firstname': 'Bob', 'age': 56, 'data': {}}
EqualityHashFirstname = curry(EqualityHashKey, 'firstname')
assert list(unique(3*[d1, d2, d3a, d3b],
key=EqualityHashFirstname)) == [d1, d2, d3a]
EqualityHashFirstnameAge = curry(EqualityHashKey, ['firstname', 'age'])
assert list(unique(3*[d1, d2, d3a, d3b],
key=EqualityHashFirstnameAge)) == [d1, d2, d3a]
list1 = [0] * 10
list2 = [0] * 100
list3a = [1] * 10
list3b = [1] * 10
EqualityHash0 = curry(EqualityHashKey, 0)
assert list(unique(3*[list1, list2, list3a, list3b],
key=EqualityHash0)) == [list1, list2, list3a]
def get_users_list(user, repo_name_list):
github = Github(login_or_token=user.github_access_token)
git_user = github.get_user()
user_list = set()
for repo_name in repo_name_list:
repo = get_a_repo(git_user, repo_name)
for g_user in repo.get_collaborators():
user_list.add(g_user)
return sorted([user for user in toolz.unique(user_list, key=lambda x: x.login)], key=lambda x: x.login)
def columns_used(self):
"""
Columns from any table used in the model. May come from either
the choosers or alternatives tables.
"""
return list(toolz.unique(toolz.concatv(
self.choosers_columns_used(),
self.alts_columns_used(),
self.interaction_columns_used())))
def schema(self):
group = self.grouper.schema[0].parameters[0]
if isinstance(self.apply.dshape[0], Record):
apply = self.apply.dshape[0].parameters[0]
else:
apply = (('0', self.apply.dshape),)
params = unique(group + apply, key=lambda x: x[0])
return dshape(Record(list(params)))
def columns_used(self):
"""
Returns all the columns used in this model for filtering
and in the model expression.
"""
return list(toolz.unique(toolz.concatv(
util.columns_in_filters(self.fit_filters),
util.columns_in_filters(self.predict_filters),
util.columns_in_formula(self.model_expression))))
def interaction_columns_used(self):
"""
Columns from the interaction dataset used for filtering and in
the model. These may come originally from either the choosers or
alternatives tables.
"""
return list(toolz.unique(toolz.concatv(
util.columns_in_filters(self.interaction_predict_filters),
util.columns_in_formula(self.model_expression))))
def columns_used(self):
"""
Returns all the columns used across all models in the group
for filtering and in the model expression.
"""
return list(toolz.unique(toolz.concatv(
util.columns_in_filters(self.fit_filters),
util.columns_in_filters(self.predict_filters),
self._group.columns_used())))
def align_partitions(*dfs):
""" Mutually partition and align DataFrame blocks
This serves as precursor to multi-dataframe operations like join, concat,
or merge.
Parameters
----------
dfs: sequence of dd.DataFrame, dd.Series and dd.base.Scalar
Sequence of dataframes to be aligned on their index
Returns
-------
dfs: sequence of dd.DataFrame, dd.Series and dd.base.Scalar
These must have consistent divisions with each other
divisions: tuple
Full divisions sequence of the entire result
result: list
A list of lists of keys that show which data exist on which
divisions
"""
_is_broadcastable = partial(is_broadcastable, dfs)
dfs1 = [df for df in dfs
if isinstance(df, _Frame) and
not _is_broadcastable(df)]
if len(dfs) == 0:
raise ValueError("dfs contains no DataFrame and Series")
if not all(df.known_divisions for df in dfs1):
raise ValueError("Not all divisions are known, can't align "
"partitions. Please use `set_index` "
"to set the index.")
divisions = list(unique(merge_sorted(*[df.divisions for df in dfs1])))
if len(divisions) == 1: # single value for index
divisions = (divisions[0], divisions[0])
dfs2 = [df.repartition(divisions, force=True)
if isinstance(df, _Frame) else df for df in dfs]
result = list()
inds = [0 for df in dfs]
for d in divisions[:-1]:
L = list()
for i, df in enumerate(dfs2):
if isinstance(df, _Frame):
j = inds[i]
divs = df.divisions
if j < len(divs) - 1 and divs[j] == d:
L.append((df._name, inds[i]))
inds[i] += 1
else:
L.append(None)
else: # Scalar has no divisions
L.append(None)
result.append(L)
return dfs2, tuple(divisions), result
def from_collections(cls, name, layer, dependencies=()):
""" Construct a HighLevelGraph from a new layer and a set of collections
This constructs a HighLevelGraph in the common case where we have a single
new layer and a set of old collections on which we want to depend.
This pulls out the ``__dask_layers__()`` method of the collections if
they exist, and adds them to the dependencies for this new layer. It
also merges all of the layers from all of the dependent collections
together into the new layers for this graph.
Parameters
----------
name : str
The name of the new layer
layer : Mapping
The graph layer itself
dependencies : List of Dask collections
A lit of other dask collections (like arrays or dataframes) that
have graphs themselves
Examples
--------
In typical usage we make a new task layer, and then pass that layer
along with all dependent collections to this method.
>>> def add(self, other):
... name = 'add-' + tokenize(self, other)
... layer = {(name, i): (add, input_key, other)
... for i, input_key in enumerate(self.__dask_keys__())}
... graph = HighLevelGraph.from_collections(name, layer, dependencies=[self])
... return new_collection(name, graph)
"""
layers = {name: layer}
deps = {}
deps[name] = set()
for collection in toolz.unique(dependencies, key=id):
if is_dask_collection(collection):
graph = collection.__dask_graph__()
if isinstance(graph, HighLevelGraph):
layers.update(graph.layers)
deps.update(graph.dependencies)
with ignoring(AttributeError):
deps[name] |= set(collection.__dask_layers__())
else:
key = id(graph)
layers[key] = graph
deps[name].add(key)
deps[key] = set()
else:
raise TypeError(type(collection))
return cls(layers, deps)
def __hash__(self):
custom_parts = tuple(
getattr(self, slot)
for slot in toolz.unique(self.__slots__ + ('nullable', )))
return hash((type(self), ) + custom_parts)
def distinct_roots(*expressions):
roots = toolz.concat(
expression._root_tables() for expression in expressions
)
return list(toolz.unique(roots, key=id))
def physical_tables_node(node):
# Iterative case. Any other Node's physical roots are the unique physical
# roots of that Node's root tables.
return list(unique(concat(map(physical_tables, node.root_tables()))))
def physical_tables_node(node):
# Iterative case. Any other Node's physical roots are the unique physical
# roots of that Node's root tables.
tables = toolz.concat(map(physical_tables, node.root_tables()))
return list(toolz.unique(tables, key=id))
def __iter__(self):
return unique(concat(self.dicts.values()))
def find_source_table(expr):
"""Find the first table expression observed for each argument that the
expression depends on
Parameters
----------
expr : ir.Expr
Returns
-------
table_expr : ir.TableExpr
Examples
--------
>>> import ibis
>>> t = ibis.table([('a', 'double'), ('b', 'string')], name='t')
>>> expr = t.mutate(c=t.a + 42.0)
>>> expr # doctest: +NORMALIZE_WHITESPACE
ref_0
UnboundTable[table]
name: t
schema:
a : double
b : string
Selection[table]
table:
Table: ref_0
selections:
Table: ref_0
c = Add[double*]
left:
a = Column[double*] 'a' from table
ref_0
right:
Literal[double]
42.0
>>> find_source_table(expr)
UnboundTable[table]
name: t
schema:
a : double
b : string
>>> left = ibis.table([('a', 'int64'), ('b', 'string')])
>>> right = ibis.table([('c', 'int64'), ('d', 'string')])
>>> result = left.inner_join(right, left.a == right.c)
>>> find_source_table(result) # doctest: +ELLIPSIS
Traceback (most recent call last):
...
NotImplementedError: More than one base table not implemented
"""
first_tables = []
stack = [expr]
seen = set()
while stack:
e = stack.pop()
op = e.op()
if op not in seen:
seen.add(op)
arguments = [
arg for arg in reversed(list(op.flat_args()))
if isinstance(arg, ir.Expr)
]
first_tables.extend(arg for arg in arguments
if isinstance(arg, ir.TableExpr))
stack.extend(arg for arg in arguments
if not isinstance(arg, ir.TableExpr))
options = list(toolz.unique(first_tables, key=id))
if len(options) > 1:
raise NotImplementedError('More than one base table not implemented')
return options[0]
def inputs(self):
return tuple(unique(concat(v.inputs for v in self.values)))
def _leaves(self):
return list(unique(tconcat(i._leaves() for i in self.children)))
def distinct_roots(*expressions):
# TODO: move to analysis
roots = toolz.concat(expr.op().root_tables() for expr in expressions)
return list(toolz.unique(roots))
def flat_unique(ls):
"""Flatten ``ls``, filter by unique id, and return a list"""
return list(unique(chain.from_iterable(ls), key=id))
def find_source_table(expr):
"""Find the first table expression observed for each argument that the
expression depends on
Parameters
----------
expr : ir.Expr
Returns
-------
table_expr : ir.TableExpr
Examples
--------
>>> import ibis
>>> t = ibis.table([('a', 'double'), ('b', 'string')], name='t')
>>> expr = t.mutate(c=t.a + 42.0)
>>> expr # doctest: +NORMALIZE_WHITESPACE
ref_0
UnboundTable[table]
name: t
schema:
a : float64
b : string
Selection[table]
table:
Table: ref_0
selections:
Table: ref_0
c = Add[float64*]
left:
a = Column[float64*] 'a' from table
ref_0
right:
Literal[float64]
42.0
>>> find_source_table(expr)
UnboundTable[table]
name: t
schema:
a : float64
b : string
>>> left = ibis.table([('a', 'int64'), ('b', 'string')])
>>> right = ibis.table([('c', 'int64'), ('d', 'string')])
>>> result = left.inner_join(right, left.a == right.c)
>>> find_source_table(result) # doctest: +ELLIPSIS
Traceback (most recent call last):
...
NotImplementedError: More than one base table not implemented
"""
def finder(expr):
if isinstance(expr, ir.TableExpr):
return lin.halt, expr
else:
return lin.proceed, None
first_tables = lin.traverse(finder, expr.op().flat_args())
options = list(toolz.unique(first_tables, key=operator.methodcaller('op')))
if len(options) > 1:
raise NotImplementedError('More than one base table not implemented')
return options[0]
def main():
yearn = Yearn(load_strategies=False)
excluded = {
"0xBa37B002AbaFDd8E89a1995dA52740bbC013D992",
"0xe2F6b9773BF3A015E2aA70741Bde1498bdB9425b",
"0xBFa4D8AA6d8a379aBFe7793399D3DdaCC5bBECBB",
}
resp = requests.get("https://raw.githubusercontent.com/iearn-finance/yearn-assets/master/icons/aliases.json").json()
aliases = {item["address"]: item for item in resp}
tokens = []
# Token derived by products
for product in yearn.registries:
vaults = [item.vault for item in yearn.registries[product].vaults if str(item.vault) not in excluded]
metadata = multicall_matrix(vaults, ["name", "symbol", "decimals"])
for vault in vaults:
tokens.append(
TokenInfo(
chainId=1,
address=str(vault),
name=aliases.get(str(vault), metadata[vault])["name"],
decimals=metadata[vault]["decimals"],
symbol=aliases.get(str(vault), metadata[vault])["symbol"],
logoURI=f"https://raw.githubusercontent.com/yearn/yearn-assets/master/icons/tokens/{vault}/logo.svg",
tags=[product],
)
)
# Token from special / side projects
special = [
Contract("0xD0660cD418a64a1d44E9214ad8e459324D8157f1") # WOOFY
]
metadata = multicall_matrix(special, ["name", "symbol", "decimals"])
for token in special:
tokens.append(
TokenInfo(
chainId=1,
address=str(token),
name=aliases.get(str(token), metadata[token])["name"],
decimals=metadata[token]["decimals"],
symbol=aliases.get(str(token), metadata[token])["symbol"],
logoURI=f"https://raw.githubusercontent.com/yearn/yearn-assets/master/icons/tokens/{token}/logo.svg",
tags=["special"],
)
)
deploy_blocks = {token.address: contract_creation_block(token.address) for token in tokens}
tokens = unique(tokens, key=lambda token: token.address)
tokens = sorted(tokens, key=lambda token: deploy_blocks[token.address])
version = Version(major=1, minor=len(tokens), patch=0)
timestamp = datetime.fromtimestamp(get_block_timestamp(max(deploy_blocks.values())), timezone.utc).isoformat()
logo = "https://raw.githubusercontent.com/yearn/yearn-assets/master/icons/tokens/0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e/logo.svg"
print(f"{version=}\n{timestamp=}")
tokenlist = TokenList("Yearn", timestamp, version, tokens, logoURI=logo)
for token in tokenlist.tokens:
assert len(token.symbol) <= 20, f"{token.symbol} > 20 chars, uniswap is unhappy"
path = Path("static/tokenlist.json")
path.parent.mkdir(exist_ok=True)
path.write_text(json.dumps(tokenlist.to_dict(), separators=(",", ":")))
print(f"saved to {path}")
def __repr__(self):
return '{}({})'.format(
self.name, ', '.join('{}={!r}'.format(slot, getattr(self, slot))
for slot in toolz.unique(self.__slots__ +
('nullable', ))))
def unify_wheres(selectables):
clauses = list(
unique((s._whereclause
for s in selectables if hasattr(s, '_whereclause')),
key=str))
return reduce(and_, clauses) if clauses else None
def get_all_froms(function):
return list(unique(concat(map(get_all_froms, function.clauses.clauses))))
def bottom_up_until_type_break(expr, scope, **kwargs):
""" Traverse bottom up until data changes significantly
Parameters
----------
expr: Expression
Expression to compute
scope: dict
namespace matching leaves of expression to data
Returns
-------
expr: Expression
New expression with lower subtrees replaced with leaves
scope: dict
New scope with entries for those leaves
Examples
--------
>>> import numpy as np
>>> s = symbol('s', 'var * {name: string, amount: int}')
>>> data = np.array([('Alice', 100), ('Bob', 200), ('Charlie', 300)],
... dtype=[('name', 'S7'), ('amount', 'i8')])
This computation completes without changing type. We get back a leaf
symbol and a computational result
>>> e = (s.amount + 1).distinct()
>>> bottom_up_until_type_break(e, {s: data}) # doctest: +SKIP
(amount, {amount: array([101, 201, 301])})
This computation has a type change midstream (``list`` to ``int``), so we
stop and get the unfinished computation.
>>> e = s.amount.sum() + 1
>>> bottom_up_until_type_break(e, {s: data})
(amount_sum + 1, {<`amount_sum` symbol; dshape='int64'>: 600})
"""
# 0. Base case. Return if expression is in scope
if expr in scope:
leaf = makeleaf(expr)
return leaf, {leaf: scope[expr]}
inputs = list(unique(expr._inputs))
# 1. Recurse down the tree, calling this function on children
# (this is the bottom part of bottom up)
exprs, new_scopes = zip(
*[bottom_up_until_type_break(i, scope, **kwargs) for i in inputs])
# 2. Form new (much shallower) expression and new (more computed) scope
new_scope = toolz.merge(new_scopes)
new_expr = expr._subs(
{i: e
for i, e in zip(inputs, exprs) if not i.isidentical(e)})
old_expr_leaves = expr._leaves()
old_data_leaves = [scope.get(leaf) for leaf in old_expr_leaves]
# 3. If the leaves have changed substantially then stop
key = lambda x: str(type(x))
if type_change(sorted(new_scope.values(), key=key),
sorted(old_data_leaves, key=key)):
return new_expr, new_scope
# 4. Otherwise try to do some actual work
try:
leaf = makeleaf(expr)
_data = [new_scope[i] for i in new_expr._inputs]
except KeyError:
return new_expr, new_scope
try:
return leaf, {
leaf: compute_up(new_expr, *_data, scope=new_scope, **kwargs)
}
except NotImplementedError:
return new_expr, new_scope
def get_all_froms(colelement):
return list(unique(concat(map(get_all_froms, colelement.get_children()))))
def plot_cache(results,
dsk,
start_time,
metric_name,
palette='GnBu',
label_size=60,
**kwargs):
"""Visualize the results of profiling in a bokeh plot.
Parameters
----------
results : sequence
Output of CacheProfiler.results
dsk : dict
The dask graph being profiled.
start_time : float
Start time of the profile.
metric_name : string
Metric used to measure cache size
palette : string, optional
Name of the bokeh palette to use, must be key in bokeh.palettes.brewer.
label_size: int (optional)
Maximum size of output labels in plot, defaults to 60
**kwargs
Other keyword arguments, passed to bokeh.figure. These will override
all defaults set by visualize.
Returns
-------
The completed bokeh plot object.
"""
defaults = dict(title="Profile Results",
tools="hover,save,reset,resize,wheel_zoom,xpan",
plot_width=800,
plot_height=300)
defaults.update(
(k, v) for (k, v) in kwargs.items() if k in bp.Figure.properties())
if results:
starts, ends = list(zip(*results))[3:]
tics = list(sorted(unique(starts + ends)))
groups = groupby(lambda d: pprint_task(d[1], dsk, label_size), results)
data = {}
for k, vals in groups.items():
cnts = dict.fromkeys(tics, 0)
for v in vals:
cnts[v.cache_time] += v.metric
cnts[v.free_time] -= v.metric
data[k] = list(accumulate(add, pluck(1, sorted(cnts.items()))))
tics = [i - start_time for i in tics]
p = bp.figure(x_range=[0, max(tics)], **defaults)
for (key, val), color in zip(data.items(),
get_colors(palette, data.keys())):
p.line('x',
'y',
line_color=color,
line_width=3,
source=bp.ColumnDataSource({
'x': tics,
'y': val,
'label': [key for i in val]
}))
else:
p = bp.figure(y_range=[0, 10], x_range=[0, 10], **defaults)
p.grid.grid_line_color = None
p.axis.axis_line_color = None
p.axis.major_tick_line_color = None
p.yaxis.axis_label = "Cache Size ({0})".format(metric_name)
p.xaxis.axis_label = "Time (s)"
hover = p.select(HoverTool)
hover.tooltips = """
<div>
<span style="font-size: 14px; font-weight: bold;">Task:</span>
<span style="font-size: 10px; font-family: Monaco, monospace;">@label</span>
</div>
"""
return p
def get_all_froms(sel):
return list(unique(sel.locate_all_froms()))
def physical_tables_join(join):
# Physical roots of Join nodes are the unique physical roots of their
# left and right TableNodes.
func = compose(physical_tables, methodcaller('op'))
return list(unique(concat(map(func, (join.left, join.right)))))
def get_inner_columns(f):
unique_columns = unique(concat(map(get_inner_columns, f.clauses)))
lowered = [x.label(getattr(x, 'name', None)) for x in unique_columns]
return [getattr(sa.func, f.name)(*lowered)]
def __getstate__(self):
return {
slot: getattr(self, slot)
for slot in toolz.unique(self.__slots__ + ('nullable', ))
}
def __iter__(self):
return toolz.unique(toolz.concat(self.layers.values()))
