def _extend_data(filters, data, inv_idx, emp_idx):
invoices = [x[inv_idx] for x in data]
get_employee_map = compose(
valmap(lambda x:
[x.get("pb_sales_employee"),
x.get("pb_sales_employee_name")]),
valmap(first),
groupby("name"),
lambda: frappe.db.sql(
"""
SELECT name, pb_sales_employee, pb_sales_employee_name FROM `tabSales Invoice`
WHERE name IN %(invoices)s
""",
values={"invoices": invoices},
as_dict=1,
),
)
employees = get_employee_map() if invoices else {}
set_employee = compose(list, lambda x: concatv(x, employees[x[inv_idx]]))
extended = [set_employee(x) for x in data]
if not filters.sales_employee:
return extended
return [x for x in extended if x[emp_idx] == filters.sales_employee]
def _get_species(patients):
species = compose(
valmap(lambda x: x['vc_species']), valmap(first), groupby('name'),
lambda: frappe.get_all('Patient',
filters=[['name', 'in', patients]],
fields=['name', 'vc_species']))
return species()
def filter_data(field, yaml_data):
"""Extract a field of data from the YAML files.
Args:
field: the name of the field to extract
yaml_data: the benchmark YAML data
Returns:
the filtered data from the YAML data
"""
return pipe(
yaml_data,
dict,
valmap(lambda val: val["data"]),
valmap(filter(lambda item: item["name"].lower() == field)),
valmap(list),
valmap(get(0, default=None)),
valfilter(lambda x: x is not None),
itemmap(lambda item: (item[0], update_dict(item[1], name=item[0]))),
lambda dict_: sorted(list(dict_.values()), key=lambda item: item["name"]),
map(
update_in(
keys=["transform"],
func=lambda x: x + [dict(expr="datum.x > 0.01", type="filter")],
)
),
)
def filter_memory_data(yaml_data):
"""Filter the memory time data from the meta.yaml's
Args:
yaml_data: the benchmark YAML data
Returns:
memory versus time data
"""
def time_ratio(data):
"""Calcuate the sim_time over wall_time ration
"""
def not0(value):
"""Set to 1e-10 if 0
"""
if value == 0:
return 1e-10
return value
return pipe(
data[-1],
juxt(lambda x: x.get('sim_time', x.get('time')),
lambda x: x.get('wall_time', x.get('time'))),
lambda x: float(x[0]) / not0(float(x[1])))
def memory_usage(data):
"""Calculate the memory usage in KB
"""
unit_map = dict(GB=1048576., KB=1., MB=1024., B=1. / 1024.)
if isinstance(data, dict):
data_ = data
else:
data_ = data[-1]
key = next(k for k in data_.keys() if 'value' in k)
return float(data_[key]) * unit_map[data_.get('unit', 'KB')]
def make_datum(data):
"""Build an item in the data list for one simulation
"""
return dict(
name='efficiency',
values=[
dict(time_ratio=time_ratio(data['run_time']),
memory_usage=memory_usage(data['memory_usage']))
],
)
return pipe(
yaml_data, dict, valmap(lambda x: x['data']),
valmap(
filter(lambda item: item['name'].lower() in
('memory_usage', 'run_time'))),
valmap(map(lambda x: (x['name'], x['values']))), valmap(dict),
valmap(make_datum),
itemmap(lambda item: (item[0], update_dict(item[1], name=item[0]))),
lambda dict_: sorted(list(dict_.values()),
key=lambda item: item['name']))
def get_end_to_end_metrics_table(y_true, results_for_methods_optimized,
results_for_methods_default):
metrics_for_methods_optimized = valmap(
lambda r: compute_classification_metrics_from_results_with_statistics(
y_true, r), results_for_methods_optimized)
metrics_for_methods_default = valmap(
lambda r: compute_classification_metrics_from_results_with_statistics(
y_true, r), results_for_methods_default)
return metrics_for_methods_optimized, metrics_for_methods_default
def _get_balance_qty(items, filters):
clauses = _get_clauses(filters)
return compose(
valmap(lambda x: x['qty']), valmap(first), groupby('item_code'),
lambda: frappe.db.sql("""
SELECT
item_code,
SUM(actual_qty) as qty
FROM `tabBin`
WHERE {clauses}
GROUP BY item_code
""".format(clauses=clauses),
values=merge(filters, {'items': items}),
as_dict=1))()
def get_data(filter_func):
"""Read in the YAML data and group by benchmark id
Args:
filter_func: function to filter data
Returns:
a dictionary with benchmark ids as keys and lists of appropriate
data for values
"""
return pipe(
get_yaml_data(),
groupby(lambda item: "{0}.{1}".format(
item[1]['benchmark']['id'], str(item[1]['benchmark']['version']))),
valmap(filter_func), valmap(vega2to3))
def transform(self, X, exposure=None):
data = valmap(growd(2), valfilter(notnone, dict(X=X,
exposure=exposure)))
return np.concatenate(tuple(
map(compose(growd(2), methodcaller('predict', **data)),
self.estimators)),
axis=1)
def ccds_to_bed(ccds_stream):
"""Convert CCDS dump to Chanjo-style BED stream.
Main entry point for default Chanjo converter (ccds). It converts
a sorted (start, chrom) CCDS database to the Chanjo BED-format.
Args:
ccds_stream (file): file handle to read CCDS lines from
Yields:
Interval: interval with merged block and superblock ids
"""
return pipe(
ccds_stream,
filter(grep('Public')), # filter out Public tx
map(text_type.rstrip), # strip \n and spaces
map(split(sep='\t')), # split into list
map(extract_intervals), # convert to Interval
concat, # flatten
map(rename_sex_interval), # rename sex contigs
partial(lazy_groupby, key=attrgetter('contig')), # group by contig
pluck(1), # extract second item
map(groupby(attrgetter('name'))), # non-lazy group by id
map(valmap(merge_related_elements)), # group intervals
map(itervalues), # extract values
map(partial(sorted, key=attrgetter('start'))), # sort by start pos
concat # flatten
)
def _append_summary(data):
def make_data(val):
clients = compose(
count,
unique,
pluck('customer'),
lambda: val
)
animals = compose(
valmap(count),
groupby('species'),
lambda: val
)
return {
'total_val': reduce(lambda total, x: total + x.get('total_vat'), val, 0.00),
'animals': _get_dict_to_csv(animals()),
'clients': clients()
}
sales_persons = compose(
valmap(make_data),
groupby('sales_person_name'),
lambda: data
)()
data.append({'invoice_no': "'-'"}) # for report html (break loop)
for k, v in sales_persons.items():
sales_person = k or 'Not specified'
data.append({'invoice_no': "'Sales Person'", 'item': f"'{sales_person}'"})
data.append({'invoice_no': "'Total Amt'", 'item': f"'{fmt_money(v.get('total_val'))}'"})
data.append({'invoice_no': "'Clients'", 'item': f"'{v.get('clients')}'"})
data.append({'invoice_no': "'Animals'", 'item': f"'{v.get('animals')}'"})
data.append({})
return data
def get_batch_no(doctype, txt, searchfield, start, page_len, filters):
from erpnext.controllers.queries import get_batch_no
result = get_batch_no(doctype, txt, searchfield, start, page_len, filters)
get_batch_prices = compose(
valmap(lambda x: x[0].get("px_price_list_rate")),
groupby("name"),
lambda batches: frappe.db.sql(
"""
SELECT name, px_price_list_rate FROM `tabBatch`
WHERE name IN %(batches)s
""",
values={"batches": [x[0] for x in batches]},
as_dict=1,
)
if batches
else {},
)
batch_prices = get_batch_prices(result)
def set_price(batch):
price = batch_prices.get(batch[0])
if not price:
return batch
return batch + tuple(["PRICE-{}".format(price)])
return [set_price(x) for x in result]
def _(dictionary):
try:
parameters = pipeline(
dictionary,
[
keyfilter(lambda k: k != '__type'),
valmap(json_deserialize_types),
],
)
Class = get_member_by_path(dictionary['__type'])
instance = make_instance(Class)
for key, value in parameters.items():
setattr(instance, key, json_deserialize_types(value))
return instance
except KeyError:
return valmap(json_deserialize_types, dictionary)
def filter_data(yaml_data):
"""Extract the free_energy data from the YAML files.
Args:
yaml_data: the benchmark YAML data
Returns:
the free_energy data from the YAML data
"""
return pipe(
yaml_data, dict, valmap(lambda val: val['data']),
valmap(filter(lambda item: item['name'].lower() == 'free_energy')),
valmap(list), valmap(get(0)),
itemmap(lambda item: (item[0], update_dict(item[1], name=item[0]))),
lambda dict_: sorted(list(dict_.values()),
key=lambda item: item['name']))
def get_char_to_xhe_shapes() -> Dict[str, List[str]]:
char_to_shape = pipe(CharHeShapeTable.select(),
map(lambda e: (e.char, e.shapes)),
filter(lambda e: e[0] != '' and e[1] != ''),
groupby(lambda e: e[0]),
valmap(lambda e: [s[1] for s in e]), dict)
return char_to_shape
def get_freight_rates():
price_list = frappe.get_cached_value(
"Selling Settings",
None,
"selling_price_list",
)
def get_rate(item):
args = {"price_list": price_list, "uom": item.get("uom")}
rate = get_item_price(args, item.get("item_code"), ignore_party=True)
if rate:
return rate[0][1]
return 0
get_freight_items = compose(
valmap(first),
groupby("based_on"),
map(lambda x: merge(x, {"rate": get_rate(x)})),
frappe.db.sql,
)
return get_freight_items(
"""
SELECT name AS item_code, stock_uom AS uom, gg_freight_based_on AS based_on
FROM `tabItem`
WHERE gg_freight_based_on IN ('Packages', 'Weight')
""",
as_dict=1,
)
def test_returns_lookp(self):
self.assertDictEqual(
# "references" in the result are iterators so we need
# to convert them to a list before asserting.
tlz.valmap(
over("references", list),
make_lookup([{
"path":
"/nix/store/foo",
"references":
["/nix/store/foo", "/nix/store/bar", "/nix/store/hello"]
}, {
"path": "/nix/store/bar",
"references": ["/nix/store/bar", "/nix/store/tux"]
}, {
"path": "/nix/store/hello",
"references": []
}])),
{
"/nix/store/foo": {
"path": "/nix/store/foo",
"references": ["/nix/store/bar", "/nix/store/hello"]
},
"/nix/store/bar": {
"path": "/nix/store/bar",
"references": ["/nix/store/tux"]
},
"/nix/store/hello": {
"path": "/nix/store/hello",
"references": []
}
})
def print_earnings():
registry = Registry()
registry.load_strategies()
earnings = []
for vault in registry.vaults:
for strategy in vault.strategies:
contract = strategy.strategy
strategist = getattr(contract, "strategist", None)
if strategist:
config = vault.vault.strategies(contract).dict()
if config.get("performanceFee"):
earnings.append({"strategist": strategist(), **config})
if earnings:
print("....All earnings:")
pprint(earnings)
data = valmap(
compose(lambda fg: sum(gain / fee for fee, gain in fg),
pluck(["performanceFee", "totalGain"])),
groupby("strategist", earnings))
print("....Earnings grouped by strategist")
pprint(data)
else:
print("No earnings found")
def get_char_to_lu_phones() -> Dict[str, List[str]]:
char_to_phones = pipe(CharPhoneTable.select(),
map(lambda e: (e.char, e.lu)),
filter(lambda e: e[0] != '' and e[1] != ''),
groupby(lambda e: e[0]),
valmap(lambda phones: [e[1] for e in phones]), dict)
return char_to_phones
def filter_data(field, yaml_data):
"""Extract a field of data from the YAML files.
Args:
field: the name of the field to extract
yaml_data: the benchmark YAML data
Returns:
the filtered data from the YAML data
"""
return pipe(
yaml_data, dict, valmap(lambda val: val['data']),
valmap(filter(lambda item: item['name'].lower() == field)),
valmap(list), valmap(get(0)),
itemmap(lambda item: (item[0], update_dict(item[1], name=item[0]))),
lambda dict_: sorted(list(dict_.values()),
key=lambda item: item['name']))
def tom_jacobian_to_jacobian(panes):
jac = defaultdict(dict)
def flip_axes_and_squeeze(x):
return x[::-1, ::-1].squeeze()
q_scale = Lc / 1000
t_scale = cp
jac["q"]["q"] = panes["q"]["q"]
jac["q"]["s"] = panes["q"]["T"] / (q_scale / t_scale)
jac["s"]["q"] = panes["T"]["q"] / (t_scale / q_scale)
jac["s"]["s"] = panes["T"]["T"]
jac = valmap(valmap(flip_axes_and_squeeze), jac)
return jac
def get_values(_type):
fields = list(map(lambda x: "{}_{}".format(_type, x), params))
_get = compose(
valmap(lambda x: x or 0),
keymap(lambda x: x.replace("{}_".format(_type), "")),
keyfilter(lambda x: x in fields),
)
return _get(data)
def tom_base_state_to_base_state(tom_base_state):
tom_base_state = valmap(compose(np.copy, np.flip), tom_base_state)
q, T, z, rho = get(["qv", "T", "z", "rho"], tom_base_state)
return {
"QT": q * 1000.0,
"SLI": T + grav / cp * z,
"height_center": z,
"density": rho
}
def _compute_scaler_data_from_xarray(dataset):
logger.info("Computing mean")
mean = dataset.mean(['x', 'y', 'time'])
logger.info("Computing std")
scale = dataset.std(['x', 'y', 'time'])
args = [valmap(torch.squeeze, _convert_dataset_to_torch_dict(arg))
for arg in [mean, scale]]
return args
def groupby_count(func):
"""Group the simulation data based on a function.
Args:
func: function to group by
Returns:
grouped data
"""
return pipe(get_yaml_data(), map(get(1)), groupby(func), valmap(count))
def from_file_path(cls,
file_path: FilePath,
sheet_name: str,
*,
row_limit: int = 100):
"""Help function to populate the columns of a sheet."""
wb = get_wb(file_path)
ws = wb[sheet_name]
rows = tz.take(row_limit, ws.rows)
header = next(rows)
names = [c.value for c in header]
letters = [c.column_letter for c in header]
indices = [c.column for c in header]
data_types = tz.pipe(
rows
# For each row, create a dict usng names has keys
,
tz.map(lambda row: dict(zip(names, row)))
# Get the .xlsx data_type for each cell
,
tz.map(tz.valmap(lambda cell: cell.data_type))
# Combine cells into a list per column
,
tz.merge_with(list)
# Count the cells for each data type in the column
,
tz.valmap(tz.frequencies)
# Consolidate types
,
tz.valmap(lambda freq: (
# If at least 1 "d"
"date" if "d" in freq else
# If at least 1 "s"
"text" if "s" in freq else
# If at least 1 "n"
"number" if "n" in freq else str(freq))),
lambda d: [v for k, v in d.items()])
cols = [
Col(name=N, letter=L, index=I, data_type=D)
for N, L, I, D in zip(names, letters, indices, data_types)
]
return cls(name=sheet_name, cols=cols)
def extract_ctr_children_sum(doc: Tabs_doc) -> Dict[Tab_num, Disp_dur]:
default = {'dispatchCount': np.nan, 'duration': np.nan}
def reduce_children(chs: List[Disp_dur]):
# TODO: extract host info?
disp_durs = lmap(extract_dur_disp_tab, chs)
return reduce_keys(op.add, disp_durs) or default
tab_docs = z.valmap(z.compose(reduce_children, itg('children')), doc)
return tab_docs
def make_data(val):
clients = compose(count, unique, pluck('customer'), lambda: val)
animals = compose(valmap(count), groupby('species'), lambda: val)
return {
'total_val':
reduce(lambda total, x: total + x.get('total_vat'), val, 0.00),
'animals':
_get_dict_to_csv(animals()),
'clients':
clients()
}
def update_dict(funcs, values):
"""Apply a dict of funcs to a dict of values
>>> values = dict(a=1., b=2., c=4.)
>>> funcs = dict(
... a=lambda a, b, c: a + b + c,
... b=lambda a, b, c: a * b * c,
... c=lambda a, b, c: a - b - c
... )
>>> assert update_dict(funcs, values) == dict(a=7, b=8, c=-5)
"""
return valmap(lambda f: f(**values), funcs)
def _prepare_kwargs(self, **kwargs):
"""Filter keywords with the function arguments. Call
any value that is callable, no arguments are applied
to these function.
"""
return valmap(
self._call_lazy_function,
merge(
keyfilter(partial(operator.contains, self.arguments), self.keywords),
kwargs,
)
)
def from_tom_base_state(tom_base_state):
"""Return WaveEq from Tom's base state"""
# TODO refactor this to an abstract factor (A)
tom_base_state = valmap(compose(np.copy, np.flip), tom_base_state)
q, T, z, rho = get(["qv", "T", "z", "rho"], tom_base_state)
base_state = {
"QT": q * 1000.0,
"SLI": T + grav / cp * z,
"height_center": z,
"density": rho
}
return WaveEq(base_state)
def save_as_html(distinct_words, file_name):
"""Generate and save an html display of the distinct words"""
## Wrangle data for presentation
# Convert tokens into a single string
def get_token_string(given_values):
"""Return a token string, if the given values are a list of dictionaries"""
# check if it is a list of token-related information
if (isinstance(given_values, list) and
len(given_values) > 0 and
isinstance(given_values[0], dict)):
return tz.pipe(
given_values,
tz.map(lambda x: x['token']),
tz.map(wrap_in_highlight_link), # wrap in link to highlight words
tz.reduce(lambda x,y: u"{}, {}".format(x, y)))
# return empty string for empty lists
elif isinstance(given_values, list) and len(given_values) == 0:
return ''
# check if it is a date range in need of formating
elif isinstance(given_values, list) and len(given_values) == 2:
return format_date_range(given_values)
else:
return given_values
def format_date_range(given_date_range):
"""Return a pretty version of the given date_range"""
date_range = map(
lambda x: dt.datetime.strptime(x, "%Y-%m-%dT%H:%M:%SZ"),
given_date_range)
return "{} to {} UTC".format(
date_range[0].strftime("%Y-%m-%d %H:%M"),
date_range[1].strftime("%H:%M"))
def wrap_in_highlight_link(given_string):
"""return the given string wrapped in the html code to highlight
other occurances of that same word"""
return u"""<a href="javascript:void($('.distinct_words').removeHighlight().highlight('{string}'));">{string}</a>""".format(string=given_string)
formated_distinct_words = tz.pipe(
distinct_words,
tz.map(
tz.valmap(get_token_string)),
list)
## Send to Template For Display
template_dir = 'templates'
loader = jinja2.FileSystemLoader(template_dir)
environment = jinja2.Environment(loader=loader)
template = environment.get_template('distinct_words.html')
with open(file_name, 'w') as f:
tz.pipe(
template.render(distinct_words = formated_distinct_words),
lambda x: x.encode('utf8'),
lambda x: f.write(x))
def broadcast_dimensions(argpairs, numblocks, sentinels=(1, (1,))):
""" Find block dimensions from arguments
Parameters
----------
argpairs: iterable
name, ijk index pairs
numblocks: dict
maps {name: number of blocks}
sentinels: iterable (optional)
values for singleton dimensions
Examples
--------
>>> argpairs = [('x', 'ij'), ('y', 'ji')]
>>> numblocks = {'x': (2, 3), 'y': (3, 2)}
>>> broadcast_dimensions(argpairs, numblocks)
{'i': 2, 'j': 3}
Supports numpy broadcasting rules
>>> argpairs = [('x', 'ij'), ('y', 'ij')]
>>> numblocks = {'x': (2, 1), 'y': (1, 3)}
>>> broadcast_dimensions(argpairs, numblocks)
{'i': 2, 'j': 3}
Works in other contexts too
>>> argpairs = [('x', 'ij'), ('y', 'ij')]
>>> d = {'x': ('Hello', 1), 'y': (1, (2, 3))}
>>> broadcast_dimensions(argpairs, d)
{'i': 'Hello', 'j': (2, 3)}
"""
# List like [('i', 2), ('j', 1), ('i', 1), ('j', 2)]
L = concat([zip(inds, dims)
for (x, inds), (x, dims)
in join(first, argpairs, first, numblocks.items())])
g = groupby(0, L)
g = dict((k, set([d for i, d in v])) for k, v in g.items())
g2 = dict((k, v - set(sentinels) if len(v) > 1 else v) for k, v in g.items())
if g2 and not set(map(len, g2.values())) == set([1]):
raise ValueError("Shapes do not align %s" % g)
return valmap(first, g2)
def vega_contours(treant, counter=0):
"""
Get the contours as Vega data.
Args:
treant: a Treant object with data files
Returns:
contours formatted as Vega data
"""
return pipe(
treant,
all_files('*.nc'),
map(contours_from_datafile),
concat,
map(pandas.DataFrame),
map(lambda x: x.rename(columns={0: 'x', 1: 'y'})),
map(lambda x: x.to_dict(orient='records')),
map(map(valmap(float))),
map(list),
enum(lambda i, x: dict(name='contour_data{0}_{1}'.format(i, counter),
values=x)),
)
