def append_urlX_to_X(target, source, **kwargs): with closing(urlopen(source.url, timeout=kwargs.pop('timeout', None))) as r: chunk_size = 16 * source.chunk_size # 11.12.18 harleen - part2 - hack to enable writing to ftp server # original # with open(target.path, 'wb') as fp: # for chunk in iter(curry(r.read, chunk_size), b''): # fp.write(chunk) # return target # changes begin f = target.path if f.split(':')[0].lower() == 'ftp': import io from io import StringIO from ftplib import FTP from pyparsing import Combine, Suppress, Word, printables, alphas, ZeroOrMore, alphanums, Group, delimitedList, nums ftp_expression = Suppress('ftp://') + Word(alphanums) + ZeroOrMore(Suppress(':')) + Word(alphanums) + ZeroOrMore(Suppress('@')) + Word(alphanums) + ZeroOrMore(Suppress(':')) + Word(nums) + ZeroOrMore(Suppress('/')) + Word(printables) username, password, ip, port, file_loc = ftp_expression.parseString(f) ftp = FTP(ip, username, password) # buffer = StringIO() # df.to_csv(buffer, index=False, header=has_header, sep=sep, encoding=encoding) # text = buffer.getvalue() # bio = io.BytesIO(str.encode(text)) # ftp.storbinary('STOR '+file_loc, bio) for chunk in iter(curry(r.read, chunk_size), b''): ftp.storbinary('APPE ' + file_loc, io.BytesIO(chunk)) return target else: with open(target.path, 'wb') as fp: for chunk in iter(curry(r.read, chunk_size), b''): fp.write(chunk) return target
def main(): """Main method.""" # create a player named tuple Player = namedtuple('Player', ['first', 'last', 'number', 'team', 'city']) # create some player named tuples m_j = Player(first='Michael', last='Jordan', number='23', team='Bulls', city='Chicago') k_b = Player(first='Kobe', last='Bryant', number='24', team='Lakers', city='Los Angeles') l_b = Player(first='LeBron', last='James', number='23', team='Cavaliers', city='Cleveland') k_p = Player(first='Kristaps', last='Porzingis', number='6', team='Knicks', city='New York') k_d = Player(first='Kevin', last='Durant', number='35', team='Warriors', city='Oakland') # store the players in tuple players = (m_j, k_b, l_b, k_p, k_d) # filter two_three = filter(lambda x: x.number == '23', players) print(tuple(two_three)) # => (Player(first='Michael', last='Jordan', number='23', team='Bulls', # city='Chicago'), Player(first='LeBron', last='James', number='23', # team='Cavaliers', city='Cleveland')) # map result = map(lambda x: ''.join([x.first, ' ', x.last]), players) print(tuple(result)) # => ('Michael Jordan', 'Kobe Bryant', 'LeBron James', 'Kristaps Porzingis', 'Kevin Durant') # reduce result = reduce(lambda x, y: x + y, [1, 2, 3], 0) print(result) # => 6 # compose nums = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5] c_greater = curry(greater) greater_zero = c_greater(y_val=0) result = compose(filter(greater_zero), filter(iseven))(nums) print(tuple(result)) # => (2, 4)
def many(a, b, binop=None, reduction=None, **kwargs): """ Apply binary operator to pairwise to sequences, then reduce. >>> many([1, 2, 3], [10, 20, 30], mul, sum) # dot product 140 """ return reduction(map(curry(binop, **kwargs), a, b))
def append_urlX_to_X(target, source, **kwargs): with closing(urlopen(source.url)) as r: chunk_size = 16 * source.chunk_size with open(target.path, 'wb') as fp: for chunk in iter(curry(r.read, chunk_size), ''): fp.write(chunk) return target
def transpose(a, axes=None): axes = axes or tuple(range(a.ndim))[::-1] return atop(curry(np.transpose, axes=axes), next(names), axes, a, tuple(range(a.ndim)), dtype=a._dtype)
def append_urlX_to_X(target, source, **kwargs): with closing(urlopen(source.url, timeout=kwargs.pop("timeout", None))) as r: chunk_size = 16 * source.chunk_size with open(target.path, "wb") as fp: for chunk in iter(curry(r.read, chunk_size), b""): fp.write(chunk) return target
def curry(*args, **kwargs): """Curry a callable function Deprecated: use toolz.curried.curry() instead. """ warnings.warn( "alt.curry() is deprecated, and will be removed in a future release. " "Use toolz.curried.curry() instead.", AltairDeprecationWarning, ) return curried.curry(*args, **kwargs)
def _func(coords, domain=None): # pylint: disable=unused-argument return pipe( (x_points, y_points, z_points), enumerate, map_(lambda x: flag_it(x[1], coords, x[0])), list, curry(fold)(lambda x, y: x & y), lambda x: (x & (coords[:, 0] < (max_x - eps(coords)))) if max_x is not None else x, np.where, first, )
class placeholder2Bis(placeholderI): MY_FUNC = curried.curry(curryX)(cls=Pipe2) MY_PIPE = Pipe2 def special__ror__np(self, argu): return self.__ror__(argu, okO=True) def __ror__(self, other, okO=False): if iterable(other) and type(other) != "str": other = Pipe2.collection(other) return placeholder2( func=lambda x, self=self, other=other: x.__ror__(other) if isinstance(other, Pipe2) and okO else self.func(other, x))
def sample_url_line_delimited(data, lines=5, encoding="utf-8", timeout=None): """Get a size `length` sample from an URL CSV or URL line-delimited JSON. Parameters ---------- data : URL(CSV) A hosted CSV lines : int, optional, default ``5`` Number of lines to read into memory """ with closing(urlopen(data.url, timeout=timeout)) as r: raw = pipe( r, take(lines), map(bytes.strip), curry(codecs.iterdecode, encoding=encoding), b"\n".decode(encoding).join ) with tmpfile(data.filename) as fn: with codecs.open(fn, "wb", encoding=encoding) as f: f.write(raw) yield fn
def sample_url_line_delimited(data, lines=5, encoding='utf-8'): """Get a size `length` sample from an URL CSV or URL line-delimited JSON. Parameters ---------- data : URL(CSV) A hosted CSV lines : int, optional, default ``5`` Number of lines to read into memory """ with closing(urlopen(data.url)) as r: raw = pipe(r, take(lines), map(bytes.strip), curry(codecs.iterdecode, encoding=encoding), b'\n'.decode(encoding).join) with tmpfile(data.filename) as fn: with codecs.open(fn, 'wb', encoding=encoding) as f: f.write(raw) yield fn
import numpy as np from toolz.curried import curry, pipe try: import edt transform_edt = curry(edt.edt)(black_border=False) except ImportError: import scipy.ndimage.morphology as morph transform_edt = morph.distance_transform_edt print("you can install edt for speed-up as - ") print("pip install edt") pass try: import torch torch_rfft = curry(torch.rfft) # pylint: disable=invalid-name torch_irfft = curry(torch.irfft) # pylint: disable=invalid-name def conjugate(x): """ returns the conjugate of a complex torch tensor """ y = torch.empty_like(x) y[..., 1] = x[..., 1] * -1 y[..., 0] = x[..., 0] return y @curry def mult(x1, x2): """ returns the product of complext torch tensors x1 and x2
import PIL.Image from toolz.curried import map, pipe, compose, get, do, curry, count, pluck, juxt, flip import pandas import skimage import skimage.measure import skimage.filters import skimage.morphology import sys import json import pickle fcompose = lambda *args: compose(*args[::-1]) ## Binary closing to reveal the Pearlite Phase closing = curry(flip(skimage.morphology.closing)) remove_small_holes = curry(skimage.morphology.remove_small_holes) reveal_pearlite = fcompose(closing(skimage.morphology.square(5)), remove_small_holes(min_size=1000)) def reveal(data): data['pearlite_image'] = reveal_pearlite(data['clean_image']) return data if __name__ == '__main__': filename = sys.argv[1] filename_cleaned = filename.split("/")[-1].split(".")[0] filename_cleaned = "-".join(filename_cleaned.split("-")[0:-1])
TCGA archives contain one file per sample. This script aggregates them into a single ngenes x nsamples expression counts matrix. This works on the data from "Genomic Classification of Cutaneous Melanoma", aggregated here: https://tcga-data.nci.nih.gov/docs/publications/skcm_2015/ And with direct link to the RNA expression Level 3 archive: https://tcga-data.nci.nih.gov/docs/publications/skcm_2015/unc.edu_SKCM.IlluminaHiSeq_RNASeqV2.Level_3.1.10.0.tar.gz """ import numpy as np from toolz import curried as tz spliteach = tz.compose(tz.map, tz.curry(str.split)) tee = tz.compose(tz.map, tz.do) arrayfromiter = tz.curry(np.fromiter)(dtype=float) def getid_counts(lines, kind='symbol'): if kind == 'symbol': kind = 0 else: kind = 1 ids = [] res = tz.pipe(lines, spliteach(sep='\t'), tz.pluck(0), spliteach(sep='|'), tz.pluck(kind), tee(ids.append), tz.frequencies) return ids, res
import numba import scipy import numpy as np from .helpers import * from scipy.spatial import cKDTree from toolz.curried import pipe, curry from scipy.ndimage import measurements from scipy.sparse.csgraph import dijkstra from skimage.morphology import remove_small_objects from skimage.morphology import skeletonize_3d as sklz erasure = curry(remove_small_objects) @curry def accessibleRegion(S, atomH, r_h, overlap=0.01): vol = np.count_nonzero(atomH) S_mod = imfilter(x_data=(S < 1e-6), f_data=padder(atomH, np.array(S.shape))) / vol S_mod = S_mod < overlap return S_mod @curry def return_labelled(x_data): S_l, n_count = measurements.label(x_data) top = list(np.unique(S_l[:, :, 0]))[1:] bot = list(np.unique(S_l[:, :, -1]))[1:] m = list(set(top).intersection(bot)) return S_l, n_count, m
import scipy.ndimage import pytesseract import PIL.Image from toolz.curried import map, pipe, compose, get, do, curry, count, pluck, juxt, flip import pandas import skimage import skimage.measure import skimage.filters import skimage.morphology import json import sys import pickle ## Binary closing to reveal the Pearlite Phase remove_small_holes = curry(skimage.morphology.remove_small_holes) def clean(data): data['clean_image'] = ~remove_small_holes(~data['threshold_image'], data['min_size']) return data if __name__ == '__main__': filename = sys.argv[1] filename_cleaned = filename.split("/")[-1].split(".")[0] filename_cleaned = "-".join(filename_cleaned.split("-")[0:-1]) data = None with open(filename, "r") as intermediate: data = pickle.load(intermediate) result = clean(data)
import numpy as np from toolz.curried import curry from toolz.curried import map from toolz.curried import pipe fft = curry(np.fft.fft) # pylint: disable=invalid-name ifft = curry(np.fft.ifft) # pylint: disable=invalid-name fftn = curry(np.fft.fftn) # pylint: disable=invalid-name ifftn = curry(np.fft.ifftn) # pylint: disable=invalid-name fftshift = curry(np.fft.fftshift) # pylint: disable=invalid-name conj = curry(np.conj) func = curry(lambda x, y: conj(x) * fftn(y)) @curry def return_slice(x_data, cutoff): s = np.asarray(x_data.shape).astype(int) // 2 + 1 if x_data.ndim == 2: return x_data[(s[0] - cutoff):(s[0] + cutoff), (s[1] - cutoff):(s[1] + cutoff)] elif x_data.ndim == 3: return x_data[(s[0] - cutoff):(s[0] + cutoff),
import torch import numpy as np from toolz.curried import pipe, curry torch_rfft = curry(torch.rfft) torch_irfft = curry(torch.irfft) fftshift = curry(np.fft.fftshift) fabs = curry(np.absolute) def conjugate(x): """ returns conjugate of the complex torch tensor x """ y = torch.empty_like(x) y[..., 1] = x[..., 1] * -1 y[..., 0] = x[... , 0] return y @curry def mult(x1, x2): """ returns product of complex torch tensors x1 and x2 """ y = torch.empty_like(x1) y[..., 0] = x1[..., 0]*x2[..., 0] - x1[..., 1]*x2[..., 1] y[..., 1] = x1[..., 0]*x2[..., 1] + x1[..., 1]*x2[..., 0] return y
makerandom = Random @curry def array_from_tuple(data, shape, dtype): arr = np.zeros(shape, dtype=dtype) for slice_, value in data: arr[slice_] = value return arr def tlam(func, args): return func(*args) fft = curry(np.fft.fft) ifft = curry(np.fft.ifft) fftn = curry(np.fft.fftn) rfftn = curry(np.fft.rfftn) ifftn = curry(np.fft.ifftn) irfftn = curry(np.fft.irfftn) fftshift = curry(np.fft.fftshift) allclose = curry(np.allclose)
# coding: utf-8 """Provides a `duplicates_allclose` for finding duplicates in a DataFrame with tolerances. """ from toolz.curried import curry, compose import pandas as pd import numpy as np __all__ = ['duplicates_allclose'] pdapply = curry(pd.DataFrame.apply) # pylint: disable=invalid-name sort_values = curry(pd.DataFrame.sort_values) # pylint: disable=invalid-name pdall = curry(pd.DataFrame.all) # pylint: disable=invalid-name duplicated = curry(pd.DataFrame.duplicated) # pylint: disable=invalid-name diff = curry(pd.DataFrame.diff) # pylint: disable=invalid-name npappend = curry(np.append) # pylint: disable=invalid-name def sequence(*args): """Compose functions in order Args: args: the functions to compose Returns: composed functions >>> assert sequence(lambda x: x + 1, lambda x: x * 2)(3) == 8 """
# author: Apaar Shanker import numba import numpy as np from scipy.spatial import cKDTree from toolz.curried import pipe, curry from ase.neighborlist import neighbor_list fft = curry(np.fft.fft) ifft = curry(np.fft.ifft) fftn = curry(np.fft.fftn) ifftn = curry(np.fft.ifftn) fftshift = curry(np.fft.fftshift) ifftshift = curry(np.fft.ifftshift) conj = curry(np.conj) func = curry(lambda x, y: conj(x) * fftn(y)) fabs = curry(lambda x: np.absolute(x)) @curry def imfilter(x_data, f_data):
tm = "" return mapl(tm, arr) def format_n(arr): return format_sep("\n", arr) def format_comma(arr): return format_sep(", ", arr) toA = [mapl, zipl, filterl, rangel] for i in toA: setattr(opy, i.__name__, convert_pipe(_c.curry(i))) toAA = [ enumeratel, enumerate, sorted, join, join_n, join_comma, format_sep, format_n, format_comma ] for i in toAA: setattr(opy, i.__name__, _fun_.__getattr__(i.__name__)) setattr(opy, "format", _fun_.__getattr__("format_")) #### add Pipe functions from JulienPalard r = dir(_p) | _p.where(_fun_.re.match("^[^_]")) | _p.as_list() u = oo + mapl(lambda a: a.__name__, toA) + mapl(lambda a: a.__name__, toAA) + ["format"] for i in r:
def _curryMe(x): e = fn(x) if callable(e): return _c.curry(e) return fn
... ] """ with open(yaml_path) as f: data = yaml.load(f) return t.pipe( data, t.remove(_not_internal_link), flatmap(_flatten_sections), t.map(t.get('url')), list, ) flatmap = t.curry(lambda f, items: chain.from_iterable(map(f, items))) def _not_internal_link(entry): return not entry.get('url', '').startswith('/') def _flatten_sections(entry): sections = entry.get('sections', []) return [t.dissoc(entry, 'sections')] + sections def _sliding_three(entries): return ([(None, entries[0], entries[1])] + list(t.sliding_window(3, entries)) + [(entries[-2], entries[-1], None)])
from os.path import basename, splitext, join, abspath import re import subprocess import toolz.curried as z import argparse import yaml fst = z.operator.itemgetter(0) snd = z.operator.itemgetter(1) NOTEBOOK_DIR = '_notebooks' # NOTEBOOK_HTML_DIR = 'ipy_html' NOTEBOOK_HTML_DIR = '_includes' base = z.comp(fst, splitext, basename) htmlname = z.comp(z.curry(join, NOTEBOOK_HTML_DIR), '{}.html'.format, base) pat = re.compile(r'''^---\n (.+?)\n --- ''', re.VERBOSE + re.DOTALL) # # ^--- # print(pat.findall(txt)[0]) def post_todict(posttxt, ret_yaml=False): m = pat.findall(posttxt) if not m: if ret_yaml: return (None, None) return [txt] = m
LU_SP_SCHEMA = ShuangPinSchema("xiao lu") ZRM_SP_SCHEMA = ShuangPinSchema("zi ran ma") BINGJI_SP_SCHEMA = ShuangPinSchema("bing ji") @curry def for_each(proc, eles): if type(eles) is dict: for (k, v) in eles.items(): proc(k, v) else: for e in eles: proc(e) for_each = curry(for_each) def split_sy(pinyin: str) -> Tuple[str, str]: if pinyin == "sh": s = "sh" y = "i" elif pinyin.startswith("zh"): s = "zh" y = pinyin[2:] elif pinyin.startswith("ch"): s = "ch" y = pinyin[2:] elif pinyin.startswith("sh"): s = "sh" y = pinyin[2:]
""" Functional Programming stuffs """ from operator import and_ from typing import Callable, Any from toolz.curried import juxt, reduce, compose, curry def verify(*conditions) -> Callable[[Any], bool]: return compose( reduce(and_), juxt(*conditions) ) getattr = curry(getattr)
from funcy import count_by def functional(): return count_by(itemgetter('hour'), map(json.loads, filter(None, mapcat(lambda output: output.strip().split('\n'), map(lambda date: logs[date.strftime('%Y/%m/%d')], map(lambda days_ago: today - timedelta(days=days_ago), range(1, days_of_logs + 1))))))) print functional() from toolz.curried import map, filter, mapcat, curry count_by = curry(count_by) from bookends import _ def piped(): return (_| range(1, days_of_logs + 1) | map(lambda days_ago: today - timedelta(days=days_ago)) | map(lambda date: logs[date.strftime('%Y/%m/%d')]) | mapcat(lambda output: output.strip().split('\n')) | filter(None) | map(json.loads) | count_by(itemgetter('hour')) |_) print piped()
import numpy as np import dask.array as da from toolz.curried import curry, pipe from toolz.curried import map as fmap from scipy.ndimage.fourier import fourier_gaussian from ..func import ( fftn, ifftn, fftshift, ifftshift, zero_pad, sequence, map_blocks, ) conj = curry(np.conjugate) # pylint: disable=invalid-name fabs = curry(np.absolute) # pylint: disable=invalid-name @curry def _imfilter(x_data, f_data): """ to convolve f_data over x_data """ return pipe(f_data, ifftshift, fftn, lambda x: x * fftn(x_data), ifftn).real @curry def _gaussian_blur_filter(grain_size, domain_size): return pipe(
#!/usr/bin/env python """genelen.py: get the length of "gene" features from GAF annotation file. TCGA uses a specific gene annotation file for many of its projects, found at: https://tcga-data.nci.nih.gov/tcgafiles/ftp_auth/distro_ftpusers/anonymous/other/GAF/GAF_bundle/outputs/TCGA.Sept2010.09202010.gaf We needed to parse out the gene lengths from that file. This script extracts gene symbol, gene ID, and gene length from it, only for gene features. (Not, e.g., for SNPs.) """ from toolz import curried as tz import pandas as pd spliteach = tz.map(tz.curry(str.split)(sep='\t')) def range2len(tup): _id, coords = tup symbol, _id = _id.split('|')[:2] length = int(coords.split('-')[-1]) return (symbol, int(_id), length) def gene_length_df(filename): """Grab Gene Symbol, Gene ID, and Gene Length from a GAF file. Parameters ---------- filename : string Path to a Gene Annotation Format (GAF) file.
) from .. import canvas from ..canvas.course import Course log = logging.getLogger(__name__) log.addHandler(logging.NullHandler()) def generator(): from ..__init__ import __version__ return f'coursework-{__version__}' resource_filename = compose( Path, curry(_resource_filename)(__name__), ) def templates_path(path: str = '.'): return Path(resource_filename('j2'), path) j2_path = templates_path def css_path(path: str = '.'): return Path(resource_filename('css'), path) def js_path(path: str = '.'):
from operator import add as default_add, ge as default_ge, le as default_le from toolz.curried import flip, curry, partial __all__ = [ "str_split", "str_startswith", "str_strip", "add", "str_find", "ge", "le", "str_replace", ] str_split = flip(str.split) str_startswith = flip(str.startswith) str_strip = flip(str.strip) str_find = flip(str.find) add = curry(default_add) ge = curry(default_ge) le = curry(default_le) def str_replace(before, after): return lambda s: str.replace(s, before, after)
if __name__ == '__main__': client = ClientBuilder().default() planets = pipe( client.planet.list().iter(), curried.filter(lambda x: x.GetClimate().Get() == "temperate"), list) # print(json.dumps(planets, default=lambda x: x.Serializable())) people = list(map(lambda x: x["eye_color"], client.people.list().iter())) people = list(pipe(client.people.list().iter(), # filter(lambda x: x.GetEyeColor().Get() == "brown" ) )) # print(json.dumps(people,default=lambda x: x.Serializable())) # # writer = csv.writer(stdout) # first, people = toolz.peek(people) # writer.writerow(pipe(first.keys())) # writer.writerows( # list(pipe(people, # gdict # ))) a = Counter(4) map = curried.curry(map) test = curried.curry(test) list(pipe(a, map(lambda x: print(x))))