def append_urlX_to_X(target, source, **kwargs):
with closing(urlopen(source.url, timeout=kwargs.pop('timeout', None))) as r:
chunk_size = 16 * source.chunk_size
# 11.12.18 harleen - part2 - hack to enable writing to ftp server
# original
# with open(target.path, 'wb') as fp:
# for chunk in iter(curry(r.read, chunk_size), b''):
# fp.write(chunk)
# return target
# changes begin
f = target.path
if f.split(':')[0].lower() == 'ftp':
import io
from io import StringIO
from ftplib import FTP
from pyparsing import Combine, Suppress, Word, printables, alphas, ZeroOrMore, alphanums, Group, delimitedList, nums
ftp_expression = Suppress('ftp://') + Word(alphanums) + ZeroOrMore(Suppress(':')) + Word(alphanums) + ZeroOrMore(Suppress('@')) + Word(alphanums) + ZeroOrMore(Suppress(':')) + Word(nums) + ZeroOrMore(Suppress('/')) + Word(printables)
username, password, ip, port, file_loc = ftp_expression.parseString(f)
ftp = FTP(ip, username, password)
# buffer = StringIO()
# df.to_csv(buffer, index=False, header=has_header, sep=sep, encoding=encoding)
# text = buffer.getvalue()
# bio = io.BytesIO(str.encode(text))
# ftp.storbinary('STOR '+file_loc, bio)
for chunk in iter(curry(r.read, chunk_size), b''):
ftp.storbinary('APPE ' + file_loc, io.BytesIO(chunk))
return target
else:
with open(target.path, 'wb') as fp:
for chunk in iter(curry(r.read, chunk_size), b''):
fp.write(chunk)
return target
def main():
"""Main method."""
# create a player named tuple
Player = namedtuple('Player', ['first', 'last', 'number', 'team', 'city'])
# create some player named tuples
m_j = Player(first='Michael', last='Jordan', number='23', team='Bulls', city='Chicago')
k_b = Player(first='Kobe', last='Bryant', number='24', team='Lakers', city='Los Angeles')
l_b = Player(first='LeBron', last='James', number='23', team='Cavaliers', city='Cleveland')
k_p = Player(first='Kristaps', last='Porzingis', number='6', team='Knicks', city='New York')
k_d = Player(first='Kevin', last='Durant', number='35', team='Warriors', city='Oakland')
# store the players in tuple
players = (m_j, k_b, l_b, k_p, k_d)
# filter
two_three = filter(lambda x: x.number == '23', players)
print(tuple(two_three)) # => (Player(first='Michael', last='Jordan', number='23', team='Bulls',
# city='Chicago'), Player(first='LeBron', last='James', number='23',
# team='Cavaliers', city='Cleveland'))
# map
result = map(lambda x: ''.join([x.first, ' ', x.last]), players)
print(tuple(result)) # => ('Michael Jordan', 'Kobe Bryant', 'LeBron James', 'Kristaps Porzingis', 'Kevin Durant')
# reduce
result = reduce(lambda x, y: x + y, [1, 2, 3], 0)
print(result) # => 6
# compose
nums = [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]
c_greater = curry(greater)
greater_zero = c_greater(y_val=0)
result = compose(filter(greater_zero), filter(iseven))(nums)
print(tuple(result)) # => (2, 4)
def many(a, b, binop=None, reduction=None, **kwargs):
"""
Apply binary operator to pairwise to sequences, then reduce.
>>> many([1, 2, 3], [10, 20, 30], mul, sum) # dot product
140
"""
return reduction(map(curry(binop, **kwargs), a, b))
def append_urlX_to_X(target, source, **kwargs):
with closing(urlopen(source.url)) as r:
chunk_size = 16 * source.chunk_size
with open(target.path, 'wb') as fp:
for chunk in iter(curry(r.read, chunk_size), ''):
fp.write(chunk)
return target
def append_urlX_to_X(target, source, **kwargs):
with closing(urlopen(source.url)) as r:
chunk_size = 16 * source.chunk_size
with open(target.path, 'wb') as fp:
for chunk in iter(curry(r.read, chunk_size), ''):
fp.write(chunk)
return target
def transpose(a, axes=None):
axes = axes or tuple(range(a.ndim))[::-1]
return atop(curry(np.transpose, axes=axes),
next(names),
axes,
a,
tuple(range(a.ndim)),
dtype=a._dtype)
def many(a, b, binop=None, reduction=None, **kwargs):
"""
Apply binary operator to pairwise to sequences, then reduce.
>>> many([1, 2, 3], [10, 20, 30], mul, sum) # dot product
140
"""
return reduction(map(curry(binop, **kwargs), a, b))
def append_urlX_to_X(target, source, **kwargs):
with closing(urlopen(source.url, timeout=kwargs.pop("timeout", None))) as r:
chunk_size = 16 * source.chunk_size
with open(target.path, "wb") as fp:
for chunk in iter(curry(r.read, chunk_size), b""):
fp.write(chunk)
return target
def curry(*args, **kwargs):
"""Curry a callable function
Deprecated: use toolz.curried.curry() instead.
"""
warnings.warn(
"alt.curry() is deprecated, and will be removed in a future release. "
"Use toolz.curried.curry() instead.",
AltairDeprecationWarning,
)
return curried.curry(*args, **kwargs)
def _func(coords, domain=None): # pylint: disable=unused-argument
return pipe(
(x_points, y_points, z_points),
enumerate,
map_(lambda x: flag_it(x[1], coords, x[0])),
list,
curry(fold)(lambda x, y: x & y),
lambda x: (x & (coords[:, 0] < (max_x - eps(coords))))
if max_x is not None else x,
np.where,
first,
)
class placeholder2Bis(placeholderI):
MY_FUNC = curried.curry(curryX)(cls=Pipe2)
MY_PIPE = Pipe2
def special__ror__np(self, argu):
return self.__ror__(argu, okO=True)
def __ror__(self, other, okO=False):
if iterable(other) and type(other) != "str":
other = Pipe2.collection(other)
return placeholder2(
func=lambda x, self=self, other=other: x.__ror__(other)
if isinstance(other, Pipe2) and okO else self.func(other, x))
def sample_url_line_delimited(data, lines=5, encoding="utf-8", timeout=None):
"""Get a size `length` sample from an URL CSV or URL line-delimited JSON.
Parameters
----------
data : URL(CSV)
A hosted CSV
lines : int, optional, default ``5``
Number of lines to read into memory
"""
with closing(urlopen(data.url, timeout=timeout)) as r:
raw = pipe(
r, take(lines), map(bytes.strip), curry(codecs.iterdecode, encoding=encoding), b"\n".decode(encoding).join
)
with tmpfile(data.filename) as fn:
with codecs.open(fn, "wb", encoding=encoding) as f:
f.write(raw)
yield fn
def sample_url_line_delimited(data, lines=5, encoding='utf-8'):
"""Get a size `length` sample from an URL CSV or URL line-delimited JSON.
Parameters
----------
data : URL(CSV)
A hosted CSV
lines : int, optional, default ``5``
Number of lines to read into memory
"""
with closing(urlopen(data.url)) as r:
raw = pipe(r, take(lines), map(bytes.strip),
curry(codecs.iterdecode, encoding=encoding),
b'\n'.decode(encoding).join)
with tmpfile(data.filename) as fn:
with codecs.open(fn, 'wb', encoding=encoding) as f:
f.write(raw)
yield fn
import numpy as np
from toolz.curried import curry, pipe
try:
import edt
transform_edt = curry(edt.edt)(black_border=False)
except ImportError:
import scipy.ndimage.morphology as morph
transform_edt = morph.distance_transform_edt
print("you can install edt for speed-up as - ")
print("pip install edt")
pass
try:
import torch
torch_rfft = curry(torch.rfft) # pylint: disable=invalid-name
torch_irfft = curry(torch.irfft) # pylint: disable=invalid-name
def conjugate(x):
"""
returns the conjugate of a complex torch tensor
"""
y = torch.empty_like(x)
y[..., 1] = x[..., 1] * -1
y[..., 0] = x[..., 0]
return y
@curry
def mult(x1, x2):
"""
returns the product of complext torch tensors x1 and x2
import PIL.Image
from toolz.curried import map, pipe, compose, get, do, curry, count, pluck, juxt, flip
import pandas
import skimage
import skimage.measure
import skimage.filters
import skimage.morphology
import sys
import json
import pickle
fcompose = lambda *args: compose(*args[::-1])
## Binary closing to reveal the Pearlite Phase
closing = curry(flip(skimage.morphology.closing))
remove_small_holes = curry(skimage.morphology.remove_small_holes)
reveal_pearlite = fcompose(closing(skimage.morphology.square(5)),
remove_small_holes(min_size=1000))
def reveal(data):
data['pearlite_image'] = reveal_pearlite(data['clean_image'])
return data
if __name__ == '__main__':
filename = sys.argv[1]
filename_cleaned = filename.split("/")[-1].split(".")[0]
filename_cleaned = "-".join(filename_cleaned.split("-")[0:-1])
TCGA archives contain one file per sample. This script aggregates them into
a single ngenes x nsamples expression counts matrix.
This works on the data from "Genomic Classification of Cutaneous Melanoma",
aggregated here:
https://tcga-data.nci.nih.gov/docs/publications/skcm_2015/
And with direct link to the RNA expression Level 3 archive:
https://tcga-data.nci.nih.gov/docs/publications/skcm_2015/unc.edu_SKCM.IlluminaHiSeq_RNASeqV2.Level_3.1.10.0.tar.gz
"""
import numpy as np
from toolz import curried as tz
spliteach = tz.compose(tz.map, tz.curry(str.split))
tee = tz.compose(tz.map, tz.do)
arrayfromiter = tz.curry(np.fromiter)(dtype=float)
def getid_counts(lines, kind='symbol'):
if kind == 'symbol':
kind = 0
else:
kind = 1
ids = []
res = tz.pipe(lines, spliteach(sep='\t'), tz.pluck(0), spliteach(sep='|'),
tz.pluck(kind), tee(ids.append), tz.frequencies)
return ids, res
import numba
import scipy
import numpy as np
from .helpers import *
from scipy.spatial import cKDTree
from toolz.curried import pipe, curry
from scipy.ndimage import measurements
from scipy.sparse.csgraph import dijkstra
from skimage.morphology import remove_small_objects
from skimage.morphology import skeletonize_3d as sklz
erasure = curry(remove_small_objects)
@curry
def accessibleRegion(S, atomH, r_h, overlap=0.01):
vol = np.count_nonzero(atomH)
S_mod = imfilter(x_data=(S < 1e-6),
f_data=padder(atomH, np.array(S.shape))) / vol
S_mod = S_mod < overlap
return S_mod
@curry
def return_labelled(x_data):
S_l, n_count = measurements.label(x_data)
top = list(np.unique(S_l[:, :, 0]))[1:]
bot = list(np.unique(S_l[:, :, -1]))[1:]
m = list(set(top).intersection(bot))
return S_l, n_count, m
import scipy.ndimage
import pytesseract
import PIL.Image
from toolz.curried import map, pipe, compose, get, do, curry, count, pluck, juxt, flip
import pandas
import skimage
import skimage.measure
import skimage.filters
import skimage.morphology
import json
import sys
import pickle
## Binary closing to reveal the Pearlite Phase
remove_small_holes = curry(skimage.morphology.remove_small_holes)
def clean(data):
data['clean_image'] = ~remove_small_holes(~data['threshold_image'], data['min_size'])
return data
if __name__ == '__main__':
filename = sys.argv[1]
filename_cleaned = filename.split("/")[-1].split(".")[0]
filename_cleaned = "-".join(filename_cleaned.split("-")[0:-1])
data = None
with open(filename, "r") as intermediate:
data = pickle.load(intermediate)
result = clean(data)
import numpy as np
from toolz.curried import curry
from toolz.curried import map
from toolz.curried import pipe
fft = curry(np.fft.fft) # pylint: disable=invalid-name
ifft = curry(np.fft.ifft) # pylint: disable=invalid-name
fftn = curry(np.fft.fftn) # pylint: disable=invalid-name
ifftn = curry(np.fft.ifftn) # pylint: disable=invalid-name
fftshift = curry(np.fft.fftshift) # pylint: disable=invalid-name
conj = curry(np.conj)
func = curry(lambda x, y: conj(x) * fftn(y))
@curry
def return_slice(x_data, cutoff):
s = np.asarray(x_data.shape).astype(int) // 2 + 1
if x_data.ndim == 2:
return x_data[(s[0] - cutoff):(s[0] + cutoff),
(s[1] - cutoff):(s[1] + cutoff)]
elif x_data.ndim == 3:
return x_data[(s[0] - cutoff):(s[0] + cutoff),
import torch
import numpy as np
from toolz.curried import pipe, curry
torch_rfft = curry(torch.rfft)
torch_irfft = curry(torch.irfft)
fftshift = curry(np.fft.fftshift)
fabs = curry(np.absolute)
def conjugate(x):
"""
returns conjugate of the complex torch tensor x
"""
y = torch.empty_like(x)
y[..., 1] = x[..., 1] * -1
y[..., 0] = x[... , 0]
return y
@curry
def mult(x1, x2):
"""
returns product of complex torch tensors x1 and x2
"""
y = torch.empty_like(x1)
y[..., 0] = x1[..., 0]*x2[..., 0] - x1[..., 1]*x2[..., 1]
y[..., 1] = x1[..., 0]*x2[..., 1] + x1[..., 1]*x2[..., 0]
return y
makerandom = Random
@curry
def array_from_tuple(data, shape, dtype):
arr = np.zeros(shape, dtype=dtype)
for slice_, value in data:
arr[slice_] = value
return arr
def tlam(func, args):
return func(*args)
fft = curry(np.fft.fft)
ifft = curry(np.fft.ifft)
fftn = curry(np.fft.fftn)
rfftn = curry(np.fft.rfftn)
ifftn = curry(np.fft.ifftn)
irfftn = curry(np.fft.irfftn)
fftshift = curry(np.fft.fftshift)
allclose = curry(np.allclose)
# coding: utf-8
"""Provides a `duplicates_allclose` for finding duplicates in a
DataFrame with tolerances.
"""
from toolz.curried import curry, compose
import pandas as pd
import numpy as np
__all__ = ['duplicates_allclose']
pdapply = curry(pd.DataFrame.apply) # pylint: disable=invalid-name
sort_values = curry(pd.DataFrame.sort_values) # pylint: disable=invalid-name
pdall = curry(pd.DataFrame.all) # pylint: disable=invalid-name
duplicated = curry(pd.DataFrame.duplicated) # pylint: disable=invalid-name
diff = curry(pd.DataFrame.diff) # pylint: disable=invalid-name
npappend = curry(np.append) # pylint: disable=invalid-name
def sequence(*args):
"""Compose functions in order
Args:
args: the functions to compose
Returns:
composed functions
>>> assert sequence(lambda x: x + 1, lambda x: x * 2)(3) == 8
"""
# author: Apaar Shanker import numba import numpy as np from scipy.spatial import cKDTree from toolz.curried import pipe, curry from ase.neighborlist import neighbor_list fft = curry(np.fft.fft) ifft = curry(np.fft.ifft) fftn = curry(np.fft.fftn) ifftn = curry(np.fft.ifftn) fftshift = curry(np.fft.fftshift) ifftshift = curry(np.fft.ifftshift) conj = curry(np.conj) func = curry(lambda x, y: conj(x) * fftn(y)) fabs = curry(lambda x: np.absolute(x)) @curry def imfilter(x_data, f_data):
tm = ""
return mapl(tm, arr)
def format_n(arr):
return format_sep("\n", arr)
def format_comma(arr):
return format_sep(", ", arr)
toA = [mapl, zipl, filterl, rangel]
for i in toA:
setattr(opy, i.__name__, convert_pipe(_c.curry(i)))
toAA = [
enumeratel, enumerate, sorted, join, join_n, join_comma, format_sep,
format_n, format_comma
]
for i in toAA:
setattr(opy, i.__name__, _fun_.__getattr__(i.__name__))
setattr(opy, "format", _fun_.__getattr__("format_"))
#### add Pipe functions from JulienPalard
r = dir(_p) | _p.where(_fun_.re.match("^[^_]")) | _p.as_list()
u = oo + mapl(lambda a: a.__name__, toA) + mapl(lambda a: a.__name__,
toAA) + ["format"]
for i in r:
def _curryMe(x):
e = fn(x)
if callable(e):
return _c.curry(e)
return fn
...
]
"""
with open(yaml_path) as f:
data = yaml.load(f)
return t.pipe(
data,
t.remove(_not_internal_link),
flatmap(_flatten_sections),
t.map(t.get('url')),
list,
)
flatmap = t.curry(lambda f, items: chain.from_iterable(map(f, items)))
def _not_internal_link(entry):
return not entry.get('url', '').startswith('/')
def _flatten_sections(entry):
sections = entry.get('sections', [])
return [t.dissoc(entry, 'sections')] + sections
def _sliding_three(entries):
return ([(None, entries[0], entries[1])] +
list(t.sliding_window(3, entries)) +
[(entries[-2], entries[-1], None)])
from os.path import basename, splitext, join, abspath
import re
import subprocess
import toolz.curried as z
import argparse
import yaml
fst = z.operator.itemgetter(0)
snd = z.operator.itemgetter(1)
NOTEBOOK_DIR = '_notebooks'
# NOTEBOOK_HTML_DIR = 'ipy_html'
NOTEBOOK_HTML_DIR = '_includes'
base = z.comp(fst, splitext, basename)
htmlname = z.comp(z.curry(join, NOTEBOOK_HTML_DIR), '{}.html'.format, base)
pat = re.compile(r'''^---\n
(.+?)\n
---
''', re.VERBOSE + re.DOTALL) # # ^---
# print(pat.findall(txt)[0])
def post_todict(posttxt, ret_yaml=False):
m = pat.findall(posttxt)
if not m:
if ret_yaml:
return (None, None)
return
[txt] = m
def transpose(a, axes=None):
axes = axes or tuple(range(a.ndim))[::-1]
return atop(curry(np.transpose, axes=axes),
next(names), axes,
a, tuple(range(a.ndim)), dtype=a._dtype)
LU_SP_SCHEMA = ShuangPinSchema("xiao lu")
ZRM_SP_SCHEMA = ShuangPinSchema("zi ran ma")
BINGJI_SP_SCHEMA = ShuangPinSchema("bing ji")
@curry
def for_each(proc, eles):
if type(eles) is dict:
for (k, v) in eles.items():
proc(k, v)
else:
for e in eles:
proc(e)
for_each = curry(for_each)
def split_sy(pinyin: str) -> Tuple[str, str]:
if pinyin == "sh":
s = "sh"
y = "i"
elif pinyin.startswith("zh"):
s = "zh"
y = pinyin[2:]
elif pinyin.startswith("ch"):
s = "ch"
y = pinyin[2:]
elif pinyin.startswith("sh"):
s = "sh"
y = pinyin[2:]
"""
Functional Programming stuffs
"""
from operator import and_
from typing import Callable, Any
from toolz.curried import juxt, reduce, compose, curry
def verify(*conditions) -> Callable[[Any], bool]:
return compose(
reduce(and_),
juxt(*conditions)
)
getattr = curry(getattr)
from funcy import count_by
def functional():
return count_by(itemgetter('hour'),
map(json.loads,
filter(None,
mapcat(lambda output: output.strip().split('\n'),
map(lambda date: logs[date.strftime('%Y/%m/%d')],
map(lambda days_ago: today - timedelta(days=days_ago),
range(1, days_of_logs + 1)))))))
print functional()
from toolz.curried import map, filter, mapcat, curry
count_by = curry(count_by)
from bookends import _
def piped():
return (_| range(1, days_of_logs + 1)
| map(lambda days_ago: today - timedelta(days=days_ago))
| map(lambda date: logs[date.strftime('%Y/%m/%d')])
| mapcat(lambda output: output.strip().split('\n'))
| filter(None)
| map(json.loads)
| count_by(itemgetter('hour'))
|_)
print piped()
import numpy as np
import dask.array as da
from toolz.curried import curry, pipe
from toolz.curried import map as fmap
from scipy.ndimage.fourier import fourier_gaussian
from ..func import (
fftn,
ifftn,
fftshift,
ifftshift,
zero_pad,
sequence,
map_blocks,
)
conj = curry(np.conjugate) # pylint: disable=invalid-name
fabs = curry(np.absolute) # pylint: disable=invalid-name
@curry
def _imfilter(x_data, f_data):
"""
to convolve f_data over x_data
"""
return pipe(f_data, ifftshift, fftn, lambda x: x * fftn(x_data),
ifftn).real
@curry
def _gaussian_blur_filter(grain_size, domain_size):
return pipe(
#!/usr/bin/env python
"""genelen.py: get the length of "gene" features from GAF annotation file.
TCGA uses a specific gene annotation file for many of its projects, found at:
https://tcga-data.nci.nih.gov/tcgafiles/ftp_auth/distro_ftpusers/anonymous/other/GAF/GAF_bundle/outputs/TCGA.Sept2010.09202010.gaf
We needed to parse out the gene lengths from that file. This script extracts
gene symbol, gene ID, and gene length from it, only for gene features. (Not,
e.g., for SNPs.)
"""
from toolz import curried as tz
import pandas as pd
spliteach = tz.map(tz.curry(str.split)(sep='\t'))
def range2len(tup):
_id, coords = tup
symbol, _id = _id.split('|')[:2]
length = int(coords.split('-')[-1])
return (symbol, int(_id), length)
def gene_length_df(filename):
"""Grab Gene Symbol, Gene ID, and Gene Length from a GAF file.
Parameters
----------
filename : string
Path to a Gene Annotation Format (GAF) file.
)
from .. import canvas
from ..canvas.course import Course
log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())
def generator():
from ..__init__ import __version__
return f'coursework-{__version__}'
resource_filename = compose(
Path,
curry(_resource_filename)(__name__),
)
def templates_path(path: str = '.'):
return Path(resource_filename('j2'), path)
j2_path = templates_path
def css_path(path: str = '.'):
return Path(resource_filename('css'), path)
def js_path(path: str = '.'):
from operator import add as default_add, ge as default_ge, le as default_le
from toolz.curried import flip, curry, partial
__all__ = [
"str_split",
"str_startswith",
"str_strip",
"add",
"str_find",
"ge",
"le",
"str_replace",
]
str_split = flip(str.split)
str_startswith = flip(str.startswith)
str_strip = flip(str.strip)
str_find = flip(str.find)
add = curry(default_add)
ge = curry(default_ge)
le = curry(default_le)
def str_replace(before, after):
return lambda s: str.replace(s, before, after)
from os.path import basename, splitext, join, abspath
import re
import subprocess
import toolz.curried as z
import argparse
import yaml
fst = z.operator.itemgetter(0)
snd = z.operator.itemgetter(1)
NOTEBOOK_DIR = '_notebooks'
# NOTEBOOK_HTML_DIR = 'ipy_html'
NOTEBOOK_HTML_DIR = '_includes'
base = z.comp(fst, splitext, basename)
htmlname = z.comp(z.curry(join, NOTEBOOK_HTML_DIR), '{}.html'.format, base)
pat = re.compile(r'''^---\n
(.+?)\n
---
''', re.VERBOSE + re.DOTALL) # # ^---
# print(pat.findall(txt)[0])
def post_todict(posttxt, ret_yaml=False):
m = pat.findall(posttxt)
if not m:
if ret_yaml:
return (None, None)
return
[txt] = m
if __name__ == '__main__':
client = ClientBuilder().default()
planets = pipe(
client.planet.list().iter(),
curried.filter(lambda x: x.GetClimate().Get() == "temperate"),
list)
# print(json.dumps(planets, default=lambda x: x.Serializable()))
people = list(map(lambda x: x["eye_color"], client.people.list().iter()))
people = list(pipe(client.people.list().iter(),
# filter(lambda x: x.GetEyeColor().Get() == "brown" )
))
# print(json.dumps(people,default=lambda x: x.Serializable()))
#
# writer = csv.writer(stdout)
# first, people = toolz.peek(people)
# writer.writerow(pipe(first.keys()))
# writer.writerows(
# list(pipe(people,
# gdict
# )))
a = Counter(4)
map = curried.curry(map)
test = curried.curry(test)
list(pipe(a, map(lambda x: print(x))))
