def build_csv(objects, headers, file_name):
with open(file_name, "w") as f:
writer = csv.writer(f)
writer.writerow(headers)
for obj in objects:
row = compose(list, map(flip(get)(obj)))(headers)
writer.writerow(row)
def get_bag(build_dir: Path, base_dtype: str = "xml") -> db.Bag:
"""possible to do any text pre-processing here"""
dtype_path = get_datapaths(build_dir).get(base_dtype)
schema = get_schema(build_dir)
filepaths = dtype_path.glob(f"**/*.{base_dtype}")
_update_authors = flip(update_in(func=flatten_authors), ("authors", ))
_update_keywords = lambda d: pipe(
d,
*[
update_in(func=split_keywords, keys=[kw])
for kw in (col for col in d.keys() if col.endswith("_keywords"))
],
)
return (db.from_sequence(filepaths).map(partial(
load_xml, schema)).map(_update_authors).map(_update_keywords))
arr: the array
Returns:
Not a new array, but the same array updated
>>> a = np.arange(6).reshape((2, 3))
>>> assign(1, (slice(None), 2), a)
array([[0, 1, 1],
[3, 4, 1]])
"""
arr[index] = value
return arr
npresize = curry(flip(np.resize)) # pylint: disable=invalid-name
@curry
def zero_pad(arr, shape, chunks):
"""Zero pad an array with zeros
Args:
arr: the array to pad
shape: the shape of the new array
chunks: how to rechunk the new array
Returns:
the new padded version of the array
>>> print(
import PIL.Image
from toolz.curried import map, pipe, compose, get, do, curry, count, pluck, juxt, flip
import pandas
import skimage
import skimage.measure
import skimage.filters
import skimage.morphology
import sys
import json
import pickle
fcompose = lambda *args: compose(*args[::-1])
## Binary closing to reveal the Pearlite Phase
closing = curry(flip(skimage.morphology.closing))
remove_small_holes = curry(skimage.morphology.remove_small_holes)
reveal_pearlite = fcompose(closing(skimage.morphology.square(5)),
remove_small_holes(min_size=1000))
def reveal(data):
data['pearlite_image'] = reveal_pearlite(data['clean_image'])
return data
if __name__ == '__main__':
filename = sys.argv[1]
filename_cleaned = filename.split("/")[-1].split(".")[0]
filename_cleaned = "-".join(filename_cleaned.split("-")[0:-1])
import typing as t
from datetime import datetime
from toolz.curried import partial, flip, valmap, compose
import snug
from . import types
xml = snug.xml
registry = snug.load.PrimitiveRegistry({
bool: dict(true=True, false=False).__getitem__,
datetime: partial(flip(datetime.strptime), '%Y-%m-%dT%H:%M:%S%z'),
str: str.strip,
**{
c: c for c in [
int,
float,
types.Journey.Status,
types.Journey.Component.Status
]
}
}) | snug.load.GenericRegistry({
t.List: snug.load.list_loader,
}) | snug.load.get_optional_loader | snug.load.DataclassRegistry({
types.Station: {**valmap(xml.textgetter, {
'code': 'Code',
'type': 'Type',
'country': 'Land',
from operator import add as default_add, ge as default_ge, le as default_le
from toolz.curried import flip, curry, partial
__all__ = [
"str_split",
"str_startswith",
"str_strip",
"add",
"str_find",
"ge",
"le",
"str_replace",
]
str_split = flip(str.split)
str_startswith = flip(str.startswith)
str_strip = flip(str.strip)
str_find = flip(str.find)
add = curry(default_add)
ge = curry(default_ge)
le = curry(default_le)
def str_replace(before, after):
return lambda s: str.replace(s, before, after)