def pickRegex(keys: Sequence[str], doc: Dict) -> Dict:
if not isinstance(doc, dict):
return None
regex_keys = _.map(__regTest__, keys)
return pick(
_.filter(lambda k: sum(_.map(lambda reg: reg(k), regex_keys)) > 0,
doc.keys()), doc)
def generateHist(leaf: Dict, names: List) -> None:
x_keys = PIPE(leaf.keys())
y_cnt = _.concat(*_.map(lambda i: [i] * leaf[str(i)]['cnt'], x_keys))
n, bins, patches = plt.hist(y_cnt,
rwidth=0.8,
log=True,
range=(-5, 5),
color=Blue.A700)
ax = plt.gca()
ax.set_xlabel("val2")
ax.set_ylabel("log Frequency")
plt.savefig('_'.join(names) + ".svg")
def apply(self, arr: List[float], yValueIndex: int = None):
"""
arr = [x1, x2, x3, ...]
"""
if self.__data__ is None:
self.initialize()
if isinstance(arr, numbers.Number):
arr = [arr]
inds = _.map(\
lambda val, ind: self.func[ind](self.__min__[ind], self.__max__[ind], self.__bin_width__[ind], self.__bin_number__[ind]-1, val),\
arr,\
range(len(arr))\
)
# Points outside the set limits are ignored.
if None in inds:
return self
# CNT only for now
self.__data__.set(inds, self.__data__.get(inds) + 1)
return self
def generateBarChart(leaf: Dict, names: List) -> None:
x_keys = PIPE(leaf.keys())
y_cnt = _.map(lambda i: math.log10(leaf[str(i)]['cnt']), x_keys)
plt.bar(x_keys, y_cnt, width=0.1)
ax = plt.gca()
ax.set_xlabel("log val2")
ax.set_ylabel("log Frequency")
ax.set_xlim([-5, 5])
plt.savefig('_'.join(names) + ".svg")
def streamFromFiles(
files: List[str]) -> Generator[List[List[float]], None, None]:
streamList = _.map(lambda f: fileStream(f), files)
for stream in streamList:
while stream:
s = next(stream, None)
if s == None:
break
else:
yield s
return None
def pick(keys: Sequence[str], doc: Dict) -> Dict:
"""Picks the provided keys from the dictionary.
Example
pick(["a", "b"], {"a": 1, "b": 2, "c": 3}) # => {"a": 1, "b": 2}
"""
if not _.isIterable(keys):
return None
if not isinstance(doc, dict):
return None
return {k: doc.get(k, None) for k in keys}
def apply(self, arr: List[float], yValueIndex: int = None):
"""
arr = [x1, x2, x3, ...]
"""
if self.__data__ is None:
self.initialize()
if isinstance(arr, numbers.Number):
arr = [arr]
inds = tuple(
_.map(
lambda val, ind: self.func[ind]
(self.__min__[ind], self.__max__[ind], self.__bin_width__[ind],
self.__bin_number__[ind] - 1, val), arr, range(len(arr))))
# Points outside the set limits are ignored.
if None in inds:
return self
agg = self.__data__[inds]
if agg is None:
agg = self.__data__[inds] = {
'cnt': 0,
'sum': 0,
'sum2': 0,
'min': 0,
'max': 0
}
agg['cnt'] += 1
y_val_ind = yValueIndex or len(arr) - 1
y_val = arr[y_val_ind]
agg['sum'] += y_val
agg['sum2'] += y_val * y_val
if agg['min'] > y_val:
agg['min'] = y_val
if agg['max'] < y_val:
agg['max'] = y_val
return self
def __init__(self, dims: List[int], typecode: str = 'd'):
length = _.reduce(lambda agg, i: agg * i, dims, 1)
sparseLS = Map().mutate()
dimsLen = len(dims)
self.__buffer__ = sparseLS
self.__dims__ = dims
if dimsLen == 1:
self.__calcPos__ = calcPos1
elif dimsLen == 2:
self.__calcPos__ = calcPos2
elif dimsLen == 3:
self.__calcPos__ = calcPos3
def __init__(self, dims: List[int], typecode: str = 'd', silent=False):
length = _.reduce(lambda agg, i: agg * i, dims, 1)
ls = array(typecode, map(lambda i: 0, range(length)))
dimsLen = len(dims)
self.__buffer__ = ls
if silent is False:
print('\nContainer size in memory: {0} KB'.format(
len(ls) * ls.itemsize / 1024))
self.__dims__ = dims
if dimsLen == 1:
self.__calcPos__ = calcPos1
elif dimsLen == 2:
self.__calcPos__ = calcPos2
elif dimsLen == 3:
self.__calcPos__ = calcPos3
def fileStream(
file_name: str,
bathcSize: int = 700
) -> Generator[List[List[float]], None, List[List[float]]]:
with open(file_name, newline="\n") as file:
data = csv.reader(file, delimiter=",")
headers = next(data)
# yield headers;
j = 0
queue = []
for i in data:
if j < bathcSize:
queue.append(_.map(float, i[1:3]))
j += 1
else:
j = 0
yield queue
queue = []
if len(queue) > 0:
return queue
def writeImg(colorData):
# plot the count matrix as an image
f = imshow(colorData)
axis('off')
savefig('figure .png', dpi=500, format="png", transparent=True)
stream = streamFromFiles(FILES)
ind = 0
startTime = time.monotonic()
batch = next(stream, None)
while batch:
shader.applyOnBatches(batch)
batch = next(stream, None)
CNT_MATRIX = shader.getAgg('cnt')
MAX_CNT = max(max(CNT_MATRIX))
print("Maximun count of Uber rides in NYC: {0}".format(MAX_CNT))
# total count
TOTAL_CNT = sum(_.map(sum, CNT_MATRIX))
print("Total number of Uber rides in NYC: {0}".format(TOTAL_CNT))
writeImg(_.map(lambda i: color_map(i), CNT_MATRIX))
endTime = time.monotonic() - startTime
print(f" Processing time:\t\t\t{endTime:.4f}s")
if b is not None:
shader.applyOnBatches(b)
if ind > 499 and ind % 500 == 0:
snapshots.append(getShapshot(shader, color_map))
ind += 1
else:
break
CNT_MATRIX = shader.getAgg('cnt')
MAX_CNT = max(max(CNT_MATRIX))
print("Maximun count of Uber rides in NYC: {0}".format(MAX_CNT))
# total count
TOTAL_CNT = sum(_.map(sum, CNT_MATRIX))
print("Total number of Uber rides in NYC: {0}".format(TOTAL_CNT))
for ind, s in enumerate(snapshots):
if ind < 10:
index = '0{0}'.format(ind)
else:
index = ind
writeSnapshot(s, index)
endTime = time.monotonic() - startTime
print(f" Processing time:\t\t\t{endTime:.4f}s")
# see https://eli.thegreenplace.net/2016/drawing-animated-gifs-with-matplotlib/
os.system('convert -delay 60 -loop 0 figure-*.jpg figure.gif')
def getValues(doc, keys):
if not _.isIterable(keys):
return None
return _.map(lambda k: get(doc, k), keys)
from funkpy import Collection as _
from funkpy.Monads import Functor, Monad, Option
from funkpy.utils import curry, compose
l = [1, 2, 3]
# concatenate n lists
_.concat(l, [4, 5, 6], [7, 8, 9], 10) # => [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# append a list
_.push(l, [4, 5, 6]) # => [1, 2, 3, [4, 5, 6]]
addOne = curry(_.strictMap)(lambda i: i+1)
print(addOne([1, 2, 3])) # => [2, 3, 4]
addTwo = compose(addOne, addOne)
print(addTwo([1, 2, 3])) # => [3, 4, 5]
# How to construct this functor thing?
Functor(5)
# OR
Functor.of(5)
# I love mapping functors
print(Functor.of("Start here").map(lambda i: i+" and keep walking!"))
# and some extra magic for free allows you to compare functors based
# on the values they hold
print(Functor.of(5) == Functor.of(5)) # => True (fingers crossed)
def applyOnBatches(self,
matrix: List[List[float]],
yValueIndex: int = None):
"""
matrix =
[
[x1, x2, x3, ...],
[x1, x2, x3, ...],
[x1, x2, x3, ...],
...
]
"""
if self.__data__ is None:
self.initialize()
if isinstance(matrix[0], numbers.Number):
print("Argument \"matrix\" can only be List[List[float]]!")
return self
zipMatrix = _.zip(*matrix)
new_matrix = []
for ind, f in enumerate(self.func):
min_val = self.__min__[ind]
max_val = self.__max__[ind]
bin_width = self.__bin_width__[ind]
bin_number = self.__bin_number__[ind] - 1
new_matrix.append(
_.map(
lambda val: f(min_val, max_val, bin_width, bin_number, val
), zipMatrix[ind]))
y_val_ind = yValueIndex or len(matrix[0]) - 1
for item in _.zip(*new_matrix):
inds = tuple(item)
# Points outside the set limits are ignored.
if None in inds:
break
agg = self.__data__[inds]
if agg is None:
agg = self.__data__[inds] = {
'cnt': 0,
'sum': 0,
'sum2': 0,
'min': 0,
'max': 0
}
agg['cnt'] += 1
y_val = item[y_val_ind]
agg['sum'] += y_val
agg['sum2'] += y_val * y_val
if agg['min'] > y_val:
agg['min'] = y_val
if agg['max'] < y_val:
agg['max'] = y_val
return self
def getShapshot(shader, color_map):
# map the count to RGB colors
# this is necessary for imshow to work
return _.map(lambda i: color_map(i), shader.getAgg('cnt'))
'xtick.major.width': .3,
'xtick.minor.width': .3,
'axes.linewidth': .3,
'axes.spines.top': False,
'axes.spines.right': False,
# stops the renderer from embedding the font
'svg.fonttype': 'none',
'font.family': 'sans-serif',
'font.sans-serif': 'Source Sans Pro'
})
rcParams['font.family'] = 'sans-serif'
rcParams['font.sans-serif'] = 'Source Sans Pro'
FILES = _.map(os.path.realpath, ['./matrix_data.csv'])
def readFile(file_name: str) -> None:
with open(file_name, newline="\n") as file:
data = csv.reader(file, delimiter=",")
headers = next(data)
return tree.apply(data, lambda i: [i[3], i[1], i[0]])
PIPE = compose(lambda i: i.sort() or i,
curry(_.strictMap)(float),
curry(_.filter)(lambda i: i != "cnt"), list)