def main(input_file, index):
log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(level=logging.INFO, format=log_fmt)
logger = logging.getLogger(__name__)
client = elasticsearch.Elasticsearch()
indices_client = client.indices
if indices_client.exists(index):
logger.info("{} already exists. Deleting.".format(index))
indices_client.delete(index)
logger.info("{} deleted.".format(index))
logger.info("Creating {}.".format(index))
indices_client.create(index=index, body=HAUNTED_PLACES_BODY)
logger.info("{} created.".format(index))
reader = DictReader(input_file)
haunted_place_actions = curry(make_haunted_place_action)(index)
# Zip the reader with an infinite incrementor for the IDs.
reader_actions = map(haunted_place_actions, iterate(lambda x: x + 1, 0),
reader)
start = time()
logger.info("Indexing documents from {} into {}.".format(
input_file.name, index))
num_ok, num_fail = bulk(client, reader_actions)
logger.info("Done. Documents indexed in {:.2f}s.".format(time() - start))
def main(report_file):
""" Creates an Elasticsearch index for the NUFORC reports and loads the
processed CSV file into it.
"""
client = elasticsearch.Elasticsearch()
index_client = elasticsearch.client.IndicesClient(client)
# Drop the index if it exists; it will be replaced. This is the most efficient
# way to delete the data from an index according to ES documentation.
if index_client.exists(nuforc_report_index_name):
index_client.delete(nuforc_report_index_name)
# Create the index with the appropriate mapping.
index_client.create(nuforc_report_index_name, nuforc_report_index_body)
reports = DictReader(report_file)
# Zip the reports with an id generator, embedding them in the actions.
report_actions = map(nuforc_bulk_action, reports,
iterate(lambda x: x + 1, 0))
# Stream the reports into the ES database.
for ok, resp in elasticsearch.helpers.streaming_bulk(
client, report_actions):
if not ok:
print(resp)
def divide_by_n(n, h0):
'''
Repetedly divide an initial valve by n
Parameters
n: Number to divide by
h0: Initial value
Returns
(Possibly) infinite sequence h0, h0/n, h0/n**2,...
'''
return iterate(lambda x: x/n, h0)
def divide_by_n(n, h0):
'''
Repetedly divide an initial valve by n
Parameters
n: Number to divide by
h0: Initial value
Returns
(Possibly) infinite sequence h0, h0/n, h0/n**2,...
'''
return iterate(lambda x: x / n, h0)
def gradient_descent(df, theta_0, eta=0.1):
'''
Parameters
df: Gradient of function f
theta0: Initial guess, theta ia a j dimensional vector ([theta_01, theta_02,...,theta0_0j])
eta: Learning rate
Returns
Generator sequence of [theta_k1, theta_k2,...,theta_kj]
where k = 0 to ...
'''
return iterate(gradient_step(df, eta), theta_0)
def gradient_descent(df, theta_0, eta=0.1):
"""
Parameters
df: Gradient of function f
theta0: Initial guess, theta ia a j dimensional vector ([theta_01, theta_02,...,theta0_0j])
eta: Learning rate
Returns
Generator sequence of [theta_k1, theta_k2,...,theta_kj]
where k = 0 to ...
"""
return iterate(gradient_step(df, eta), theta_0)
def process(text):
origlines = lcompact(text.splitlines())
answer_one = proc_line(origlines, 0, 0, [])
assert answer_one == 1675
ctr = partial(next, iterate(lambda x: x + 1, 0))
lines = origlines[:]
answer_two = None
while True:
acc = proc_line2(lines, 0, 0, [], False)
if acc != False:
answer_two = acc
break
i = ctr()
lines = origlines[:]
if "jmp" in lines[i]:
lines[i] = lines[i].replace("jmp", "nop")
elif "nop" in lines[i]:
lines[i] = lines[i].replace("nop", "jmp")
assert answer_two == 1532
print("Answer one: ", answer_one)
print("Answer two: ", answer_two)
return
import matplotlib.pyplot as plt, numpy as np from toolz import iterate, nth X = np.random.multivariate_normal(np.array([0, 0]), np.array([[5,1.5], [1.5, 2]]), 20) r = nth(1000, iterate(lambda r: X.T @ X @ r / np.linalg.norm(X.T @ X @ r), np.random.rand(2))) plt.plot(X[:,0], X[:,1], "ro") plt.plot(np.linspace(-8, 8, 200), np.linspace(-8, 8, 200) * r[1] / r[0], "g") plt.show()
from itertools import repeat
from functools import partial
import types
import pytest
import toolz as tlz
map_c = tlz.curry(tlz.map)
reduce_c = tlz.curry(tlz.reduce)
from smpl_tokenizer import utils
is_generator = lambda obj: isinstance(obj, types.GeneratorType)
var_len_strings = lambda n: list(
tlz.take(n, tlz.iterate(lambda string: string + "a", "")))
"""
@pytest.mark.parametrize("test_input,expected", [
("3+5", 8),
("2+4", 6),
("6*9", 42),
])
def _eval(test_input, expected):
assert _eval(test_input) == expected
@pytest.mark.parametrize("x", [0, 1])
@pytest.mark.parametrize("y", [2, 3])
def test_foo(x, y):
pass
"""
def main(n=20):
for b in islice(iterate(lambda b: b.next(), Block.create_genesis_block()),
1, n + 1):
print('Block #{} has been added to the blockchain!'.format(b.index))
print('Hash: {}\n'.format(b.hash))
def sgd(theta: Params, x: Vector, y: Vector) -> Params:
step = tz.curry(sgd_step)(x, y)
converge = lambda x: abs(sum(map(sub, x[0], x[1]))) > 1e-5
return until_convergence(tz.iterate(step, theta))
from collections import Counter, defaultdict
from functools import partial, reduce
from itertools import product
from pathlib import Path
from toolz import ( # type: ignore
compose_left, iterate,
)
lfilter = compose_left(filter, list) # lambda f, l: [*filter(f, l)]
lmap = compose_left(map, list) # lambda f, l: [*map(f, l)]
lcompact = partial(lfilter, None)
splitstrip = compose_left(str.split, partial(lmap, str.strip), lcompact)
make_incr = lambda: partial(next, iterate(lambda x: x + 1, 0))
def parsecommand(line):
regstr, value = splitstrip(line, " = ")
register = regstr.removeprefix("mem[").removesuffix("]")
return {"r": int(register), "v": int(value)}
def int_to_36bit(i):
return bin(i).removeprefix("0b").zfill(36)
def get_mask(maskstr):
vals = {}
for i, bit in enumerate(maskstr.removeprefix("0b")):
if bit in ("0", "1"):
vals[i] = bit
return vals
def newton_converge(n, guess):
next_step = lambda a: (a + n/a)/2
return until_convergence(iterate(next_step, guess))
def newton_f(n, guess=2, step=10):
next_step = lambda a: (a + n/a)/2
return nth(step, iterate(next_step, guess))
np.set_printoptions(precision = 4, suppress = True)
f = lambda x, w : x@w
E = lambda X, W, T : 0.5 * np.sum((f(X, W)-T).T @ (f(X, W)-T))
model = lambda x : -(2.3*x-2)**2 -10
def gradient_descent(W, X, T, e):
d = (X.T @ (X@W-T))
return W - d*e / np.linalg.norm(d) #Adagrad
if __name__ == "__main__":
data_sigma = 0.1
data_num = 4
M = 6 #次数+1を指定(3次式なら4を入力)
e = np.array([[0.8], [0.8], [0.3], [0.3], [0.1], [0.5]]) * 0.001
w_init = np.random.normal(-5, 10, M).reshape(M, 1)
X = np.array([[i**j for j in range(M)] for i in np.linspace(0.1, 2*np.pi, data_num)])
T = (model(X.T[1]) + np.random.normal(0, data_sigma, data_num)).reshape(data_num, 1)
w_ans = nth(5000000, iterate(lambda w:gradient_descent(w, X, T, e), w_init))
print("\nerror ",E(X, w_ans, T)) #評価関数
print("w_ans ", (w_ans.T[0])) #推定したparemeter
print("Taylor", [(-1)**int(i%4/2)/factorial(i)*(i%2) for i in range(M)]) #sinのテイラー展開parameter
plotX = np.array([[i**j for j in range(M)] for i in np.linspace(0, 2*np.pi, 100)])
plt.plot(X.T[1], T, "bo") #観測データ
plt.plot(plotX.T[1], f(plotX, w_ans), "g") #モデル計算後
plt.plot(plotX.T[1], model(plotX.T[1])) #理論値
plt.show()
# X is a nexamples x nfeatures matrix
# syn0 is a nfeatures x nhidden matrix of weights
# l1 is a nexamples x nhidden matrix
# syn1 is a nhidden x 1 matrix
X = np.array([[0, 0, 1],
[0, 1, 1],
[1, 0, 1],
[1, 1, 1]])
y = np.array([[0],
[1],
[1],
[0]])
nfeatures = X.shape[1]
nexamples = X.shape[0]
nhidden = 16
noutputs = y.shape[1]
syn0, l1, syn1, l2 = nn.initialize(nexamples, nfeatures, nhidden, noutputs)
network0 = (X, syn0, l1, syn1, l2)
alpha = 10.0
out = list(take(20000, iterate(partial(nn.fit, y, alpha), network0)))
print(out[-1])
print(nn.error(y, out[-1]))
errors = [nn.error(y, o) for o in out]
fig, ax = plt.subplots(nrows=1, ncols=1)
ax.semilogy(range(0, len(out)), errors)
plt.show()
plt.clf()
from itertools import repeat
from functools import partial
import types
import pytest
import toolz as tlz
map_c = tlz.curry(tlz.map)
reduce_c = tlz.curry(tlz.reduce)
from smpl_tokenizer import utils
is_generator = lambda obj: isinstance(obj, types.GeneratorType)
var_len_strings = lambda n: list(tlz.take(n, tlz.iterate(lambda string: string + "a", "")))
"""
@pytest.mark.parametrize("test_input,expected", [
("3+5", 8),
("2+4", 6),
("6*9", 42),
])
def _eval(test_input, expected):
assert _eval(test_input) == expected
@pytest.mark.parametrize("x", [0, 1])
@pytest.mark.parametrize("y", [2, 3])
def test_foo(x, y):
