def main(input_file, index): log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' logging.basicConfig(level=logging.INFO, format=log_fmt) logger = logging.getLogger(__name__) client = elasticsearch.Elasticsearch() indices_client = client.indices if indices_client.exists(index): logger.info("{} already exists. Deleting.".format(index)) indices_client.delete(index) logger.info("{} deleted.".format(index)) logger.info("Creating {}.".format(index)) indices_client.create(index=index, body=HAUNTED_PLACES_BODY) logger.info("{} created.".format(index)) reader = DictReader(input_file) haunted_place_actions = curry(make_haunted_place_action)(index) # Zip the reader with an infinite incrementor for the IDs. reader_actions = map(haunted_place_actions, iterate(lambda x: x + 1, 0), reader) start = time() logger.info("Indexing documents from {} into {}.".format( input_file.name, index)) num_ok, num_fail = bulk(client, reader_actions) logger.info("Done. Documents indexed in {:.2f}s.".format(time() - start))
def main(report_file): """ Creates an Elasticsearch index for the NUFORC reports and loads the processed CSV file into it. """ client = elasticsearch.Elasticsearch() index_client = elasticsearch.client.IndicesClient(client) # Drop the index if it exists; it will be replaced. This is the most efficient # way to delete the data from an index according to ES documentation. if index_client.exists(nuforc_report_index_name): index_client.delete(nuforc_report_index_name) # Create the index with the appropriate mapping. index_client.create(nuforc_report_index_name, nuforc_report_index_body) reports = DictReader(report_file) # Zip the reports with an id generator, embedding them in the actions. report_actions = map(nuforc_bulk_action, reports, iterate(lambda x: x + 1, 0)) # Stream the reports into the ES database. for ok, resp in elasticsearch.helpers.streaming_bulk( client, report_actions): if not ok: print(resp)
def divide_by_n(n, h0): ''' Repetedly divide an initial valve by n Parameters n: Number to divide by h0: Initial value Returns (Possibly) infinite sequence h0, h0/n, h0/n**2,... ''' return iterate(lambda x: x/n, h0)
def divide_by_n(n, h0): ''' Repetedly divide an initial valve by n Parameters n: Number to divide by h0: Initial value Returns (Possibly) infinite sequence h0, h0/n, h0/n**2,... ''' return iterate(lambda x: x / n, h0)
def gradient_descent(df, theta_0, eta=0.1): ''' Parameters df: Gradient of function f theta0: Initial guess, theta ia a j dimensional vector ([theta_01, theta_02,...,theta0_0j]) eta: Learning rate Returns Generator sequence of [theta_k1, theta_k2,...,theta_kj] where k = 0 to ... ''' return iterate(gradient_step(df, eta), theta_0)
def gradient_descent(df, theta_0, eta=0.1): """ Parameters df: Gradient of function f theta0: Initial guess, theta ia a j dimensional vector ([theta_01, theta_02,...,theta0_0j]) eta: Learning rate Returns Generator sequence of [theta_k1, theta_k2,...,theta_kj] where k = 0 to ... """ return iterate(gradient_step(df, eta), theta_0)
def process(text): origlines = lcompact(text.splitlines()) answer_one = proc_line(origlines, 0, 0, []) assert answer_one == 1675 ctr = partial(next, iterate(lambda x: x + 1, 0)) lines = origlines[:] answer_two = None while True: acc = proc_line2(lines, 0, 0, [], False) if acc != False: answer_two = acc break i = ctr() lines = origlines[:] if "jmp" in lines[i]: lines[i] = lines[i].replace("jmp", "nop") elif "nop" in lines[i]: lines[i] = lines[i].replace("nop", "jmp") assert answer_two == 1532 print("Answer one: ", answer_one) print("Answer two: ", answer_two) return
import matplotlib.pyplot as plt, numpy as np from toolz import iterate, nth X = np.random.multivariate_normal(np.array([0, 0]), np.array([[5,1.5], [1.5, 2]]), 20) r = nth(1000, iterate(lambda r: X.T @ X @ r / np.linalg.norm(X.T @ X @ r), np.random.rand(2))) plt.plot(X[:,0], X[:,1], "ro") plt.plot(np.linspace(-8, 8, 200), np.linspace(-8, 8, 200) * r[1] / r[0], "g") plt.show()
from itertools import repeat from functools import partial import types import pytest import toolz as tlz map_c = tlz.curry(tlz.map) reduce_c = tlz.curry(tlz.reduce) from smpl_tokenizer import utils is_generator = lambda obj: isinstance(obj, types.GeneratorType) var_len_strings = lambda n: list( tlz.take(n, tlz.iterate(lambda string: string + "a", ""))) """ @pytest.mark.parametrize("test_input,expected", [ ("3+5", 8), ("2+4", 6), ("6*9", 42), ]) def _eval(test_input, expected): assert _eval(test_input) == expected @pytest.mark.parametrize("x", [0, 1]) @pytest.mark.parametrize("y", [2, 3]) def test_foo(x, y): pass """
def main(n=20): for b in islice(iterate(lambda b: b.next(), Block.create_genesis_block()), 1, n + 1): print('Block #{} has been added to the blockchain!'.format(b.index)) print('Hash: {}\n'.format(b.hash))
def sgd(theta: Params, x: Vector, y: Vector) -> Params: step = tz.curry(sgd_step)(x, y) converge = lambda x: abs(sum(map(sub, x[0], x[1]))) > 1e-5 return until_convergence(tz.iterate(step, theta))
from collections import Counter, defaultdict from functools import partial, reduce from itertools import product from pathlib import Path from toolz import ( # type: ignore compose_left, iterate, ) lfilter = compose_left(filter, list) # lambda f, l: [*filter(f, l)] lmap = compose_left(map, list) # lambda f, l: [*map(f, l)] lcompact = partial(lfilter, None) splitstrip = compose_left(str.split, partial(lmap, str.strip), lcompact) make_incr = lambda: partial(next, iterate(lambda x: x + 1, 0)) def parsecommand(line): regstr, value = splitstrip(line, " = ") register = regstr.removeprefix("mem[").removesuffix("]") return {"r": int(register), "v": int(value)} def int_to_36bit(i): return bin(i).removeprefix("0b").zfill(36) def get_mask(maskstr): vals = {} for i, bit in enumerate(maskstr.removeprefix("0b")): if bit in ("0", "1"): vals[i] = bit return vals
def newton_converge(n, guess): next_step = lambda a: (a + n/a)/2 return until_convergence(iterate(next_step, guess))
def newton_f(n, guess=2, step=10): next_step = lambda a: (a + n/a)/2 return nth(step, iterate(next_step, guess))
np.set_printoptions(precision = 4, suppress = True) f = lambda x, w : x@w E = lambda X, W, T : 0.5 * np.sum((f(X, W)-T).T @ (f(X, W)-T)) model = lambda x : -(2.3*x-2)**2 -10 def gradient_descent(W, X, T, e): d = (X.T @ (X@W-T)) return W - d*e / np.linalg.norm(d) #Adagrad if __name__ == "__main__": data_sigma = 0.1 data_num = 4 M = 6 #次数+1を指定(3次式なら4を入力) e = np.array([[0.8], [0.8], [0.3], [0.3], [0.1], [0.5]]) * 0.001 w_init = np.random.normal(-5, 10, M).reshape(M, 1) X = np.array([[i**j for j in range(M)] for i in np.linspace(0.1, 2*np.pi, data_num)]) T = (model(X.T[1]) + np.random.normal(0, data_sigma, data_num)).reshape(data_num, 1) w_ans = nth(5000000, iterate(lambda w:gradient_descent(w, X, T, e), w_init)) print("\nerror ",E(X, w_ans, T)) #評価関数 print("w_ans ", (w_ans.T[0])) #推定したparemeter print("Taylor", [(-1)**int(i%4/2)/factorial(i)*(i%2) for i in range(M)]) #sinのテイラー展開parameter plotX = np.array([[i**j for j in range(M)] for i in np.linspace(0, 2*np.pi, 100)]) plt.plot(X.T[1], T, "bo") #観測データ plt.plot(plotX.T[1], f(plotX, w_ans), "g") #モデル計算後 plt.plot(plotX.T[1], model(plotX.T[1])) #理論値 plt.show()
# X is a nexamples x nfeatures matrix # syn0 is a nfeatures x nhidden matrix of weights # l1 is a nexamples x nhidden matrix # syn1 is a nhidden x 1 matrix X = np.array([[0, 0, 1], [0, 1, 1], [1, 0, 1], [1, 1, 1]]) y = np.array([[0], [1], [1], [0]]) nfeatures = X.shape[1] nexamples = X.shape[0] nhidden = 16 noutputs = y.shape[1] syn0, l1, syn1, l2 = nn.initialize(nexamples, nfeatures, nhidden, noutputs) network0 = (X, syn0, l1, syn1, l2) alpha = 10.0 out = list(take(20000, iterate(partial(nn.fit, y, alpha), network0))) print(out[-1]) print(nn.error(y, out[-1])) errors = [nn.error(y, o) for o in out] fig, ax = plt.subplots(nrows=1, ncols=1) ax.semilogy(range(0, len(out)), errors) plt.show() plt.clf()
from itertools import repeat from functools import partial import types import pytest import toolz as tlz map_c = tlz.curry(tlz.map) reduce_c = tlz.curry(tlz.reduce) from smpl_tokenizer import utils is_generator = lambda obj: isinstance(obj, types.GeneratorType) var_len_strings = lambda n: list(tlz.take(n, tlz.iterate(lambda string: string + "a", ""))) """ @pytest.mark.parametrize("test_input,expected", [ ("3+5", 8), ("2+4", 6), ("6*9", 42), ]) def _eval(test_input, expected): assert _eval(test_input) == expected @pytest.mark.parametrize("x", [0, 1]) @pytest.mark.parametrize("y", [2, 3]) def test_foo(x, y):