def test_merge_complete(self): m = Merger() l1 = Layer("1") l1.addChannel(2, 1) l1.addChannel(3, 255) l1.addChannel(4, 127) l2 = Layer("2") l2.addChannel(3, 0, 0.5) l2.addChannel(4, 255, "max") l2.addChannel(5, 255, "min") l3 = Layer("3") l3.addChannel(2, 255, 0.3) l4 = Layer("4") l4.addChannel(2, 127, 0.6) m.addLayer(l1) m.addLayer(l2) m.addLayer(l3) m.addLayer(l4) m.merge() self.assertEqual(m.galaxy[1], 0) self.assertEqual(m.galaxy[2], 107) self.assertEqual(m.galaxy[3], 128) self.assertEqual(m.galaxy[4], 255) self.assertEqual(m.galaxy[5], 0)
def test_merge_simple(self): m = Merger() l = Layer("1") l.addChannel(1, 255) l.addChannel(2, 127) m.addLayer(l) m.merge() self.assertEqual(m.galaxy[1], 255) self.assertEqual(m.galaxy[2], 127)
def merge(self, corpus_size): """ The function will merge all the data in the posting files using the BSBI algorithm """ docs_file = self.get_docs_file() for key in self.postings_data: if os.listdir(self.postings_data[key]['path']): # directory is not empty merger = Merger(self.postings_data[key]['path'], "pkl", docs_file, corpus_size) merger.merge(self.postings_data[key]['name']) # The merger updates the docs data. After the merge of all the letters - all the documents data # Is updated and need to be saved on disk to reduce the memory load utils.save_obj(docs_file, f"{self.posting_dir_path}\\docs\\docs_index")
def get_IMPA_Merger(name): imp = iMPA(name) terc = imp.terc data = imp.getAddresses() s = min(map(lambda x: x.center.y, data)) w = min(map(lambda x: x.center.x, data)) n = max(map(lambda x: x.center.y, data)) e = max(map(lambda x: x.center.x, data)) addr = getAddresses(map(str, (s, w, n, e))) m = Merger(data, addr, terc) m.post_func.append(m.merge_addresses) m.merge() return m
def get_impa_merger(name): imp = iMPA(name) terc = imp.terc data = imp.get_addresses() s = min(map(lambda x: x.center.y, data)) w = min(map(lambda x: x.center.x, data)) n = max(map(lambda x: x.center.y, data)) e = max(map(lambda x: x.center.x, data)) addr = get_addresses(map(str, (s, w, n, e))) m = Merger(data, addr, terc, "%s.e-mapa.net" % name) m.post_func.append(m.merge_addresses) m.merge() return m
def do_patch(self, file): merger = Merger(file) compile = False for p in self.patches: if p._apply: compile = True print("Merging program %s for \"%s\"..." % (p.get_program(), p.name())) f_m = open(p.get_program(), 'r+b') merger.merge(f_m) f_m.close() print("Program %s for \"%s\" merged.\n" % (p.get_program(), p.name())) if compile: print("Compiling ...") merger.compile() print("") for p in self.patches: if p._apply: print("Patching \"%s\"..." % (p.name())) file = p.patch(file) print("") return file
if __name__ == "__main__": if len(sys.argv) != 2: print "usage: python builder.py [filename]" archive = sys.argv[1] fp = FileParser() fp.extract(archive, "tmp") extract_all(archive) # get filename from archive: nz.tar => tmp/nz_merged files = fp.getFiles("tmp/" + archive.split(".")[0] + "_merged") t = time.time() r = IndexBuilder(files, UrlTable(), CParser(), Pipeline()) printf("parsing %d files:" % len(r.files)) r.process() print "\nparsed %d pages in %d files for %f seconds" % (r.page_id, r.id, time.time() - t) print "avarage %f second for parsing each files" % ((time.time() - t) / r.id) print "started to build revert_index: " t = time.time() m = Merger("tmp") m.merge() print "\nbuild reverted index for %d records in %f seconds" % (r.uid, (time.time() - t)) cleanup("tmp") build_index("rindex")
import matplotlib.pyplot as plt import numpy as np from keras import Sequential from keras.callbacks import History from keras.layers import Dense, BatchNormalization from sklearn.model_selection import train_test_split from loader import Loader from merger import Merger params, scores = Loader.get_flow_data(6767, 100) qualities = Loader.get_task_qualities() description = Loader.get_description(6767) merger = Merger(params, description, scores, qualities) X, y = merger.merge(100) # Split X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) # model # --> 0.0017802061972600456 model = Sequential() model.add(Dense(32, input_shape=(X.shape[1], ), activation='relu')) model.add(BatchNormalization()) model.add(Dense(32, activation='relu')) model.add(BatchNormalization())