def test_merge_complete(self):
m = Merger()
l1 = Layer("1")
l1.addChannel(2, 1)
l1.addChannel(3, 255)
l1.addChannel(4, 127)
l2 = Layer("2")
l2.addChannel(3, 0, 0.5)
l2.addChannel(4, 255, "max")
l2.addChannel(5, 255, "min")
l3 = Layer("3")
l3.addChannel(2, 255, 0.3)
l4 = Layer("4")
l4.addChannel(2, 127, 0.6)
m.addLayer(l1)
m.addLayer(l2)
m.addLayer(l3)
m.addLayer(l4)
m.merge()
self.assertEqual(m.galaxy[1], 0)
self.assertEqual(m.galaxy[2], 107)
self.assertEqual(m.galaxy[3], 128)
self.assertEqual(m.galaxy[4], 255)
self.assertEqual(m.galaxy[5], 0)
def test_merge_simple(self):
m = Merger()
l = Layer("1")
l.addChannel(1, 255)
l.addChannel(2, 127)
m.addLayer(l)
m.merge()
self.assertEqual(m.galaxy[1], 255)
self.assertEqual(m.galaxy[2], 127)
def merge(self, corpus_size):
"""
The function will merge all the data in the posting files using the BSBI algorithm
"""
docs_file = self.get_docs_file()
for key in self.postings_data:
if os.listdir(self.postings_data[key]['path']): # directory is not empty
merger = Merger(self.postings_data[key]['path'], "pkl", docs_file, corpus_size)
merger.merge(self.postings_data[key]['name'])
# The merger updates the docs data. After the merge of all the letters - all the documents data
# Is updated and need to be saved on disk to reduce the memory load
utils.save_obj(docs_file, f"{self.posting_dir_path}\\docs\\docs_index")
def get_IMPA_Merger(name):
imp = iMPA(name)
terc = imp.terc
data = imp.getAddresses()
s = min(map(lambda x: x.center.y, data))
w = min(map(lambda x: x.center.x, data))
n = max(map(lambda x: x.center.y, data))
e = max(map(lambda x: x.center.x, data))
addr = getAddresses(map(str, (s, w, n, e)))
m = Merger(data, addr, terc)
m.post_func.append(m.merge_addresses)
m.merge()
return m
def get_impa_merger(name):
imp = iMPA(name)
terc = imp.terc
data = imp.get_addresses()
s = min(map(lambda x: x.center.y, data))
w = min(map(lambda x: x.center.x, data))
n = max(map(lambda x: x.center.y, data))
e = max(map(lambda x: x.center.x, data))
addr = get_addresses(map(str, (s, w, n, e)))
m = Merger(data, addr, terc, "%s.e-mapa.net" % name)
m.post_func.append(m.merge_addresses)
m.merge()
return m
def do_patch(self, file):
merger = Merger(file)
compile = False
for p in self.patches:
if p._apply:
compile = True
print("Merging program %s for \"%s\"..." % (p.get_program(), p.name()))
f_m = open(p.get_program(), 'r+b')
merger.merge(f_m)
f_m.close()
print("Program %s for \"%s\" merged.\n" % (p.get_program(), p.name()))
if compile:
print("Compiling ...")
merger.compile()
print("")
for p in self.patches:
if p._apply:
print("Patching \"%s\"..." % (p.name()))
file = p.patch(file)
print("")
return file
if __name__ == "__main__":
if len(sys.argv) != 2:
print "usage: python builder.py [filename]"
archive = sys.argv[1]
fp = FileParser()
fp.extract(archive, "tmp")
extract_all(archive)
# get filename from archive: nz.tar => tmp/nz_merged
files = fp.getFiles("tmp/" + archive.split(".")[0] + "_merged")
t = time.time()
r = IndexBuilder(files, UrlTable(), CParser(), Pipeline())
printf("parsing %d files:" % len(r.files))
r.process()
print "\nparsed %d pages in %d files for %f seconds" % (r.page_id, r.id, time.time() - t)
print "avarage %f second for parsing each files" % ((time.time() - t) / r.id)
print "started to build revert_index: "
t = time.time()
m = Merger("tmp")
m.merge()
print "\nbuild reverted index for %d records in %f seconds" % (r.uid, (time.time() - t))
cleanup("tmp")
build_index("rindex")
import matplotlib.pyplot as plt
import numpy as np
from keras import Sequential
from keras.callbacks import History
from keras.layers import Dense, BatchNormalization
from sklearn.model_selection import train_test_split
from loader import Loader
from merger import Merger
params, scores = Loader.get_flow_data(6767, 100)
qualities = Loader.get_task_qualities()
description = Loader.get_description(6767)
merger = Merger(params, description, scores, qualities)
X, y = merger.merge(100)
# Split
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.33,
random_state=42)
# model
# --> 0.0017802061972600456
model = Sequential()
model.add(Dense(32, input_shape=(X.shape[1], ), activation='relu'))
model.add(BatchNormalization())
model.add(Dense(32, activation='relu'))
model.add(BatchNormalization())
