def generate_minhashes(input_file, bbox, memory, n, k):
hashes = [lsh.hashFamily(i) for i in xrange(n)]
if memory:
test_suite = loadTestSuite(input_file, bbox=bbox, k=k)
# generate minhashes signatures
mh_t = time.clock()
tcs_minhashes = {
tc[0]: lsh.tcMinhashing(tc, hashes)
for tc in test_suite.items()
}
mh_time = time.clock() - mh_t
ptime_start = time.clock()
else:
# loading input file and generating minhashes signatures
sigfile = input_file.replace(".txt", ".sig")
sigtimefile = "{}_sigtime.txt".format(input_file.split(".")[0])
if not os.path.exists(sigfile):
mh_t = time.clock()
storeSignatures(input_file, sigfile, hashes, bbox, k)
mh_time = time.clock() - mh_t
with open(sigtimefile, "w") as fout:
fout.write(repr(mh_time))
else:
with open(sigtimefile, "r") as fin:
mh_time = eval(fin.read().replace("\n", ""))
ptime_start = time.clock()
tcs_minhashes, _ = loadSignatures(sigfile)
return hashes, tcs_minhashes, mh_time, ptime_start
def storeSignatures(input_file, sigfile, hashes, bbox=False, k=5):
with open(sigfile, "w") as sigfile:
with open(input_file) as fin:
tcID = 1
for tc in fin:
if bbox:
# shingling
tc_ = tc[:-1]
tc_shingles = set()
for i in range(len(tc_) - k + 1):
tc_shingles.add(hash(tc_[i:i + k]))
sig = lsh.tcMinhashing((tcID, set(tc_shingles)), hashes)
else:
tc_ = tc[:-1].split()
sig = lsh.tcMinhashing((tcID, set(tc_)), hashes)
for hash_ in sig:
sigfile.write(hash_)
sigfile.write(" ")
sigfile.write("\n")
tcID += 1
def fast_(input_file, selsize, r, b, bbox=False, k=5, memory=False, B=0):
"""INPUT
(str)input_file: path of input file
(fun)selsize: size of candidate set
(int)r: number of rows
(int)b: number of bands
(bool)bbox: True if BB prioritization
(int)k: k-shingle size (for BB prioritization)
(bool)memory: if True keep signature in memory and do not store them to file
OUTPUT
(list)P: prioritized test suite
"""
n = r * b # number of hash functions
hashes = [lsh.hashFamily(i) for i in range(n)]
if memory:
test_suite = loadTestSuite(input_file, bbox=bbox, k=k)
# generate minhashes signatures
mh_t = time.perf_counter()
tcs_minhashes = {
tc[0]: lsh.tcMinhashing(tc, hashes)
for tc in test_suite.items()
}
mh_time = time.perf_counter() - mh_t
else:
# loading input file and generating minhashes signatures
sigfile = input_file.replace(".txt", ".sig")
sigtimefile = "{}_sigtime.txt".format(input_file.split(".")[0])
if not os.path.exists(sigfile):
mh_t = time.perf_counter()
storeSignatures(input_file, sigfile, hashes, bbox, k)
mh_time = time.perf_counter() - mh_t
with open(sigtimefile, "w") as fout:
fout.write(repr(mh_time))
else:
with open(sigtimefile, "r") as fin:
mh_time = eval(fin.read().replace("\n", ""))
tcs_minhashes, load_time = loadSignatures(sigfile)
ptime_start = time.perf_counter()
tcs = set(tcs_minhashes.keys())
# budget B modification
if B == 0:
B = len(tcs)
BASE = 0.5
SIZE = int(len(tcs) * BASE) + 1
bucket = lsh.LSHBucket(tcs_minhashes.items(), b, r, n)
prioritized_tcs = [0]
# First TC
selected_tcs_minhash = lsh.tcMinhashing((0, set()), hashes)
first_tc = random.choice(list(tcs_minhashes.keys()))
for i in range(n):
if tcs_minhashes[first_tc][i] < selected_tcs_minhash[i]:
selected_tcs_minhash[i] = tcs_minhashes[first_tc][i]
prioritized_tcs.append(first_tc)
tcs -= set([first_tc])
del tcs_minhashes[first_tc]
iteration, total = 0, float(len(tcs_minhashes))
while len(tcs_minhashes) > 0:
iteration += 1
if iteration % 100 == 0:
sys.stdout.write(" Progress: {}%\r".format(
round(100 * iteration / total, 2)))
sys.stdout.flush()
if len(tcs_minhashes) < SIZE:
bucket = lsh.LSHBucket(tcs_minhashes.items(), b, r, n)
SIZE = int(SIZE * BASE) + 1
sim_cand = lsh.LSHCandidates(bucket, (0, selected_tcs_minhash), b, r,
n)
filtered_sim_cand = sim_cand.difference(prioritized_tcs)
candidates = tcs - filtered_sim_cand
if len(candidates) == 0:
selected_tcs_minhash = lsh.tcMinhashing((0, set()), hashes)
sim_cand = lsh.LSHCandidates(bucket, (0, selected_tcs_minhash), b,
r, n)
filtered_sim_cand = sim_cand.difference(prioritized_tcs)
candidates = tcs - filtered_sim_cand
if len(candidates) == 0:
candidates = tcs_minhashes.keys()
to_sel = min(selsize(len(candidates)), len(candidates))
selected_tc_set = random.sample(tuple(candidates), to_sel)
for selected_tc in selected_tc_set:
for i in range(n):
if tcs_minhashes[selected_tc][i] < selected_tcs_minhash[i]:
selected_tcs_minhash[i] = tcs_minhashes[selected_tc][i]
prioritized_tcs.append(selected_tc)
# select budget B
if len(prioritized_tcs) >= B + 1:
break
tcs -= set([selected_tc])
del tcs_minhashes[selected_tc]
# select budget B
if len(prioritized_tcs) >= B + 1:
break
ptime = time.perf_counter() - ptime_start
max_ts_size = sum((1 for line in open(input_file)))
return mh_time, ptime, prioritized_tcs[1:max_ts_size]
def fast_(input_file, wBoxFile, selsize, r, b, bbox=False, k=5, memory=False):
n = r * b # number of hash functions
tC0 = time.clock()
C = loadCoverage(wBoxFile)
tC1 = time.clock()
maxCov = reduce(lambda x, y: x | y, C.values())
hashes = [lsh.hashFamily(i) for i in range(n)]
if memory:
test_suite = loadTestSuite(input_file, bbox=bbox, k=k)
# generate minhashes signatures
mh_t = time.clock()
tcs_minhashes = {
tc[0]: lsh.tcMinhashing(tc, hashes)
for tc in test_suite.items()
}
mh_time = time.clock() - mh_t
ptime_start = time.clock()
else:
# loading input file and generating minhashes signatures
sigfile = input_file.replace(".txt", ".sig")
sigtimefile = "{}_sigtime.txt".format(input_file.split(".")[0])
if not os.path.exists(sigfile):
mh_t = time.clock()
storeSignatures(input_file, sigfile, hashes, bbox, k)
mh_time = time.clock() - mh_t
with open(sigtimefile, "w") as fout:
fout.write(repr(mh_time))
else:
with open(sigtimefile, "r") as fin:
mh_time = eval(fin.read().replace("\n", ""))
ptime_start = time.clock()
tcs_minhashes, load_time = loadSignatures(sigfile)
tcs = set(tcs_minhashes.keys())
BASE = 0.5
SIZE = int(len(tcs) * BASE) + 1
bucket = lsh.LSHBucket(tcs_minhashes.items(), b, r, n)
prioritized_tcs = [0]
# First TC
selected_tcs_minhash = lsh.tcMinhashing((0, set()), hashes)
first_tc = random.choice(list(tcs_minhashes.keys()))
for i in range(n):
if tcs_minhashes[first_tc][i] < selected_tcs_minhash[i]:
selected_tcs_minhash[i] = tcs_minhashes[first_tc][i]
prioritized_tcs.append(first_tc)
cov = C[first_tc]
for tc in C.keys():
C[tc] = C[tc] - cov
if tc in tcs and len(C[tc]) == 0:
tcs -= set([tc])
del tcs_minhashes[tc]
iteration, total = 0, float(len(tcs_minhashes))
while cov != maxCov:
iteration += 1
if iteration % 100 == 0:
sys.stdout.write(" Progress: {}%\r".format(
round(100 * iteration / total, 2)))
sys.stdout.flush()
if len(tcs_minhashes) < SIZE:
bucket = lsh.LSHBucket(tcs_minhashes.items(), b, r, n)
SIZE = int(SIZE * BASE) + 1
sim_cand = lsh.LSHCandidates(bucket, (0, selected_tcs_minhash), b, r,
n)
filtered_sim_cand = sim_cand.difference(prioritized_tcs)
candidates = tcs - filtered_sim_cand
if len(candidates) == 0:
selected_tcs_minhash = lsh.tcMinhashing((0, set()), hashes)
sim_cand = lsh.LSHCandidates(bucket, (0, selected_tcs_minhash), b,
r, n)
filtered_sim_cand = sim_cand.difference(prioritized_tcs)
candidates = tcs - filtered_sim_cand
if len(candidates) == 0:
candidates = tcs_minhashes.keys()
to_sel = min(selsize(len(candidates)), len(candidates))
selected_tc_set = random.sample(tuple(candidates), to_sel)
for selected_tc in selected_tc_set:
for i in range(n):
if tcs_minhashes[selected_tc][i] < selected_tcs_minhash[i]:
selected_tcs_minhash[i] = tcs_minhashes[selected_tc][i]
prioritized_tcs.append(selected_tc)
cov = cov | C[selected_tc]
for tc in C.keys():
C[tc] = C[tc] - cov
if tc in tcs and len(C[tc]) == 0:
tcs -= set([tc])
del tcs_minhashes[tc]
ptime = time.clock() - ptime_start
max_ts_size = sum((1 for line in open(input_file)))
return mh_time, tC1 - tC0, ptime, prioritized_tcs[1:max_ts_size]
def fast_pw(input_file, r, b, bbox=False, k=5, memory=False, B=0):
n = r * b # number of hash functions
hashes = [lsh.hashFamily(i) for i in range(n)]
if memory:
test_suite = loadTestSuite(input_file, bbox=bbox, k=k)
# generate minhashes signatures
mh_t = time.perf_counter()
tcs_minhashes = {tc[0]: lsh.tcMinhashing(tc, hashes)
for tc in test_suite.items()}
mh_time = time.perf_counter() - mh_t
ptime_start = time.perf_counter()
else:
# loading input file and generating minhashes signatures
sigfile = input_file.replace(".txt", ".sig")
sigtimefile = "{}_sigtime.txt".format(input_file.split(".")[0])
if not os.path.exists(sigfile):
mh_t = time.perf_counter()
storeSignatures(input_file, sigfile, hashes, bbox, k)
mh_time = time.perf_counter() - mh_t
with open(sigtimefile, "w") as fout:
fout.write(repr(mh_time))
else:
with open(sigtimefile, "r") as fin:
mh_time = eval(fin.read().replace("\n", ""))
ptime_start = time.perf_counter()
tcs_minhashes, load_time = loadSignatures(sigfile)
tcs = set(tcs_minhashes.keys())
# budget B modification
if B == 0:
B = len(tcs)
BASE = 0.5
SIZE = int(len(tcs)*BASE) + 1
bucket = lsh.LSHBucket(tcs_minhashes.items(), b, r, n)
prioritized_tcs = [0]
# First TC
selected_tcs_minhash = lsh.tcMinhashing((0, set()), hashes)
first_tc = random.choice(list(tcs_minhashes.keys()))
for i in range(n):
if tcs_minhashes[first_tc][i] < selected_tcs_minhash[i]:
selected_tcs_minhash[i] = tcs_minhashes[first_tc][i]
prioritized_tcs.append(first_tc)
tcs -= set([first_tc])
del tcs_minhashes[first_tc]
iteration, total = 0, float(len(tcs_minhashes))
while len(tcs_minhashes) > 0:
iteration += 1
if iteration % 100 == 0:
sys.stdout.write(" Progress: {}%\r".format(
round(100*iteration/total, 2)))
sys.stdout.flush()
if len(tcs_minhashes) < SIZE:
bucket = lsh.LSHBucket(tcs_minhashes.items(), b, r, n)
SIZE = int(SIZE*BASE) + 1
sim_cand = lsh.LSHCandidates(bucket, (0, selected_tcs_minhash),
b, r, n)
filtered_sim_cand = sim_cand.difference(prioritized_tcs)
candidates = tcs - filtered_sim_cand
if len(candidates) == 0:
selected_tcs_minhash = lsh.tcMinhashing((0, set()), hashes)
sim_cand = lsh.LSHCandidates(bucket, (0, selected_tcs_minhash),
b, r, n)
filtered_sim_cand = sim_cand.difference(prioritized_tcs)
candidates = tcs - filtered_sim_cand
if len(candidates) == 0:
candidates = tcs_minhashes.keys()
selected_tc, max_dist = random.choice(tuple(candidates)), -1
for candidate in tcs_minhashes:
if candidate in candidates:
dist = lsh.jDistanceEstimate(
selected_tcs_minhash, tcs_minhashes[candidate])
if dist > max_dist:
selected_tc, max_dist = candidate, dist
for i in range(n):
if tcs_minhashes[selected_tc][i] < selected_tcs_minhash[i]:
selected_tcs_minhash[i] = tcs_minhashes[selected_tc][i]
prioritized_tcs.append(selected_tc)
# select budget B
if len(prioritized_tcs) >= B+1:
break
tcs -= set([selected_tc])
del tcs_minhashes[selected_tc]
ptime = time.perf_counter() - ptime_start
max_ts_size = sum((1 for line in open(input_file)))
return mh_time, ptime, prioritized_tcs[1:max_ts_size]
def fast(input_file,
r,
b,
sel_fun,
times=None,
bbox=False,
k=5,
memory=False,
sub_set=[]):
"""INPUT
(str)input_file: path of input file
(int)r: number of rows
(int)b: number of bands
(bool)bbox: True if BB prioritization
(int)k: k-shingle size (for BB prioritization)
(bool)memory: if True keep signature in memory and do not store them to file
OUTPUT
(list)P: prioritized test suite
"""
n = r * b # number of hash functions
hashes, tcs_minhashes, mh_time, ptime_start = generate_minhashes(
input_file, bbox, memory, n, k)
tcs_minhashes_original = tcs_minhashes.copy()
if len(sub_set) > 0:
# ALLOW US TO APPLY FAST ONLY IN A SUBSET OF TCS
tcs_minhashes = dict(
(k, tcs_minhashes[k]) for k in sub_set if k in tcs_minhashes)
tcs = set(tcs_minhashes.keys())
BASE = 0.5
SIZE = int(len(tcs) * BASE) + 1
bucket = lsh.LSHBucket(tcs_minhashes.items(), b, r, n)
prioritized_tcs = [0]
# First TC
selected_tcs_minhash = lsh.tcMinhashing((0, set()), hashes)
first_tc = random.choice(tcs_minhashes.keys())
for i in xrange(n):
if tcs_minhashes[first_tc][i] < selected_tcs_minhash[i]:
selected_tcs_minhash[i] = tcs_minhashes[first_tc][i]
prioritized_tcs.append(first_tc)
tcs -= set([first_tc])
del tcs_minhashes[first_tc]
iteration, total = 0, float(len(tcs_minhashes))
while len(tcs_minhashes) > 0:
iteration += 1
if iteration % 100 == 0:
sys.stdout.write(" Progress: {}%\r".format(
round(100 * iteration / total, 2)))
sys.stdout.flush()
if len(tcs_minhashes) < SIZE:
bucket = lsh.LSHBucket(tcs_minhashes.items(), b, r, n)
SIZE = int(SIZE * BASE) + 1
sim_cand = lsh.LSHCandidates(bucket, (0, selected_tcs_minhash), b, r,
n)
filtered_sim_cand = sim_cand.difference(prioritized_tcs)
candidates = tcs - filtered_sim_cand
if len(candidates) == 0:
selected_tcs_minhash = lsh.tcMinhashing((0, set()), hashes)
sim_cand = lsh.LSHCandidates(bucket, (0, selected_tcs_minhash), b,
r, n)
filtered_sim_cand = sim_cand.difference(prioritized_tcs)
candidates = tcs - filtered_sim_cand
if len(candidates) == 0:
candidates = tcs_minhashes.keys()
sel_fun(candidates, tcs_minhashes, selected_tcs_minhash,
prioritized_tcs, tcs, n, times)
ptime = time.clock() - ptime_start
# Calculate dissimilarity of prioritization
dissimilarity = dissimilarity_obj_fun(prioritized_tcs[1:],
tcs_minhashes_original)
return mh_time, ptime, prioritized_tcs[1:], dissimilarity
