def test_basic_sanity():
# Basic sanity check of know resuts.
expected_itemsets = {
ItemSet("e"),
ItemSet("de"),
ItemSet("ade"),
ItemSet("ce"),
ItemSet("ae"),
ItemSet("d"),
ItemSet("cd"),
ItemSet("bcd"),
ItemSet("acd"),
ItemSet("bd"),
ItemSet("abd"),
ItemSet("ad"),
ItemSet("c"),
ItemSet("bc"),
ItemSet("abc"),
ItemSet("ac"),
ItemSet("b"),
ItemSet("ab"),
ItemSet("a"),
}
(itemsets, _, _) = mine_fp_tree(test_transactions,
2 / len(test_transactions))
assert (set(itemsets) == expected_itemsets)
def main():
parser = ArgumentParser(
description="Association rule data mining in Python")
parser.add_argument("--input", dest="input", required=True)
parser.add_argument("--output", dest="output", required=True)
parser.add_argument("--min-confidence",
dest="min_confidence",
type=float_between_0_and_1,
required=True)
parser.add_argument("--min-support",
dest="min_support",
type=float_between_0_and_1,
required=True)
parser.add_argument("--min-lift",
dest="min_lift",
type=float_gteq_1,
required=True)
args = parser.parse_args()
program_start = time.time()
start = program_start
print("ARMPY - Association Rule Mining using Python.")
print("Input file: {}".format(args.input))
print("Output file: {}".format(args.output))
print("Minimum support: {}".format(args.min_confidence))
print("Minimum confidence: {}".format(args.min_support))
print("Minimum lift: {}".format(args.min_lift))
print("Generating frequent itemsets using FPGrowth...", flush=True)
reader = DatasetReader(args.input)
(itemsets, itemset_counts,
num_transactions) = mine_fp_tree(reader, args.min_support)
duration = time.time() - start
print("FPGrowth mined {} items in {:.2f} seconds".format(
len(itemsets), duration),
flush=True)
start = time.time()
rules = generate_rules(itemsets, itemset_counts, num_transactions,
args.min_confidence, args.min_lift)
duration = time.time() - start
print("Generated {} rules in {:.2f} seconds".format(len(rules), duration),
flush=True)
start = time.time()
with open(args.output, "w") as f:
f.write("Antecedent->Consequent,Confidence,Lift,Support\n")
for (antecedent, consequent, confidence, lift, support) in rules:
f.write("{} -> {},{:.4f},{:.4f},{:.4f}\n".format(
set_to_string(antecedent), set_to_string(consequent),
confidence, lift, support))
print("Wrote rules to disk in {:.2f} seconds".format(duration), flush=True)
duration = time.time() - program_start
print("Total runtime {:.2f} seconds".format(duration))
return 0
def test_stress():
datasets = [
("datasets/UCI-zoo.csv", 0.3),
("datasets/mushroom.csv", 0.4),
# ("datasets/BMS-POS.csv", 0.05),
# ("datasets/kosarak.csv", 0.05),
]
for (csvFilePath, min_support) in datasets:
# Run Apriori and FP-Growth and assert both have the same results.
print("Running Apriori for {}".format(csvFilePath))
start = time.time()
index = InvertedIndex()
index.load_csv(csvFilePath)
apriori_itemsets = apriori(index, min_support)
apriori_duration = time.time() - start
print(
"Apriori complete. Generated {} itemsets in {:.2f} seconds".format(
len(apriori_itemsets),
apriori_duration))
print("Running FPTree for {}".format(csvFilePath))
start = time.time()
with open(csvFilePath, newline='') as csvfile:
test_transactions = list(csv.reader(csvfile))
fptree_itemsets = mine_fp_tree(test_transactions, min_support)
fptree_duration = time.time() - start
print(
"fp_growth complete. Generated {} itemsets in {:.2f} seconds".format(
len(fptree_itemsets),
fptree_duration))
if set(fptree_itemsets) == set(apriori_itemsets):
print("SUCCESS({}): Apriori and fptree results match".format(csvFilePath))
else:
print("FAIL({}): Apriori and fptree results differ!".format(csvFilePath))
assert(set(fptree_itemsets) == set(apriori_itemsets))
if apriori_duration > fptree_duration:
print(
"FPTree was faster by {:.2f} seconds".format(
apriori_duration -
fptree_duration))
else:
print(
"Apriori was faster by {:.2f} seconds".format(
fptree_duration -
apriori_duration))
print("")
def main():
args = parse_args()
program_start = time.time()
if args.trace_malloc:
tracemalloc.start()
print(
"Virtual Change/Drift Detection - Association Rule Mining using Python."
)
print("Drift Algorithm: {}".format(args.drift_algorithm))
print("Input file: {}".format(args.input))
print("Output file prefix: {}".format(args.output))
print("Training window size: {}".format(args.training_window_size))
print("Minimum confidence: {}".format(args.min_confidence))
print("Minimum support: {}".format(args.min_support))
print("Minimum lift: {}".format(args.min_lift))
if args.fixed_drift_confidence is not None:
print("Fixed drift confidence of: {}".format(
args.fixed_drift_confidence))
print("Tracing memory allocations: {}".format(args.trace_malloc))
print("Save rules: {}".format(args.save_rules))
print("Generating maximal itemsets: {}".format(args.maximal_itemsets))
reader = iter(DatasetReader(args.input))
transaction_num = 0
end_of_last_window = 0
cohort_num = 1
volatility_detector = make_volatility_detector(args)
while True:
window = take(args.training_window_size, reader)
if len(window) == 0:
break
print("")
print("Mining window [{},{}]".format(transaction_num,
transaction_num + len(window)))
end_of_last_window = transaction_num + len(window)
transaction_num += len(window)
print("Running FP-Growth...", flush=True)
start = time.time()
(itemsets, itemset_counts,
num_transactions) = mine_fp_tree(window, args.min_support,
args.maximal_itemsets)
assert (num_transactions == len(window))
duration = time.time() - start
print("FPGrowth mined {} items in {:.2f} seconds".format(
len(itemsets), duration))
print("Generating rules...", flush=True)
start = time.time()
rules = list(
generate_rules(itemsets, itemset_counts, num_transactions,
args.min_confidence, args.min_lift))
duration = time.time() - start
print("Generated {} rules in {:.2f} seconds".format(
len(rules), duration),
flush=True)
if len(rules) == 0:
print("No rules; just noise. Skipping change detection.")
print(
"Consider increasing training window size or lowering minsup/conf."
)
continue
if args.save_rules:
start = time.time()
output_filename = args.output + "." + str(cohort_num)
cohort_num += 1
write_rules_to_file(rules, output_filename)
print("Wrote rules for cohort {} to file {} in {:.2f} seconds".
format(cohort_num, output_filename, duration),
flush=True)
drift_detector = make_drift_detector(args, volatility_detector)
drift_detector.train(window, rules)
# Read transactions until a drift is detected.
for transaction in reader:
transaction_num += 1
drift = drift_detector.check_for_drift(transaction,
transaction_num)
if drift is not None:
print(
"Detected drift of type {} at transaction {}, {} after end of training window"
.format(drift.drift_type, transaction_num,
transaction_num - end_of_last_window))
if drift.hellinger_value is not None:
print(
"Hellinger value: {}, confidence interval: {} ± {} ([{},{}])"
.format(drift.hellinger_value, drift.mean,
drift.confidence,
drift.mean - drift.confidence,
drift.mean + drift.confidence))
# Record the drift in the volatility detector. This is used inside
# the drift detector to help determine how large a confidence interval
# is required when detecting drifts.
if volatility_detector is not None:
volatility_detector.add(transaction_num)
# Break out of the inner loop, we'll jump back up to the top and mine
# a new training window.
break
if len(window) < args.training_window_size:
break
print("\nEnd of stream\n")
duration = time.time() - program_start
print("Total runtime {:.2f} seconds".format(duration))
if args.trace_malloc:
(_, peak_memory) = tracemalloc.get_traced_memory()
tracemalloc_memory = tracemalloc.get_tracemalloc_memory()
print("Peak memory usage: {:.3f} MB".format(peak_memory / 10**6))
print("tracemalloc overhead: {:.3f} MB".format(
(tracemalloc_memory / 10**6)))
print("Peak memory usage minus tracemalloc overhead: {:.3f} MB".format(
(peak_memory - tracemalloc_memory) / 10**6))
snapshot = tracemalloc.take_snapshot()
bytes_allocated = sum(x.size for x in snapshot.traces)
print("Total traced memory allocated: {:.3f} MB".format(
bytes_allocated / 10**6))
tracemalloc.stop()
return 0