def handle(self, *args, **options):
bf = BloomFilter(n=options.get("n") * 2, p=options.get("p"))
with open(options.get("input"), "r") as inp:
for i in trange(options.get("n")):
line = inp.readline()
if not line:
break
bf.add(line.strip().upper().encode("ascii"))
filename = settings.BASE_PASSWORD_STRENGTH_BLOOM_FILE
tmpfile = f"{filename}.tmp"
logger.info(f"Writing new password bloom filter: {tmpfile}")
with open(tmpfile, "wb") as outp:
bf.write(outp)
logger.info(f"Replacing old password bloom filter: {filename}")
os.rename(tmpfile, filename)
bloom = BloomFilter(n=1000000, p=0.0001)
# 3. insert DGA domains from json dict into bloomfilter and create dict for family lookup
dga_lookup_dict = {}
first_entry = False
test_domain = None
test_family = None
for family, domain_list in dga_json.items():
if not first_entry:
test_domain = domain_list[0]
test_family = family
first_entry = True
for domain in domain_list:
# insert domain into bloomfilter
if not domain.lower().encode("utf8") in bloom:
bloom.add(domain.lower().encode("utf8"))
# insert into family/domain lookup table
dga_lookup_dict[domain] = family
# test with first DGA domain/family pair that should be present in the bloomfilter and the dga_dict
if not (first_entry and test_domain and test_family):
logging.error("Unknown error while creating bloomfilter and DGA dict")
sys.exit(-1)
if test_domain.lower().encode("utf8") in bloom and dga_lookup_dict.get(
test_domain.lower(), "") == test_family:
logging.info("%s (%s)", test_domain, test_family)
logging.info("Bloomfilter and DGA dict successfully created")
else:
logging.error("Unknown error while creating bloomfilter and DGA dict")
sys.exit(-1)
bloom = BloomFilter(n=1000000, p=0.0001)
# 3. insert DGA domains from json dict into bloomfilter and create dict for family lookup
dga_lookup_dict = {}
first_entry = False
test_domain = None
test_family = None
for family, domain_list in dga_json.items():
if not first_entry:
test_domain = domain_list[0]
test_family = family
first_entry = True
for domain in domain_list:
# insert domain into bloomfilter
if not domain.lower().encode() in bloom:
bloom.add(domain.lower().encode())
# insert into family/domain lookup table
dga_lookup_dict[domain] = family
# test with first DGA domain/family pair that should be present in the bloomfilter and the dga_dict
if not (first_entry and test_domain and test_family):
logging.error("Unknown error while creating bloomfilter and DGA dict")
sys.exit(-1)
if test_domain.lower().encode() in bloom and dga_lookup_dict.get(test_domain.lower(), "") == test_family:
logging.info("%s (%s)", test_domain, test_family)
logging.info("Bloomfilter and DGA dict successfully created")
else:
logging.error("Unknown error while creating bloomfilter and DGA dict")
sys.exit(-1)