def test_generate_and_write_filter(self): self.setup_data() mlbf = MLBF.generate_from_db(123456) mlbf.generate_and_write_filter() with open(mlbf.filter_path, 'rb') as filter_file: buffer = filter_file.read() bfilter = FilterCascade.from_buf(buffer) blocked_versions = fetch_blocked_from_db() blocked_guids = blocked_versions.values() for guid, version_str in blocked_guids: key = MLBF.KEY_FORMAT.format(guid=guid, version=version_str) assert key in bfilter all_addons = fetch_all_versions_from_db(blocked_versions.keys()) for guid, version_str in all_addons: # edge case where a version_str exists in both if (guid, version_str) in blocked_guids: continue key = MLBF.KEY_FORMAT.format(guid=guid, version=version_str) assert key not in bfilter # Occasionally a combination of salt generated with secrets.token_bytes # and the version str generated in version_factory results in a # collision in layer 1 of the bloomfilter, leading to a second layer # being generated. When this happens the bitCount and size is larger. expected_size, expected_bit_count = ( (203, 1384) if bfilter.layerCount() == 1 else (393, 2824) ) assert os.stat(mlbf.filter_path).st_size == expected_size, ( blocked_guids, all_addons, ) assert bfilter.bitCount() == expected_bit_count, (blocked_guids, all_addons)
def test_generate_and_write_mlbf(self): self.setup_data() mlbf = MLBF(123456) mlbf.generate_and_write_mlbf() with open(mlbf.filter_path, 'rb') as filter_file: buffer = filter_file.read() bfilter = FilterCascade.from_buf(buffer) blocked_versions = mlbf.fetch_blocked_from_db() blocked_guids = blocked_versions.values() for guid, version_str in blocked_guids: key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str) assert key in bfilter all_addons = mlbf.fetch_all_versions_from_db(blocked_versions.keys()) for guid, version_str in all_addons: # edge case where a version_str exists in both if (guid, version_str) in blocked_guids: continue key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str) assert key not in bfilter assert os.stat(mlbf.filter_path).st_size == 203, ( blocked_guids, all_addons) assert bfilter.bitCount() == 1384, ( blocked_guids, all_addons)
def load_filter(self, *, filter_path, coverage_path): self.filter_file = filter_path self.coverage_file = coverage_path self.filtercascade = FilterCascade.from_buf( self.filter_file.read_bytes()) self.issuer_to_revocations = collections.defaultdict(list) self.stash_files = list() self.coverage = {} with open(coverage_path, "r") as f: for ct_log in json.load(f): log_id = base64.b64decode(ct_log["logID"]) self.coverage[log_id] = (ct_log["minTimestamp"], ct_log["maxTimestamp"])
def test_generate_and_write_mlbf(self): mlbf = MLBF(123456) mlbf.generate_and_write_mlbf() with open(mlbf.filter_path, 'rb') as filter_file: buffer = filter_file.read() bfilter = FilterCascade.from_buf(buffer) assert bfilter.bitCount() == 3008 blocked_versions = mlbf.get_blocked_versions() for guid, version_str in blocked_versions.values(): key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str) assert key in bfilter for guid, version_str in mlbf.get_all_guids(blocked_versions.keys()): key = mlbf.KEY_FORMAT.format(guid=guid, version=version_str) assert key not in bfilter assert os.stat(mlbf.filter_path).st_size == 406