def __init__(self, cache_nodes, db_node, write_mode):
super().__init__(cache_nodes, db_node, write_mode)
self.node_loads = {}
self.hash_fn = pyhash.fnv1_32()
self.hash_seed_a = 0
self.hash_seed_b = 1
for node in self.cache_nodes:
self.node_loads[node.id] = 0
def __init__(self, cache_nodes, db_node, write_mode, nrkeys, c):
super().__init__(cache_nodes, db_node, write_mode)
self.nrkeys = nrkeys
self.c = c
self.node_loads = {}
self.key_rates = {}
self.replicated_keys = SortedSet(
key=lambda x: self.key_rates[x].rate())
self.key_node_map = {}
self.hash_fn = pyhash.fnv1_32()
for node in self.cache_nodes:
self.node_loads[node.id] = 0
def __init__(self,filter_size):
'''
First, we initialize the Bloom filter and create a bit array of filter_size entered by the user.
In addition, we initialize each of the hash functions (Murmur3 and FNV-1) of our filter.
filter_size: size of the vector
'''
self.filter = filter_size
self.bit_array = bitarray(filter_size)
self.bit_array.setall(0) #since bitarray doesn't ensure that all bits are set to 0
self.hasher1 = pyhash.fnv1_32()
self.hasher2 = pyhash.murmur3_32()
def __init__(self, bit_vector_size):
self.bit_vector_size = bit_vector_size
# The size of the bit_vector must be positive
if bit_vector_size <= 0:
raise ValueError("Bit vector too small.")
# Initialize vector with all zeros.
self.bit_vector = [False] * self.bit_vector_size
# Define the hash functions to use.
self.hash_functions = [pyhash.fnv1_32(), pyhash.murmur1_32()]
# Number of hash functions used.
self.number_of_hashes = len(self.hash_functions)
and i don't care if it indicates if an item is there or if it
is not
EG: I run website and want to keep track of IP
addresses that are blocked. I dont care if a blocked IP
is occasionally able to access my website, but I do care if someone not
on the blocked list is unable to access the site
bit_vector is list of bits
"""
import pyhash
bit_vector = [0] * 20
#Non Cryptographic hash functions (murmer and FNV)
fnv = pyhash.fnv1_32()
murmur = pyhash.murmur3_32()
#Calculate the output of FNV and Murmur hash functions for Pikachu and Charmander
fnv_pika = fnv("Pikachu") % 20
fnv_char = fnv("Charmander") % 20
murmur_pika = murmur("Pikachu") % 20
murmur_char = murmur("Charmander") % 20
bit_vector[fnv_pika] = 1
bit_vector[murmur_pika] = 1
bit_vector[fnv_char] = 1
bit_vector[murmur_char] = 1
def bloomIPs(clientSize):
"""
I will implement the IP blocker above example.
For simplicity, lets assume our IP system is composed of values between
0-100000
"""
#My bit vector
bitVector = [0] * clientSize
"""
We use 3 hash functions, Murmur, FNV and metro hash systems. They are
non cryptographic hence will return the same value any time we pass in
the same value. We modularize them by our bit Vector size to make them
fit into it as it is our reference sheet.
"""
fnv = pyhash.fnv1_32()
murmur = pyhash.murmur3_32()
metro = pyhash.metro_128()
"""
Now lets imagine we identified a set of just 1000 hackers in our world.
Their IPs range from 0 to 1000 as follows.
"""
hackerSize = 1000
hackersList = range(0, hackerSize)
"""
To keep our random clients happy, we come up with a repo of all the
hackers Known. Our customers are very stubborn but they love being safe.
Its a very dangerous world out there.
We mark the hackers.
"""
for hacker in hackersList:
#Hash them with our 3 functions
bitVector[fnv(str(hacker)) % clientSize] = 1
bitVector[murmur(str(hacker)) % clientSize] = 1
bitVector[metro(str(hacker)) % clientSize] = 1
"""
Now our true clients make requests. We have say 100000 of them.
We look them up in our list and determine if they are hackers or not
An approved request is marked as Perfect. Lets count, of the 700,
False Positives are clients Noted as Hackers
How many will be marked perfect
"""
perfect = 0
falsePositive = 0
for cust in range(0, 100000):
trueClient = random.randrange(10000, 100000)
check1 = bitVector[fnv(str(trueClient)) % clientSize]
check2 = bitVector[murmur(str(trueClient)) % clientSize]
check3 = bitVector[metro(str(trueClient)) % clientSize]
#print("{}-{}-{}").format(check1,check2,check3)
"""
We will not grant perfection to them if they are detected as hackers
by any of our security systems, we mark the false positive.
Othewise we just think of them as false negatives
"""
checkFinale = (check1 == check2 == check3 == 1)
if checkFinale is True:
falsePositive += 1
else:
perfect += 1
doc = """
Running our check, we wil throw warnings to some true clients thinking
they are hackers when they are in fact not. Run it again to see how
many true clients connect. At least we know they are safe. But as seen.
there is a possibility of our clients, whose IPs are not even in the
same range as the hackers to be detected as hackers."""
return {
"hackerSize": hackerSize,
"falsePositive": falsePositive,
"clientSize": clientSize,
"doc": doc,
"perfect": perfect
}
the_page = response.read()
return the_page
def print_wget(name):
filename = "frequency/{}.request".format(name)
command = """
if [ ! -f {} ]; then
echo "fetching {}"...
curl -s {} -o "{}"
fi
"""
print(command.format(filename, nameurl(name), nameurl(name), filename))
hasher = pyhash.fnv1_32()
def get_id(page):
global hasher
b = str(tuple(sorted([str(x) for x in page['dados']])))
c = hasher(b)
return c
savefile = "sobrenomes.json"
def saveData():
global pg
global acc
import pyhash as ph
# Non cryptographic hash functions (Murmur and FNV)
fnv = ph.fnv1_32()
murmur = ph.murmur3_32()
# Calculate the output of FNV and Murmur hash functions for pikachu and chamander
bit_vector = [0] * 20
fnv_pika = fnv("Pikachu") % 20
murmur_pika = murmur("Pikachu") % 20
fnv_char = fnv("Charmander") % 20
murmur_char = murmur("Charmander") % 20
print("fnv_pika\t", fnv_pika)
print("fnv_char\t", fnv_char)
print("murmur_pika\t", murmur_pika)
print("murmur_char\t", murmur_char)
bit_vector[fnv_char] = 1
bit_vector[fnv_pika] = 1
bit_vector[murmur_char] = 1
bit_vector[murmur_pika] = 1
print(bit_vector)
# Se um deles der 0 ou os dois, o item não está no bloom filter
return hash(req.string)
def python_integer(req):
"""
Hashing an integer key
"""
return hash(req.integer)
import pyhash
#https://code.google.com/p/pyfasthash/
h_fnv1_32 = pyhash.fnv1_32()
def fnv1_32(req):
return h_fnv1_32(str(req))
h_lookup3 = pyhash.lookup3_big()
def lookup3(req):
return h_lookup3(str(req))
h_super_fast_hash = pyhash.super_fast_hash()
def super_fast_hash(req):
return h_super_fast_hash(str(req))
h_murmur2_x64_64a = pyhash.murmur2_x64_64a()
def murmur2_x64_64a(req):
def __init__(self):
self.DIVISOR = 20
self.big_vector = [0] * self.DIVISOR
self.fnv = fnv1_32()
self.murmur = murmur3_32()
def get_hasher():
return pyhash.fnv1_32()
def __init__(self, Nwords=100000):
self.hasher = pyhash.fnv1_32()
self.Nwords = Nwords
import pyhash bit_vector = [0]*20 fvn = pyhash.fnv1_32() murmur = pyhash.murmur3_32()
help="approximate set type")
parser.add_argument('--outputfile', default="", help="output file")
args = parser.parse_args()
keys = []
check_keys = []
with open(args.keysfile) as f:
for _ in range(args.nkeys):
keys.append(f.readline())
for _ in range(args.ntrials):
check_keys.append(f.readline())
hash_fns = []
for i in range(args.k):
hash_fns.append(approxset.HashFn(pyhash.fnv1_32(), i))
if args.type == 'bloom':
aset = approxset.BloomFilter(hash_fns, args.m)
elif args.type == 'cmsketch':
aset = approxset.CMSketch(hash_fns, args.m // args.k)
simulator = Simulator(aset, keys, check_keys)
(false_pos, false_neg) = simulator.run()
print("Expected false positive rate:", "{0:.6f}".format(aset.expected_false_positives()))
print("False positive rate:", "{0:.6f}".format(false_pos))
print("False negative rate:", "{0:.6f}".format(false_neg))
if len(args.outputfile) > 0:
with open(args.outputfile, 'a') as f:
