def make_hash(body): body = re.sub('[^a-z]', '', body.lower()) if body == '': return None return CityHash64(body)
def city_hash_uint64(string): if type(string) is not str: string = str(string) out = CityHash64(string) & 0xffffffff if (out > 0x7fFFffFF): out -= 0x100000000 return out
def shingle(self, words): shingles = set() for index in range(0, len(words) - (self.shingle_size - 1)): shingle = " ".join(words[index:index + self.shingle_size]) shingles.add(str(CityHash64(shingle))) return shingles
def set_motifs(self, motifs): self.motifs = motifs self.motif_ids = [m.id for m in read_motifs(open(motifs))] self.checksum = {} if self.use_cache: chksum = CityHash64("\n".join(sorted(self.motif_ids))) self.checksum[self.motifs] = chksum
def inspect_packet(p): aux_id = str(p.src_ip) + str(p.dst_ip) + str(p.src_port) + str(p.dst_port) aux_id_revr = str(p.dst_ip) + str(p.src_ip) + str(p.dst_port) + str( p.src_port) connId = CityHash64(aux_id) connId_revr = CityHash64(aux_id_revr) if debug_showdata == True: print 'ConnId: ' + str(connId) print 'ConnId_r: ' + str(connId_revr) if p.direction == const.INBOUND: print "packet inbound" else: print "packet outbound" #Access to these global variables global flows global connNum global learn_protocol_on global detect_protocol_on #pdb.set_trace() if connId in flows: update_flow(flows[connId], p) if learn_protocol_on: learning(flows[connId]) elif detect_protocol_on: inspect_flow(flows[connId]) return False elif connId not in flows and connId_revr in flows: update_flow(flows[connId_revr], p) if learn_protocol_on: learning(flows[connId_revr]) elif detect_protocol_on: inspect_flow(flows[connId_revr]) return False else: #New connection - We add first packet (p) to new flow (flow) flow = types.Flow() update_flow(flow, p) flows.update({connId: flow}) #Adding 5tuple info to the flow add_5tuple_info(flows[connId], p) print "New connection established. ConnId: %i" % (connId) return True """
def _scan_sequences(self, seqs, nreport, scan_rc, cutoff=0.95): motif_file = self.motifs motif_digest = self.checksum.get(motif_file, None) scan_seqs = seqs if self.use_cache: # determine which sequences are not in the cache hashes = dict([(s.upper(), CityHash64(s.upper())) for s in seqs]) scan_seqs = [] for seq, seq_hash in hashes.items(): key = str((seq_hash, motif_digest, nreport, scan_rc, cutoff)) ret = self.cache.get(key) if ret == NO_VALUE or ret is None: scan_seqs.append(seq.upper()) # scan the sequences that are not in the cache if len(scan_seqs) > 0: n = 12 motifs = load_motifs(motif_file, cutoff) scan_func = partial(scan_seq_mult, motifs=motifs, nreport=nreport, scan_rc=scan_rc) jobs = [] chunksize = len(scan_seqs) / n + 1 for i in range(n): job = pool.apply_async( scan_func, (scan_seqs[i * chunksize:(i + 1) * chunksize], )) jobs.append(job) # store values in cache i = 0 for job in jobs: for ret in job.get(): if self.use_cache: h = hashes[scan_seqs[i]] key = str((h, motif_digest, nreport, scan_rc, cutoff)) self.cache.set(key, ret) else: yield ret i += 1 if self.use_cache: # return results from cache for seq in seqs: key = str((hashes[seq.upper()], motif_digest, nreport, scan_rc, cutoff)) ret = self.cache.get(key) if ret == NO_VALUE or ret is None: raise Exception("cache is not big enough to hold all " "results, try increasing the cache size " "or disable cache") yield ret
def chash(obj): """Convenience function for calling CityHash64 :param obj: input string/hashable object :type obj: object :return: integer :rtype: int """ return long2int(CityHash64(obj))
def _create_hash(string: str) -> str: """Create CityHash64 bit hash of string Args: string (str): string to create CityHash64 from Returns: str: CityHash64 """ return str(CityHash64(string))
def generate_strings() -> Iterable[Tuple[str, int]]: pos = 0 while True: pos += (500000000 + (10000000 - random.randint(1, 20000000))) shift = pos ret = [] while shift > 0: ret.insert(0, ALPHABET[shift % (END - START)]) shift = floor(shift / (END - START)) id = "".join(ret) yield (id, CityHash64(id))
def add(self, bot, update, args): """ Adds a rss subscription to user """ telegram_user = update.message.from_user if len(args) != 2: message = "Sorry! I could not add the entry! Please use the the command passing the following arguments:\n\n /add <url> <entryname> \n\n Here is a short example: \n\n /add http://www.feedforall.com/sample.xml ExampleEntry" update.message.reply_text(message) return arg_url = args[0] arg_entry = args[1] # Check if argument matches url format feed=FeedHandler.is_parsable(url=arg_url) if not feed: message = f"Sorry! It seems like {arg_url}" + \ "' doesn't provide an RSS news feed... Have you tried another URL from that provider?" update.message.reply_text(message) return # Check if entry does not exists entries = self.db.get_urls_for_user(telegram_id=telegram_user.id) if any(arg_url.lower() in entry.lower() for entry,_ in entries): message = f"Sorry, {telegram_user.first_name}" + \ "! I already have that url stored in your subscriptions." update.message.reply_text(message) return if any(arg_entry in entry for entry in entries): message = f"Sorry! I already have an entry with name {arg_entry}" + \ " stored in your subscriptions.. Please choose another entry name or delete the entry using '/remove {arg_entry}'" update.message.reply_text(message) return urls = self.db.get_all_urls() if not (arg_url in urls): items = {} for item in feed: for key in ['summary', 'title', 'link']: if not(key in item.keys()): item[key]='' hash=CityHash64(item['summary']+item['title']+item['link']) items[hash] = {'active': True, 'last_date': DateHandler.get_datetime_now()} self.db.add_url(url=arg_url, items=items) self.db.add_user_bookmark(telegram_id=telegram_user.id, url=arg_url, alias=arg_entry) message = f"I successfully added {arg_entry} to your subscriptions!" update.message.reply_text(message)
def _fill_example_ids(self, eids, slide_cache): sampled_hids = [] max_idx = -1 for idx, eid in eids: if idx <= slide_cache.get_stats_index(): continue max_idx = idx hkey = '{}{}'.format(self._hash_prefix, eid) hid = CityHash64(hkey) if hid % JoinerStats._SampleRateReciprocal == 0: sampled_hids.append(hid) if max_idx >= 0: return slide_cache.fill_hash_ids(max_idx, sampled_hids) return []
def update_feed(self, url): try: feed = FeedHandler.parse_feed(url[0]) except: feed = False traceback.print_exc() # ??? if feed: print(f'{url[0]}:') print(f'Longitud de feed: {len(feed)}') url_items = self.db.get_url_items(url=url[0]) for item in url_items: url_items[item]['active'] = False new_items = [] for item in feed: hash = str( CityHash64(item['summary'] + item['title'] + item['link'])) if not (hash in url_items): new_items.append(item) url_items[hash] = { 'active': True, 'last_date': DateHandler.get_datetime_now() } for item, value in url_items.copy().items(): if not value['active']: print(f'Desactivando {item}') if not value['active'] and DateHandler.is_older_than_days( value['last_date'], 5): print(f'Borrando {item}') url_items.pop(item) self.db.update_url_items(url=url[0], items=url_items) telegram_users = self.db.get_users_for_url(url=url[0]) for user in telegram_users: if user[6]: # is_active if not feed: message = "Something went wrong when I tried to parse the URL: \n\n " + \ url[0] + "\n\nCould you please check that for me? Remove the url from your subscriptions using the /remove command, it seems like it does not work anymore!" self.bot.send_message(chat_id=user[0], text=message, parse_mode=ParseMode.HTML) return for post in new_items: self.send_message(post=post, user=user)
def hash64(to_hash: str) -> int: """ Create int64 hash from input. Compatible with postgres BIGINT as the hash is signed. Parameters ---------- to_hash : str String to hash Returns ------- int Hashed input as int64 """ return CityHash64(to_hash) - ((2**63) - 1)
def compute_hash64(text: Union[str, pd.Series]) -> Union[int, pd.Series]: """Compute consistent 64-bit signed integer for text or texts""" if isinstance(text, str): text_hash = c_long(CityHash64(text)).value elif isinstance(text, pd.Series): text_hash = text.apply(CityHash64) text_hash = text_hash.astype( np.int64) if text_hash.dtype == np.uint64 else text_hash if text_hash.dtype != np.int64: raise ValueError( f'Computed hashes must be of type np.int64 instead of {text_hash.dtype}!' ) else: raise TypeError( f"Input must of type str or Series, but got {type(text)}!") return text_hash
def hash_nanopub(nanopub: Mapping[str, Any]) -> str: """Create CityHash64 from nanopub for duplicate check TODO - check that this hash value is consistent between C# and Python running on laptop and server Build string to hash Collect flat array of (all values.strip()): nanopub.type.name nanopub.type.version One of: nanopub.citation.database.name nanopub.citation.database.id OR nanopub.citation.database.uri OR nanopub.citation.database.reference Extend with sorted list of assertions (SRO as single string with space between S, R and O) Extend with sorted list of annotations (nanopub.annotations.type + ' ' + nanopub.annotations.id) Convert array to string by joining array elements separated by a space Create CityHash64(str) and return """ hash_list = [] # Type hash_list.append(nanopub["nanopub"]["type"].get("name", "").strip()) hash_list.append(nanopub["nanopub"]["type"].get("version", "").strip()) # Citation if nanopub["nanopub"]["citation"].get("database", False): hash_list.append(nanopub["nanopub"]["citation"]["database"].get( "name", "").strip()) hash_list.append(nanopub["nanopub"]["citation"]["database"].get( "id", "").strip()) elif nanopub["nanopub"]["citation"].get("uri", False): hash_list.append(nanopub["nanopub"]["citation"].get("uri", "").strip()) elif nanopub["nanopub"]["citation"].get("reference", False): hash_list.append(nanopub["nanopub"]["citation"].get("reference", "").strip()) # Assertions assertions = [] for assertion in nanopub["nanopub"]["assertions"]: if assertion.get("relation") is None: assertion["relation"] = "" if assertion.get("object") is None: assertion["object"] = "" assertions.append(" ".join(( assertion["subject"].strip(), assertion.get("relation", "").strip(), assertion.get("object", "").strip(), )).strip()) assertions = sorted(assertions) hash_list.extend(assertions) # Annotations annotations = [] for anno in nanopub["nanopub"]["annotations"]: annotations.append(" ".join( (anno.get("type", "").strip(), anno.get("id", "").strip())).strip()) annotations = sorted(annotations) hash_list.extend(annotations) np_string = " ".join([l.lower() for l in hash_list]) return "{:x}".format(CityHash64(np_string))
def _oneway_hash_list(items): return [hex(CityHash64(str(item)))[2:] for item in items]
def generate_uuids() -> Iterable[Tuple[str, int]]: while True: unique_id = str(uuid.uuid4()) hash = CityHash64(unique_id) yield (unique_id, hash)
def hash_nanopub(nanopub: Mapping[str, Any]) -> str: """Create CityHash64 from nanopub for duplicate check TODO - check that this hash value is consistent between C# and Python running on laptop and server Build string to hash Collect flat array of (all values.strip()): nanopub.type.name nanopub.type.version One of: nanopub.citation.database.name nanopub.citation.database.id OR nanopub.citation.database.uri OR nanopub.citation.database.reference Extend with sorted list of assertions (SRO as single string with space between S, R and O) Extend with sorted list of annotations (nanopub.annotations.type + ' ' + nanopub.annotations.id) Convert array to string by joining array elements separated by a space Create CityHash64(str) and return """ hash_list = [] # Type hash_list.append(nanopub['nanopub']['type'].get('name', '').strip()) hash_list.append(nanopub['nanopub']['type'].get('version', '').strip()) # Citation if nanopub['nanopub']['citation'].get('database', False): hash_list.append(nanopub['nanopub']['citation']['database'].get('name', '').strip()) hash_list.append(nanopub['nanopub']['citation']['database'].get('id', '').strip()) elif nanopub['nanopub']['citation'].get('uri', False): hash_list.append(nanopub['nanopub']['citation'].get('uri', '').strip()) elif nanopub['nanopub']['citation'].get('reference', False): hash_list.append(nanopub['nanopub']['citation'].get('reference', '').strip()) # Assertions assertions = [] for assertion in nanopub['nanopub']['assertions']: if assertion.get('relation') is None: assertion['relation'] = '' if assertion.get('object') is None: assertion['object'] = '' assertions.append(' '.join((assertion['subject'].strip(), assertion.get('relation', '').strip(), assertion.get('object', '').strip())).strip()) assertions = sorted(assertions) hash_list.extend(assertions) # Annotations annotations = [] for anno in nanopub['nanopub']['annotations']: annotations.append(' '.join((anno.get('type', '').strip(), anno.get('id', '').strip())).strip()) annotations = sorted(annotations) hash_list.extend(annotations) np_string = ' '.join([l.lower() for l in hash_list]) return '{:x}'.format(CityHash64(np_string))
def partition(self, key, partitions=None): if not partitions: partitions = self.partitions value = CityHash64(key) idx = value % len(partitions) return partitions[idx]
def str2hashnumber(x: str): return CityHash64(x)
from cityhash import CityHash32, CityHash64, CityHash128 except ImportError: print("CityHash not installed, pip install cityhash") else: # cityhash32 pt = time() for i in range(num): key = b"myval-%d" % i CityHash32(b"%s" % key) print("CityHash32 took {} s".format(time() - pt)) # cityhash64 pt = time() for i in range(num): key = b"myval-%d" % i CityHash64(b"%s" % key) print("CityHash64 took {} s".format(time() - pt)) # cityhash128 pt = time() for i in range(num): key = b"myval-%d" % i CityHash128(b"%s" % key) print("CityHash128 took {} s".format(time() - pt)) # murmurhash v3 try: import mmh3 except ImportError: print("MurmurHash v3 not installed, pip install murmurhash3") else:
def make_cityhash(): return CityHash64(sys.argv[1] + sys.argv[2] + sys.argv[3])
def hash_uri(uri_string): '''broken out 64bit hashing function, so it's easy to replace. Right now we use Google's fast crawler hash function CityHash, the 64 bit version. Returns a 64 bit hash based on an input (uri) string.''' return CityHash64(uri_string)
def get_hash(text) -> str: return str(CityHash64(text))
from util.database import DatabaseHandler from util.feedhandler import FeedHandler from util.datehandler import DateHandler from cityhash import CityHash64 db = DatabaseHandler("resources/datastore.db") arg_url = 'http://yle.fi/uutiset/rss/paauutiset.rss' feed = FeedHandler.is_parsable(url=arg_url) items = {} for item in feed: hash = CityHash64(item['summary'] + item['title'] + item['link']) if (hash in items): print(item['link'], item['summary'], items[hash]) items[hash] = { 'active': True, 'last_date': DateHandler.get_datetime_now(), 'link': item['link'] } #self.db.add_url(url=arg_url, items=items) url_items = db.get_url_items(url=arg_url) for item in url_items: url_items[item]['active'] = False new_items = [] for item in feed: hash = CityHash64(item['summary'] + item['title'] + item['link']) if not (str(hash) in url_items):
def get_hash(text) -> str: """Hash the given text using `CityHash64 <https://opensource.googleblog.com/2011/04/introducing-cityhash.html>`_. """ return str(CityHash64(text))
def chs64(s): return str(CityHash64(s))
from cityhash import CityHash32, CityHash64, CityHash128 except ImportError: print('CityHash not installed, pip install cityhash') else: # cityhash32 pt = time() for i in range(num): key = b'myval-%d' % i CityHash32(b'%s' % key) print('CityHash32 took {} s'.format(time() - pt)) # cityhash64 pt = time() for i in range(num): key = b'myval-%d' % i CityHash64(b'%s' % key) print('CityHash64 took {} s'.format(time() - pt)) # cityhash128 pt = time() for i in range(num): key = b'myval-%d' % i CityHash128(b'%s' % key) print('CityHash128 took {} s'.format(time() - pt)) # murmurhash v3 try: import mmh3 except ImportError: print('MurmurHash v3 not installed, pip install murmurhash3') else: