def test_get_hash(self): """Test we can extract bitly hashes from a bitly URL""" url_hashes = [("http://bit.ly/Wozuff", "Wozuff"), ("http://bit.ly/Wozuff/", "Wozuff")] for url, hsh in url_hashes: returned_hsh = tools.get_hash(url) self.assertEqual(hsh, returned_hsh)
def get_bitly_links_to_update(): all_links = config.mongo_bitly_links_raw.find() bitly_links_to_update = [] recent_cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=config.UPDATE_FROM_N_HOURS_AGO) for link in all_links: aggregate_link = link['aggregate_link'] global_hash = tools.get_hash(aggregate_link) clicks = config.mongo_bitly_clicks.find_one({'global_hash': global_hash}) updated_at = config.A_LONG_TIME_AGO hash_is_active = True if clicks: updated_at = clicks['updated_at'] click_events = clicks['clicks'] if click_events: hash_is_active = _hash_is_active(updated_at, click_events) else: config.logger.warning("This hash has 0 clicks, our logic keeps it alive regardless of its age:" + repr(aggregate_link)) # retain this for debugging but it isn't useful to report during # production #if not hash_is_active: #config.logger.info("THIS HASH IS CONSIDERED TO BE OUT OF DATE:" + repr(aggregate_link)) if updated_at < recent_cutoff and hash_is_active: bitly_links_to_update.append(aggregate_link) return bitly_links_to_update
def __sign(self, obj): if(isinstance(obj, Transaction)): try: transaction_content = get_hash(obj) signature = sign(self.__private_key, transaction_content) obj.add_signing(signature) obj.add_hash(transaction_content) return True except: return False else: try: block_content = get_hash(obj) signature = sign(self.__private_key, block_content) obj.add_signing(signature) return True except: return False
def __init__(self, name, torrent_hash, torrent_file_name, hash_=None, timeout=2538000): # default timeout set to 30 days self.name = name self.timeout = timeout self.torrent_hash = torrent_hash self.torrent_file_name = torrent_file_name self.hash = hash_ if (self.hash is None) or (len(self.hash)) == 0: self.hash = tools.get_hash(self.name)
def __sign(self, transaction): try: transaction_content = get_hash(transaction) signature = sign(self.__private_key, transaction_content) transaction.add_signing(signature) transaction.add_hash(transaction_content) return True except: return False
def _get_new_link_clicks_then_add_to_mongodb(aggregate_link): NEW_LINK_CLICKS_UNIT = "hour" # "hour" for hourly clicks, "day" for daily click totals response = bitly.link_clicks(link=aggregate_link, rollup=False, unit=NEW_LINK_CLICKS_UNIT) hsh = tools.get_hash(aggregate_link) document = get_existing_bitly_clicks_for(hsh) nbr_clicks_before_update = len(document['clicks']) add_response_to_document(response, document) nbr_clicks_after_update = len(document['clicks']) store_bitly_clicks_for(document) config.logger.debug("Stored {} new clicks (new total {}) with {} bin size for {}".format(nbr_clicks_after_update - nbr_clicks_before_update, nbr_clicks_after_update, NEW_LINK_CLICKS_UNIT, aggregate_link))
def download_file(filepath, url, hash_string): with open(filepath, "wb") as f: response = requests.get(url, stream=True) if not response.ok: return for block in response.iter_content(1024): f.write(block) file_hash = get_hash(filepath) if file_hash != hash_string: raise InvalidFileException()
def download_file(filepath, url, hash_string): with open(filepath, "wb") as f: response = requests.get(url, stream=True) if not response.ok: return for block in response.iter_content(1024): f.write(block) file_hash = get_hash(filepath) if file_hash != hash_string: raise InvalidFileException("file hash does not match for file %s: Expected %s, Got: %s" % (url, hash_string, file_hash))
def download_file(filepath, url, hash_string): with open(filepath, "wb") as f: response = requests.get(url, stream=True) if not response.ok: return for block in response.iter_content(1024): f.write(block) file_hash = get_hash(filepath) if file_hash != hash_string: raise InvalidFileException( "file hash does not match for file %s: Expected %s, Got: %s" % (url, hash_string, file_hash))
def get_dir_by_name(self, dir_path, dir_name): ''' gets a dir from es by id (full_path hashed) ''' if(not dir_path or not dir_name): return None full_path = path.join(dir_path, dir_name) dir = None try: id = tools.get_hash(full_path) dir = DirResource.get(id, None, index=self.index) except Exception as ex: pass return dir
def get_file_by_path(self, file_path): ''' get a file from es by id id is full file name hash returns a FileResource object ''' if(not file_path): return None file = None try: id = tools.get_hash(file_path) file = FileResource.get(id, None, index=self.index) except Exception as ex: pass return file
def __init__(self, path, destination_name, hash=None, relative_path=None): """ :type destination_name : str :type hash : str :param hash: hash of the file, if None the hash is computed :param path: Path of the file :param destination_name: destination name """ File.__init__(self, path) self.id = None self.hash = hash self.destination_name = destination_name if hash is None: self.hash = tools.get_hash(self.fullPath) self.relativePath = path if relative_path is not None: self.relativePath = relative_path
def get_clicks_for_domain(domain, filter_from, filter_to): documents = config.mongo_bitly_links_raw.find({"domain": domain}) hashes = [tools.get_hash(document['aggregate_link']) for document in documents] counter = Counter() total_clicks = 0 for global_hash in hashes: clicks = config.mongo_bitly_clicks.find_one({"global_hash": global_hash}) if clicks is not None: for date, clicks_per_day in clicks['clicks']: if filter_from < date and filter_to > date: items_to_count = [date] * clicks_per_day counter.update(items_to_count) total_clicks += clicks_per_day dates_clicks = counter.items() dates_clicks.sort() return dates_clicks, total_clicks
def get_clicks_for_domain_links(domain, filter_from, filter_to): documents = config.mongo_bitly_links_raw.find({"domain": domain}) hashes = [tools.get_hash(document['aggregate_link']) for document in documents] clicks_per_link = {} for global_hash in hashes: clicks = config.mongo_bitly_clicks.find_one({"global_hash": global_hash}) if clicks: counts = 0 for date, clicks_per_day in clicks['clicks']: if filter_from < date and filter_to > date: #items_to_count = [date] * clicks_per_day #counter.update(items_to_count) counts += clicks_per_day if counts > 0: clicks_per_link[global_hash] = counts c = Counter(clicks_per_link) return c
def save(self, ** kwargs): # set id as uriHash self.meta.id = tools.get_hash(self.full_dir_path) return super(DirResource, self).save(** kwargs)
if __name__ == "__main__": parser = argparse.ArgumentParser(description='Project description') parser.add_argument('--global-hash', '-g', nargs="*", help='Plot a specific global hash(es) e.g. " -g XR1aPQ VWUGMX" ') parser.add_argument('--all-hashes', '-a', help='Plots all the hashes that we know about') parser.add_argument('--domain', '-d', help='Graph all results for a domain (e.g. "-d asos.com")') parser.add_argument('--logy', '-l', action="store_true", default=False, help="Plot y axis with Log scale (defaults to Linear)") args = parser.parse_args() fig = plt.figure() ax = fig.add_subplot(111) title = "Bit.ly clicks per day" if args.domain: documents = config.mongo_bitly_links_raw.find({"domain": args.domain}) hashes = [tools.get_hash(document['aggregate_link']) for document in documents] plotted_lines = [] for global_hash in hashes: clicks = config.mongo_bitly_clicks.find_one({"global_hash": global_hash}) #import pdb; pdb.set_trace() lines = plot_clicks(clicks, ax) if lines: plotted_lines += lines title = "Bit.ly clicks per day for {} hashes for {}".format(len(hashes), args.domain) dc = DataCursor(plotted_lines) if args.global_hash: for global_hash in args.global_hash: clicks = config.mongo_bitly_clicks.find_one({"global_hash": global_hash}) plot_clicks(clicks, ax)
def test_get_hash2(self): """From a search result check we can get a URL hash""" returned_hsh = tools.get_hash(fixtures.links0["aggregate_link"]) self.assertEqual("Wozuff", returned_hsh)