def test_get_hash(self):
"""Test we can extract bitly hashes from a bitly URL"""
url_hashes = [("http://bit.ly/Wozuff", "Wozuff"),
("http://bit.ly/Wozuff/", "Wozuff")]
for url, hsh in url_hashes:
returned_hsh = tools.get_hash(url)
self.assertEqual(hsh, returned_hsh)
def get_bitly_links_to_update():
all_links = config.mongo_bitly_links_raw.find()
bitly_links_to_update = []
recent_cutoff = datetime.datetime.utcnow() - datetime.timedelta(hours=config.UPDATE_FROM_N_HOURS_AGO)
for link in all_links:
aggregate_link = link['aggregate_link']
global_hash = tools.get_hash(aggregate_link)
clicks = config.mongo_bitly_clicks.find_one({'global_hash': global_hash})
updated_at = config.A_LONG_TIME_AGO
hash_is_active = True
if clicks:
updated_at = clicks['updated_at']
click_events = clicks['clicks']
if click_events:
hash_is_active = _hash_is_active(updated_at, click_events)
else:
config.logger.warning("This hash has 0 clicks, our logic keeps it alive regardless of its age:" + repr(aggregate_link))
# retain this for debugging but it isn't useful to report during
# production
#if not hash_is_active:
#config.logger.info("THIS HASH IS CONSIDERED TO BE OUT OF DATE:" + repr(aggregate_link))
if updated_at < recent_cutoff and hash_is_active:
bitly_links_to_update.append(aggregate_link)
return bitly_links_to_update
def __sign(self, obj):
if(isinstance(obj, Transaction)):
try:
transaction_content = get_hash(obj)
signature = sign(self.__private_key, transaction_content)
obj.add_signing(signature)
obj.add_hash(transaction_content)
return True
except:
return False
else:
try:
block_content = get_hash(obj)
signature = sign(self.__private_key, block_content)
obj.add_signing(signature)
return True
except:
return False
def __init__(self, name, torrent_hash, torrent_file_name, hash_=None, timeout=2538000):
# default timeout set to 30 days
self.name = name
self.timeout = timeout
self.torrent_hash = torrent_hash
self.torrent_file_name = torrent_file_name
self.hash = hash_
if (self.hash is None) or (len(self.hash)) == 0:
self.hash = tools.get_hash(self.name)
def __sign(self, transaction):
try:
transaction_content = get_hash(transaction)
signature = sign(self.__private_key, transaction_content)
transaction.add_signing(signature)
transaction.add_hash(transaction_content)
return True
except:
return False
def _get_new_link_clicks_then_add_to_mongodb(aggregate_link):
NEW_LINK_CLICKS_UNIT = "hour" # "hour" for hourly clicks, "day" for daily click totals
response = bitly.link_clicks(link=aggregate_link, rollup=False, unit=NEW_LINK_CLICKS_UNIT)
hsh = tools.get_hash(aggregate_link)
document = get_existing_bitly_clicks_for(hsh)
nbr_clicks_before_update = len(document['clicks'])
add_response_to_document(response, document)
nbr_clicks_after_update = len(document['clicks'])
store_bitly_clicks_for(document)
config.logger.debug("Stored {} new clicks (new total {}) with {} bin size for {}".format(nbr_clicks_after_update - nbr_clicks_before_update, nbr_clicks_after_update, NEW_LINK_CLICKS_UNIT, aggregate_link))
def download_file(filepath, url, hash_string):
with open(filepath, "wb") as f:
response = requests.get(url, stream=True)
if not response.ok:
return
for block in response.iter_content(1024):
f.write(block)
file_hash = get_hash(filepath)
if file_hash != hash_string:
raise InvalidFileException()
def download_file(filepath, url, hash_string):
with open(filepath, "wb") as f:
response = requests.get(url, stream=True)
if not response.ok:
return
for block in response.iter_content(1024):
f.write(block)
file_hash = get_hash(filepath)
if file_hash != hash_string:
raise InvalidFileException()
def __init__(self,
name,
torrent_hash,
torrent_file_name,
hash_=None,
timeout=2538000):
# default timeout set to 30 days
self.name = name
self.timeout = timeout
self.torrent_hash = torrent_hash
self.torrent_file_name = torrent_file_name
self.hash = hash_
if (self.hash is None) or (len(self.hash)) == 0:
self.hash = tools.get_hash(self.name)
def download_file(filepath, url, hash_string):
with open(filepath, "wb") as f:
response = requests.get(url, stream=True)
if not response.ok:
return
for block in response.iter_content(1024):
f.write(block)
file_hash = get_hash(filepath)
if file_hash != hash_string:
raise InvalidFileException("file hash does not match for file %s: Expected %s, Got: %s" % (url, hash_string, file_hash))
def download_file(filepath, url, hash_string):
with open(filepath, "wb") as f:
response = requests.get(url, stream=True)
if not response.ok:
return
for block in response.iter_content(1024):
f.write(block)
file_hash = get_hash(filepath)
if file_hash != hash_string:
raise InvalidFileException(
"file hash does not match for file %s: Expected %s, Got: %s" %
(url, hash_string, file_hash))
def get_dir_by_name(self, dir_path, dir_name):
'''
gets a dir from es by id (full_path hashed)
'''
if(not dir_path or not dir_name):
return None
full_path = path.join(dir_path, dir_name)
dir = None
try:
id = tools.get_hash(full_path)
dir = DirResource.get(id, None, index=self.index)
except Exception as ex:
pass
return dir
def get_file_by_path(self, file_path):
'''
get a file from es by id
id is full file name hash
returns a FileResource object
'''
if(not file_path):
return None
file = None
try:
id = tools.get_hash(file_path)
file = FileResource.get(id, None, index=self.index)
except Exception as ex:
pass
return file
def __init__(self, path, destination_name, hash=None, relative_path=None):
"""
:type destination_name : str
:type hash : str
:param hash: hash of the file, if None the hash is computed
:param path: Path of the file
:param destination_name: destination name
"""
File.__init__(self, path)
self.id = None
self.hash = hash
self.destination_name = destination_name
if hash is None:
self.hash = tools.get_hash(self.fullPath)
self.relativePath = path
if relative_path is not None:
self.relativePath = relative_path
def __init__(self, path, destination_name, hash=None, relative_path=None):
"""
:type destination_name : str
:type hash : str
:param hash: hash of the file, if None the hash is computed
:param path: Path of the file
:param destination_name: destination name
"""
File.__init__(self, path)
self.id = None
self.hash = hash
self.destination_name = destination_name
if hash is None:
self.hash = tools.get_hash(self.fullPath)
self.relativePath = path
if relative_path is not None:
self.relativePath = relative_path
def get_clicks_for_domain(domain, filter_from, filter_to):
documents = config.mongo_bitly_links_raw.find({"domain": domain})
hashes = [tools.get_hash(document['aggregate_link']) for document in documents]
counter = Counter()
total_clicks = 0
for global_hash in hashes:
clicks = config.mongo_bitly_clicks.find_one({"global_hash": global_hash})
if clicks is not None:
for date, clicks_per_day in clicks['clicks']:
if filter_from < date and filter_to > date:
items_to_count = [date] * clicks_per_day
counter.update(items_to_count)
total_clicks += clicks_per_day
dates_clicks = counter.items()
dates_clicks.sort()
return dates_clicks, total_clicks
def get_clicks_for_domain_links(domain, filter_from, filter_to):
documents = config.mongo_bitly_links_raw.find({"domain": domain})
hashes = [tools.get_hash(document['aggregate_link']) for document in documents]
clicks_per_link = {}
for global_hash in hashes:
clicks = config.mongo_bitly_clicks.find_one({"global_hash": global_hash})
if clicks:
counts = 0
for date, clicks_per_day in clicks['clicks']:
if filter_from < date and filter_to > date:
#items_to_count = [date] * clicks_per_day
#counter.update(items_to_count)
counts += clicks_per_day
if counts > 0:
clicks_per_link[global_hash] = counts
c = Counter(clicks_per_link)
return c
def save(self, ** kwargs):
# set id as uriHash
self.meta.id = tools.get_hash(self.full_dir_path)
return super(DirResource, self).save(** kwargs)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Project description')
parser.add_argument('--global-hash', '-g', nargs="*", help='Plot a specific global hash(es) e.g. " -g XR1aPQ VWUGMX" ')
parser.add_argument('--all-hashes', '-a', help='Plots all the hashes that we know about')
parser.add_argument('--domain', '-d', help='Graph all results for a domain (e.g. "-d asos.com")')
parser.add_argument('--logy', '-l', action="store_true", default=False, help="Plot y axis with Log scale (defaults to Linear)")
args = parser.parse_args()
fig = plt.figure()
ax = fig.add_subplot(111)
title = "Bit.ly clicks per day"
if args.domain:
documents = config.mongo_bitly_links_raw.find({"domain": args.domain})
hashes = [tools.get_hash(document['aggregate_link']) for document in documents]
plotted_lines = []
for global_hash in hashes:
clicks = config.mongo_bitly_clicks.find_one({"global_hash": global_hash})
#import pdb; pdb.set_trace()
lines = plot_clicks(clicks, ax)
if lines:
plotted_lines += lines
title = "Bit.ly clicks per day for {} hashes for {}".format(len(hashes), args.domain)
dc = DataCursor(plotted_lines)
if args.global_hash:
for global_hash in args.global_hash:
clicks = config.mongo_bitly_clicks.find_one({"global_hash": global_hash})
plot_clicks(clicks, ax)
def test_get_hash2(self):
"""From a search result check we can get a URL hash"""
returned_hsh = tools.get_hash(fixtures.links0["aggregate_link"])
self.assertEqual("Wozuff", returned_hsh)
