/
utils.py
68 lines (47 loc) · 1.5 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
"""
placeholder
"""
import heapq
import urllib2
from urlnorm import url_normalize
class PriorityQueue(object):
""" A wrapper over python heapq which maintains uniqueness"""
def __init__(self, ):
self.heap = []
self.hashtable = set()
def __repr__(self):
return str(self.heap)
def add(self, element, priority):
"""
Appends element to heap.
"""
element = url_normalize(element) # only use normalized urls
if element not in self.hashtable:
heapq.heappush(self.heap, (priority, element))
self.hashtable.add(element)
def get(self):
"""
@return tuple
Pops element out from the heap.
"""
return heapq.heappop(self.heap)[1]
class BalerionRedirectHandler(urllib2.HTTPRedirectHandler):
""" add Custom URL redirects handlers here. """
def __init__(self):
pass
def http_error_302(self, req, fp, code, msg, headers):
return fp
http_error_301 = http_error_303 = http_error_307 = http_error_302
class AttrDict(dict):
""" A dict that allows for object-like property access syntax. """
def __init__(self, new_dict=None):
dict.__init__(self)
if new_dict:
self.update(new_dict)
def __getattr__(self, name):
try:
return self[name]
except KeyError:
raise AttributeError(name)
def __setattr__(self, key, value):
self.update({key: value})