class LRUCache(object):
def __init__(self, capacity):
"""
:type capacity: int
"""
self.dictionary = OrderedDict()
self.capacity = capacity
def get(self, key):
"""
:rtype: int
"""
if key in self.dictionary:
temp = self.dictionary.pop(key)
self.dictionary[key] = temp
return temp
else: return -1
def set(self, key, value):
"""
:type key: int
:type value: int
:rtype: nothing
"""
if key in self.dictionary:
self.dictionary.pop(key)()
self.dictionary[key] = value
else:
if len(self.dictionary) >= self.capacity:
self.dictionary.popitem(last=False)
self.dictionary[key] = value
else:
self.dictionary[key] = value
class MotionMonitor(object):
def __init__(self):
self.__logger = logging.getLogger("%s.%s" % (self.__class__.__module__, self.__class__.__name__))
self.config = motionmonitor.config
self.bus = EventBus(self)
self._jobs = OrderedDict()
self.bus.listen(EVENT_JOB, self.job_handler)
self.__socket_listener = motionmonitor.socketlistener.SocketListener(self)
self.__camera_monitor = motionmonitor.cameramonitor.CameraMonitor(self)
self.__zabbixwriter = motionmonitor.extensions.zabbixwriter.ZabbixWriter(self)
self.__sqlwriter = motionmonitor.sqlexchanger.SQLWriter(self)
# This is the sweeper and auditor.
self.__sweeper = motionmonitor.filemanager.Sweeper(self)
self.__auditor = motionmonitor.filemanager.Auditor(self)
self.__json_interface = motionmonitor.jsoninterface.JSONInterface(self, self.__camera_monitor)
self.__logger.info("Initialised...")
def job_handler(self, event):
self.__logger.debug("Handling a job event: {}".format(event))
job = event.data
self._jobs[job.id] = job
while (len(self._jobs) > MAX_JOBQ_SIZE):
self.__logger.debug("Too many jobs in the queue, popping the oldest")
self._jobs.popitem(False)
class MemoryRegion(CacheRegion):
NO_VALUE = {}
def __init__(self, max_keys):
self.max_keys = max_keys
self._cache = OrderedDict()
def get(self, key):
value = self._cache.get(key, self.NO_VALUE)
if value is self.NO_VALUE:
return False, None
else:
return True, value
def set(self, key, value):
if key in self._cache:
getLogger('pymor.core.cache.MemoryRegion').warn('Key already present in cache region, ignoring.')
return
if len(self._cache) == self.max_keys:
self._cache.popitem(last=False)
self._cache[key] = value
def clear(self):
self._cache = OrderedDict()
class LRUCache(object):
def __init__(self, capacity):
"""
:type capacity: int
"""
self.capacity = capacity
self.cache = OrderedDict()
def get(self, key):
"""
:rtype: int
"""
try:
value = self.cache.pop(key)
self.cache[key] = value
except:
value = -1
return value
def set(self, key, value):
"""
:type key: int
:type value: int
:rtype: nothing
"""
try:
self.cache.pop(key)
except:
if len(self.cache) == self.capacity:
self.cache.popitem(last=False)
self.cache[key] = value
class _SlideCache(object):
def __init__(self, cache_size, dz_opts):
self.cache_size = cache_size
self.dz_opts = dz_opts
self._lock = Lock()
self._cache = OrderedDict()
def get(self, path):
with self._lock:
if path in self._cache:
# Move to end of LRU
slide = self._cache.pop(path)
self._cache[path] = slide
return slide
osr = OpenSlide(path)
slide = DeepZoomGenerator(osr, **self.dz_opts)
try:
mpp_x = osr.properties[openslide.PROPERTY_NAME_MPP_X]
mpp_y = osr.properties[openslide.PROPERTY_NAME_MPP_Y]
slide.mpp = (float(mpp_x) + float(mpp_y)) / 2
except (KeyError, ValueError):
slide.mpp = 0
with self._lock:
if path not in self._cache:
if len(self._cache) == self.cache_size:
self._cache.popitem(last=False)
self._cache[path] = slide
return slide
class AliasGenerator(object):
def __init__(self, size=200):
self.MAX_CACHE = size # TODO: Make configurable
self.cache = OrderedDict()
def generate_alias(self, str_value):
"""
Generate an alias for the given string.
:param str_value: the given string
:return: a 24 character alias string
"""
alias = self.get_alias(str_value)
if not alias:
h = hashlib.md5()
h.update(str_value.encode('utf-8'))
alias = h.hexdigest()
self.cache[str_value] = alias
if len(self.cache) > self.MAX_CACHE:
self.cache.popitem(last=False) # FIFO
else:
alias = alias
return alias
def needs_alias(self, str_value):
return len(str_value) > MAX_NAME_LEN or len(RE_CLEANSE.findall(str_value)) > 0
def get_alias(self, str_value):
return self.cache.get(str_value)
def clear_aliases(self):
self.cache.clear()
class Cache(object):
def __init__(self):
self.reset_caches()
def reset_caches(self):
self._category_cache = OrderedDict()
self._search_cache = OrderedDict()
def search_cache(self, search):
old = self._search_cache.pop(search, None)
if old is None or old[0] <= self.db.last_modified():
matches = self.search_for_books(search) or []
self._search_cache[search] = (utcnow(), frozenset(matches))
if len(self._search_cache) > 50:
self._search_cache.popitem(last=False)
else:
self._search_cache[search] = old
return self._search_cache[search][1]
def categories_cache(self, restrict_to=frozenset([])):
base_restriction = self.search_cache('')
if restrict_to:
restrict_to = frozenset(restrict_to).intersection(base_restriction)
else:
restrict_to = base_restriction
old = self._category_cache.pop(frozenset(restrict_to), None)
if old is None or old[0] <= self.db.last_modified():
categories = self.db.get_categories(ids=restrict_to)
self._category_cache[restrict_to] = (utcnow(), categories)
if len(self._category_cache) > 20:
self._category_cache.popitem(last=False)
else:
self._category_cache[frozenset(restrict_to)] = old
return self._category_cache[restrict_to][1]
def convert_dn_to_cobra(dn):
# Taken from https://gist.github.com/mtimm/7c238d1c682a684867fd
from cobra.mit.naming import Dn
cobra_dn = Dn.fromString(dn)
parentMoOrDn = "''"
dn_dict = OrderedDict()
for rn in cobra_dn.rns:
rn_str = str(rn)
dn_dict[rn_str] = {}
dn_dict[rn_str]['namingVals'] = tuple(rn.namingVals)
dn_dict[rn_str]['moClassName'] = rn.meta.moClassName
dn_dict[rn_str]['className'] = rn.meta.className
dn_dict[rn_str]['parentMoOrDn'] = parentMoOrDn
parentMoOrDn = rn.meta.moClassName
code = []
dn_dict.popitem()
for arn in dn_dict.items():
if len(arn[1]['namingVals']) > 0:
nvals_str = ", '" + ", ".join(map(str, arn[1]['namingVals'])) + "'"
else:
nvals_str = ""
code.append("{0} = {1}({2}{3})".format(arn[1]['moClassName'],
arn[1]['className'],
arn[1]['parentMoOrDn'],
nvals_str))
return arn[1]['moClassName'], '\n'.join(code)
class LRUCache:
# @param capacity, an integer
def __init__(self, capacity):
self.capacity = capacity
# use OrderedDict to simulate LRU cache
self.cache = OrderedDict()
def get(self, key):
if key in self.cache:
value = self.cache[key]
# remove key then add it again
del self.cache[key]
self.cache[key] = value
return value
else:
return ""
def set(self, key, value):
if key in self.cache:
# remove key if already in cache
del self.cache[key]
self.cache[key] = value
# pop an item (oldest) if cache is full
if len(self.cache) > self.capacity:
self.cache.popitem(False)
class Cache(object):
def __init__(self, size_limit):
super(Cache, self).__init__()
self._c = OrderedDict()
self._size_limit = size_limit
def insert(self, k, v):
"""
add a key and value to the front
"""
self._c[k] = v
if len(self._c) > self._size_limit:
self._c.popitem(last=False)
def exists(self, k):
return k in self._c
def touch(self, k):
"""
bring a key to the front
"""
v = self._c[k]
del self._c[k]
self._c[k] = v
def get(self, k):
return self._c[k]
class LRUCache(object):
def __init__(self, capacity):
"""
:type capacity: int
"""
self.c = capacity
self.mapi = OrderedDict()
def get(self, key):
"""
:rtype: int
"""
if key in self.mapi:
ret = self.mapi.pop(key)
self.mapi[key] = ret
return ret
return -1
def set(self, key, value):
"""
:type key: int
:type value: int
:rtype: nothing
"""
if key in self.mapi:
self.mapi.pop(key)
self.mapi[key] = value
if len(self.mapi) > self.c:
self.mapi.popitem(last=False)
class LRUCache:
# @param capacity, an integer
def __init__(self, capacity):
self.capacity = capacity
self.history = OrderedDict()
# @return an integer
def get(self, key):
if key in self.history:
self.set(key, self.history[key])
return self.history[key]
else:
return -1
# @param key, an integer
# @param value, an integer
# @return nothing
def set(self, key, value):
if key in self.history:
del self.history[key]
self.history[key] = value
else:
if len(self.history) == self.capacity:
self.history.popitem(False)
self.history[key] = value
def findSubstring(self, s, words):
sets = OrderedDict()
dict = defaultdict(int)
for i in words:
dict[i] += 1
width = len(words[0])
start = 0
res = []
for i in range(width, len(s) + 1, width):
# print i, sets
tmp = s[i - width : i]
if tmp in words:
if tmp in sets:
if sets[tmp][1] > dict[tmp]:
sets[tmp] = (i, sets[tmp][1] + 1)
last = sets.popitem(last=False)
while last[0] != tmp:
last = sets.popitem(last=False)
start = last[1][0]
else:
sets[tmp] = (i, 1)
if len(sets) == len(words):
res.append(start)
sets.popitem(last=False)
else:
start = i
sets = OrderedDict()
return res
class LRUCache:
# @param capacity, an integer
def __init__(self, capacity):
self.capacity = capacity
self.cache = OrderedDict()
# @return an integer
def get(self, key):
if key in self.cache:
value = self.cache[key]
del self.cache[key]
self.cache[key] = value
return value
return -1
# @param key, an integer
# @param value, an integer
# @return nothing
def set(self, key, value):
if key in self.cache:
del self.cache[key]
elif len(self.cache) >= self.capacity:
self.cache.popitem(False)
self.cache[key] = value
class SIMCache(object):
def __init__(self, size, ttl):
self.cache = OrderedDict()
self.size = size
self.ttl = ttl
def set_key(self, key, value, ttl=None):
"""Set a key value in the cache with its expiration time.
If no ttl (in seconds) is provided self.ttl is taken by default.
If cache length exceeds CACHE_SIZE when adding a key, the oldest (first inserted) key is removed (FIFO)
"""
self.cache[key] = (time.time() + (ttl or self.ttl), value)
if len(self.cache) > self.size:
self.cache.popitem(last=False)
def get_key(self, key):
"""Retrieve a key value from the cache.
Returns None if does not exist or the key expired.
If the key expired it is removed from the cache.
"""
content = self.cache.get(key, None)
if content:
if content[0] > time.time():
return content[1]
else:
del self.cache[key]
return None
def clear_keys(self, ):
"""Remove all cache keys content
"""
self.cache = OrderedDict()
class LRUCache(OrderedDict):
"""
#实现一个LRU算法的dict，用于记录order_id
#避免storm平台重发log记录，造成redis数据偏大！
"""
def __init__(self, capacity):
"""
@param capacity 字典长度
"""
self.capacity = capacity
self.cache = OrderedDict()
def get(self,key):
if self.cache.has_key(key):
value = self.cache.pop(key)
self.cache[key] = value
else:
value = None
return value
def set(self,key,value):
if self.cache.has_key(key):
value = self.cache.pop(key)
self.cache[key] = value
else:
if len(self.cache) == self.capacity:
#pop出第一个item
self.cache.popitem(last = False)
self.cache[key] = value
else:
self.cache[key] = value
class LRUCache_146(object):
def __init__(self, capacity):
"""
:type capacity: int
"""
self.capacity = capacity
self.cache = OrderedDict()
def get(self, key):
"""
:rtype: int
"""
if key in self.cache:
value = self.cache[key]
del self.cache[key]
self.cache[key] = value
return value
return -1
def set(self, key, value):
"""
:type key: int
:type value: int
:rtype: nothing
"""
if key in self.cache:
del self.cache[key]
self.cache[key] = value
if len(self.cache) > self.capacity:
self.cache.popitem(False)
def dict_to_object(self, dd):
''' Instantiates the object using the encoded data
:param dd: dictionary with all memberattributes necessary to construct the object
:type obj: OrderedDict
:returns: encoded object
:rtype: type of the object
'''
#print("___ OrderedDecoder.dict_to_object(dd) ___")
#print(dd)
d = OrderedDict(dd)
class_name = d.popitem(last=False)[1]
module_name = d.popitem(last=False)[1]
#print("--- class_name", class_name, "module_name", module_name)
module = __import__(module_name)
#print("--- module:", module)
#class_ is of type 'type'
class_type = getattr(module, class_name)
#print("--- type(class_type):" , type(class_type))
#print("--- class_type:", class_type)
#print("--- d.items()", d.items())
args = dict(d.items())
#print("--- args:", args)
inst = class_type(args)
#print("==============\n")
return inst
class BTB:
def __init__(self,w,n,default):
self.btb = OrderedDict()
self.width = w
self.num_entries = n
self.default_pkt = default
print "BTB : %d entries" % self.num_entries
def predict(self, pc):
# print (("pred pc: 0x%x : " % pc) + self.__str__())
if pc in self.btb:
return (True, self.btb[pc])
else:
return (False, self.default_pkt)
def update(self, pc, taken, target):
if (not taken):
return
self.btb[pc] = target
if len(self.btb) > self.num_entries:
self.btb.popitem(last=False)
def __str__(self):
string = "("
for pc in self.btb:
string += ("(0x%x,0x%x.." % (pc, self.btb[pc][0]))
string += ")"
return string
class LRUDict(MutableMapping):
"""
A dictionary of limited size where items are evicted in LRU-order
inspired by http://stackoverflow.com/a/2438926
"""
def __init__(self, size, *args, **kwargs):
self.size = size
self.dict = OrderedDict(*args, **kwargs)
while len(self) > self.size:
self.dict.popitem(last=False)
def __iter__(self):
return iter(self.dict)
def __len__(self):
return len(self.dict)
def __getitem__(self, key):
return self.dict[key]
def __setitem__(self, key, value):
if key not in self and len(self) == self.size:
self.dict.popitem(last=False)
if key in self: # need to delete and reinsert to maintain order
del self[key]
self.dict[key] = value
def __delitem__(self, key):
del self.dict[key]
class LimitedMemoryBackend(BasicInterface, dc.api.CacheBackend):
def __init__(self, argument_dict):
'''If argument_dict contains a value for max_kbytes this the total memory limit in kByte that is enforced on the
internal cache dictionary, otherwise it's set to sys.maxint.
If argument_dict contains a value for max_keys this maximum amount of cache values kept in the
internal cache dictionary, otherwise it's set to sys.maxlen.
If necessary values are deleted from the cache in FIFO order.
'''
self.logger.debug('LimitedMemoryBackend args {}'.format(pformat(argument_dict)))
self._max_keys = argument_dict.get('max_keys', sys.maxsize)
self._max_bytes = argument_dict.get('max_kbytes', sys.maxint / 1024) * 1024
self._cache = OrderedDict()
def get(self, key):
return self._cache.get(key, dc.api.NO_VALUE)
def print_limit(self, additional_size=0):
self.logger.info('LimitedMemoryBackend at {}({}) keys -- {}({}) Byte'
.format(len(self._cache), self._max_keys,
memory.getsizeof(self._cache) / 8, self._max_bytes))
def _enforce_limits(self, new_value):
additional_size = memory.getsizeof(new_value) / 8
while len(self._cache) > 0 and not (len(self._cache) <= self._max_keys and
(memory.getsizeof(self._cache) + additional_size) / 8 <= self._max_bytes):
self.logger.debug('shrinking limited memory cache')
self._cache.popitem(last=False)
def set(self, key, value):
self._enforce_limits(value)
self._cache[key] = value
def delete(self, key):
self._cache.pop(key)
def get_treasury_source():
url = """\
http://data.treasury.gov/feed.svc/DailyTreasuryYieldCurveRateData\
"""
res = requests.get(url, stream=True)
stream = iter_to_stream(res.iter_lines())
elements = ET.iterparse(stream, ("end", "start-ns", "end-ns"))
namespaces = OrderedDict()
properties_xpath = [""]
def updated_namespaces():
if "" in namespaces and "m" in namespaces:
properties_xpath[0] = "{%s}content/{%s}properties" % (namespaces[""], namespaces["m"])
else:
properties_xpath[0] = ""
for event, element in elements:
if event == "end":
tag = get_localname(element)
if tag == "entry":
properties = element.find(properties_xpath[0])
datum = {get_localname(node): node.text for node in properties.getchildren() if ET.iselement(node)}
# clear the element after we've dealt with it:
element.clear()
yield datum
elif event == "start-ns":
namespaces[element[0]] = element[1]
updated_namespaces()
elif event == "end-ns":
namespaces.popitem()
updated_namespaces()
def main():
events = Queue.Queue(MAX_QUEUE)
wifithread = wifisniffer.Sniffer(events)
wifithread.daemon = True
btthread = btsniffer.Sniffer(events)
btthread.daemon = True
webthread = web.HoneyServer()
webthread.daemon = True
macaddr.init_cache()
wifithread.start()
btthread.start()
webthread.start()
seen = OrderedDict()
try:
while True:
interface, mac, when = events.get()
name = '"%s"'%macaddr.identify(mac)
notes = "Occurred %s" % time.ctime(when)
if mac in seen:
del seen[mac]
else:
push.note(interface, name, mac, notes)
if len(seen) >= MAX_REMEMBER:
seen.popitem()
seen[mac] = when
finally:
macaddr.save_cache()
def containsNearbyAlmostDuplicate(self, nums, k, t):
"""
Two Intervals k & t:
* Use OrderedDict to remember the index and n/t to shrink the interval to -1, 0, 1.
* In terms of index difference, since there are i-k and i+k, when scanning from left to right, only consider i-k.
Alternative algorithms:
Window + TreeSet (Java) / SortedSet (C++) # TODO
:type nums: list[int]
:rtype: bool
"""
if k < 1 or t < 0:
return False
if t == 0:
return self.containsNearByDuplicate(nums, k)
od = OrderedDict() # keep the window
for n in nums:
key = n/t
for j in (-1, 0, 1): # (n-t, n, n+t), shrink the interval
m = od.get(key+j)
if m is not None and abs(m-n) <= t: # need to recheck, consider case {1, 7}, t=4
return True
while len(od) >= k:
od.popitem(False) # not last, i.e. the first
od[key] = n
return False
def overwrite_asdc_authors(string_authors, asdc_auth):
"""
takes the authors string and update regarding what the admins decide
param: take a long string of authors
authors to underline
return: string with updated information for asdc author
"""
repls = {'<b>': '', '</b>': '', '...': ''}
string_clean = reduce(lambda a, kv: a.replace(*kv), repls.iteritems(), string_authors)
author_list = string_clean.split(";")
asdc_auth = OrderedDict((auth.split("_")[0], auth.split("_")[1]) for auth in asdc_auth)
auth_dict = OrderedDict((auth.split(",")[0].lower().strip().replace(" ", ""), auth) for auth in author_list)
append_last_element = False
if next(reversed(auth_dict)).strip() == 'et al.':
auth_dict.popitem()
append_last_element = True
for auth in asdc_auth.keys():
auth_dict[auth] = '<b> %s </b>' % (asdc_auth[auth].replace("%2C", ", ")).replace("+", " ")
if append_last_element:
auth_dict.update({' et al.': ' et al.'})
return ';'.join(auth_dict.itervalues())
class LastResourceUsed:
"""Class implementing LRU using OrderedDict as base.
Know isssue: age update is O(n)
"""
def __init__(self, max_size=100):
self._max_size = max_size
self._items = OrderedDict()
def __setitem__(self, key, value):
if len(self) == self._max_size:
self._items.popitem(last=False)
self._items[key] = value
def __len__(self):
return len(self._items)
def __getitem__(self, key):
try:
value = self._items[key]
except KeyError:
raise LRUMiss(f'Item {key} no present on LRU') from KeyError
else:
# Updating age before returning value
self._items.pop(key)
self._items[key] = value
return value
class Density(object):
def __init__(self, params=None):
self.params = params
self.cache = OrderedDict()
self.max_cache_size = 10000
def log_p(self, data, params):
'''
Args:
data : (nametuple) Data for density.
params : (nametuple) Parameters in density.
Kwargs:
global_params: (namedtuple) Parameters which are shared across all atoms. If this is None it will use the
current value.
'''
key = (data, params, self.params)
if key not in self.cache:
self.cache[key] = self._log_p(data, params)
if len(self.cache) > self.max_cache_size:
self.cache.popitem(last=False)
return self.cache[key]
def _log_p(self, data, params):
raise NotImplemented
class _MultiCache(object):
"""
A utility class for caching items in a of a dict-of-dicts
"""
def __init__(self, first_uid, default_factory=lambda: None, maxcaches=None):
self._maxcaches = maxcaches
self.caches = OrderedDict()
self.add(first_uid, default_factory=default_factory)
def add(self, uid, default_factory=lambda: None):
assert isinstance(uid[1], tuple)
if uid not in self.caches:
cache = defaultdict(default_factory)
self.caches[uid] = cache
else:
raise Exception("MultiCache.add: uid %s is already in use" % str(uid))
# remove oldest cache, if necessary
old_uid = None
if self._maxcaches and len(self.caches) > self._maxcaches:
old_uid, v = self.caches.popitem(False) # removes item in FIFO order
return old_uid
def touch(self, uid):
c = self.caches[uid]
del self.caches[uid]
self.caches[uid] = c
def set_maxcaches(self, newmax):
self._maxcaches = newmax
while len(self.caches) > self._maxcaches:
old_uid, v = self.caches.popitem(False) # removes item in FIFO order
def dict_OrderedDict():
"""
字典是哈希表,默认迭代是无序的,如果希望按照元素添加顺序输出结果,可以用OrderedDict
:return:
"""
print ">>>>>dict_OrderedDict:"
from collections import OrderedDict
d = dict()
d["a"] = 1
d["b"] = 2
d["c"] = 3
print u"#非按添加序列输出"
for k, v in d.items():print k,v
#a 1
#c 3
#b 2
od = OrderedDict()
od["a"] = 1
od["b"] = 2
od["c"] = 3
print u"#按添加序列输出"
for k, v in od.items():print k, v
#a 1
#b 2
#c 3
print u"#按LIFO顺序弹出"
print od.popitem() #('c', 3)
print od.popitem() #('c', 3)
print od.popitem() #('a', 1)
class LRUCache(object):
def __init__(self, size=1024):
self.size = size
self.cache = OrderedDict()
def add(self, key, value):
if key in self.cache:
return
if len(self.cache) == self.size:
self._remove_oldest_entry_and_add_new(key, value)
return
self._add_new_entry(key, value)
def get(self, key):
val = self.cache.get(key, None)
if val is not None:
self._update_key_access_time(key)
return val
def _remove_oldest_entry_and_add_new(self, key, value):
self.cache.popitem(last=False)
self.cache[key] = value
def _add_new_entry(self, key, value):
self.cache[key] = value
def _update_key_access_time(self, key):
val = self.cache.pop(key)
self.cache[key] = val
class LRUDict(object):
"""A LRU dictionary is a dictionary with a fixed maximum number of keys"""
def __init__(self, size):
"""Initialization
In:
- ``size`` -- maximum number of keys
"""
self.size = size
self.items = OrderedDict()
def __contains__(self, k):
"""Test if a key exists into this dictionary
In:
- ``k`` -- the key
Return:
- a boolean
"""
return k in self.items
def __getitem__(self, k):
"""Return the value of a key.
The key becomes the most recently used key.
In:
- ``k`` -- the key
Return:
- the value
"""
v = self.items.pop(k)
self.items[k] = v
return v
def __setitem__(self, k, v):
"""Insert a key as the last recently used
In:
- ``k`` -- the key
- ``v`` -- the value
"""
self.items.pop(k, None)
self.items[k] = v
if len(self.items) > self.size:
self.items.popitem(False)
def __delitem__(self, k):
"""Delete a key.
In:
- ``k`` -- the key
"""
del self.items[k]
def __repr__(self):
return repr(self.items)
class StatsEntry(object):
"""
Represents a single stats entry (name and method)
"""
name = None
""" Name (URL) of this stats entry """
method = None
""" Method (GET, POST, PUT, etc.) """
num_requests = None
""" The number of requests made """
num_none_requests = None
""" The number of requests made with a None response time (typically async requests) """
num_failures = None
""" Number of failed request """
total_response_time = None
""" Total sum of the response times """
min_response_time = None
""" Minimum response time """
max_response_time = None
""" Maximum response time """
num_reqs_per_sec = None
""" A {second => request_count} dict that holds the number of requests made per second """
num_fail_per_sec = None
""" A (second => failure_count) dict that hold the number of failures per second """
response_times = None
"""
A {response_time => count} dict that holds the response time distribution of all
the requests.
The keys (the response time in ms) are rounded to store 1, 2, ... 9, 10, 20. .. 90,
100, 200 .. 900, 1000, 2000 ... 9000, in order to save memory.
This dict is used to calculate the median and percentile response times.
"""
use_response_times_cache = False
"""
If set to True, the copy of the response_time dict will be stored in response_times_cache
every second, and kept for 20 seconds (by default, will be CURRENT_RESPONSE_TIME_PERCENTILE_WINDOW + 10).
We can use this dict to calculate the *current* median response time, as well as other response
time percentiles.
"""
response_times_cache = None
"""
If use_response_times_cache is set to True, this will be a {timestamp => CachedResponseTimes()}
OrderedDict that holds a copy of the response_times dict for each of the last 20 seconds.
"""
total_content_length = None
""" The sum of the content length of all the requests for this entry """
start_time = None
""" Time of the first request for this entry """
last_request_timestamp = None
""" Time of the last request for this entry """
def __init__(self, stats, name, method, use_response_times_cache=False):
self.stats = stats
self.name = name
self.method = method
self.use_response_times_cache = use_response_times_cache
self.reset()
def reset(self):
self.start_time = time.time()
self.num_requests = 0
self.num_none_requests = 0
self.num_failures = 0
self.total_response_time = 0
self.response_times = {}
self.min_response_time = None
self.max_response_time = 0
self.last_request_timestamp = None
self.num_reqs_per_sec = {}
self.num_fail_per_sec = {}
self.total_content_length = 0
if self.use_response_times_cache:
self.response_times_cache = OrderedDict()
self._cache_response_times(int(time.time()))
def log(self, response_time, content_length):
# get the time
current_time = time.time()
t = int(current_time)
if self.use_response_times_cache and self.last_request_timestamp and t > int(
self.last_request_timestamp):
# see if we shall make a copy of the respone_times dict and store in the cache
self._cache_response_times(t - 1)
self.num_requests += 1
self._log_time_of_request(current_time)
self._log_response_time(response_time)
# increase total content-length
self.total_content_length += content_length
def _log_time_of_request(self, current_time):
t = int(current_time)
self.num_reqs_per_sec[t] = self.num_reqs_per_sec.setdefault(t, 0) + 1
self.last_request_timestamp = current_time
def _log_response_time(self, response_time):
if response_time is None:
self.num_none_requests += 1
return
self.total_response_time += response_time
if self.min_response_time is None:
self.min_response_time = response_time
self.min_response_time = min(self.min_response_time, response_time)
self.max_response_time = max(self.max_response_time, response_time)
# to avoid to much data that has to be transfered to the master node when
# running in distributed mode, we save the response time rounded in a dict
# so that 147 becomes 150, 3432 becomes 3400 and 58760 becomes 59000
if response_time < 100:
rounded_response_time = round(response_time)
elif response_time < 1000:
rounded_response_time = round(response_time, -1)
elif response_time < 10000:
rounded_response_time = round(response_time, -2)
else:
rounded_response_time = round(response_time, -3)
# increase request count for the rounded key in response time dict
self.response_times.setdefault(rounded_response_time, 0)
self.response_times[rounded_response_time] += 1
def log_error(self, error):
self.num_failures += 1
t = int(time.time())
self.num_fail_per_sec[t] = self.num_fail_per_sec.setdefault(t, 0) + 1
@property
def fail_ratio(self):
try:
return float(self.num_failures) / self.num_requests
except ZeroDivisionError:
if self.num_failures > 0:
return 1.0
else:
return 0.0
@property
def avg_response_time(self):
try:
return float(self.total_response_time) / (self.num_requests -
self.num_none_requests)
except ZeroDivisionError:
return 0
@property
def median_response_time(self):
if not self.response_times:
return 0
median = median_from_dict(self.num_requests - self.num_none_requests,
self.response_times) or 0
# Since we only use two digits of precision when calculating the median response time
# while still using the exact values for min and max response times, the following checks
# makes sure that we don't report a median > max or median < min when a StatsEntry only
# have one (or very few) really slow requests
if median > self.max_response_time:
median = self.max_response_time
elif median < self.min_response_time:
median = self.min_response_time
return median
@property
def current_rps(self):
if self.stats.last_request_timestamp is None:
return 0
slice_start_time = max(
int(self.stats.last_request_timestamp) - 12,
int(self.stats.start_time or 0))
reqs = [
self.num_reqs_per_sec.get(t, 0)
for t in range(slice_start_time,
int(self.stats.last_request_timestamp) - 2)
]
return avg(reqs)
@property
def current_fail_per_sec(self):
if self.stats.last_request_timestamp is None:
return 0
slice_start_time = max(
int(self.stats.last_request_timestamp) - 12,
int(self.stats.start_time or 0))
reqs = [
self.num_fail_per_sec.get(t, 0)
for t in range(slice_start_time,
int(self.stats.last_request_timestamp) - 2)
]
return avg(reqs)
@property
def total_rps(self):
if not self.stats.last_request_timestamp or not self.stats.start_time:
return 0.0
try:
return self.num_requests / (self.stats.last_request_timestamp -
self.stats.start_time)
except ZeroDivisionError:
return 0.0
@property
def total_fail_per_sec(self):
if not self.stats.last_request_timestamp or not self.stats.start_time:
return 0.0
try:
return self.num_failures / (self.stats.last_request_timestamp -
self.stats.start_time)
except ZeroDivisionError:
return 0.0
@property
def avg_content_length(self):
try:
return self.total_content_length / self.num_requests
except ZeroDivisionError:
return 0
def extend(self, other):
"""
Extend the data from the current StatsEntry with the stats from another
StatsEntry instance.
"""
if self.last_request_timestamp is not None and other.last_request_timestamp is not None:
self.last_request_timestamp = max(self.last_request_timestamp,
other.last_request_timestamp)
elif other.last_request_timestamp is not None:
self.last_request_timestamp = other.last_request_timestamp
self.start_time = min(self.start_time, other.start_time)
self.num_requests = self.num_requests + other.num_requests
self.num_none_requests = self.num_none_requests + other.num_none_requests
self.num_failures = self.num_failures + other.num_failures
self.total_response_time = self.total_response_time + other.total_response_time
self.max_response_time = max(self.max_response_time,
other.max_response_time)
if self.min_response_time is not None and other.min_response_time is not None:
self.min_response_time = min(self.min_response_time,
other.min_response_time)
elif other.min_response_time is not None:
# this means self.min_response_time is None, so we can safely replace it
self.min_response_time = other.min_response_time
self.total_content_length = self.total_content_length + other.total_content_length
for key in other.response_times:
self.response_times[key] = self.response_times.get(
key, 0) + other.response_times[key]
for key in other.num_reqs_per_sec:
self.num_reqs_per_sec[key] = self.num_reqs_per_sec.get(
key, 0) + other.num_reqs_per_sec[key]
for key in other.num_fail_per_sec:
self.num_fail_per_sec[key] = self.num_fail_per_sec.get(
key, 0) + other.num_fail_per_sec[key]
def serialize(self):
return {
"name": self.name,
"method": self.method,
"last_request_timestamp": self.last_request_timestamp,
"start_time": self.start_time,
"num_requests": self.num_requests,
"num_none_requests": self.num_none_requests,
"num_failures": self.num_failures,
"total_response_time": self.total_response_time,
"max_response_time": self.max_response_time,
"min_response_time": self.min_response_time,
"total_content_length": self.total_content_length,
"response_times": self.response_times,
"num_reqs_per_sec": self.num_reqs_per_sec,
"num_fail_per_sec": self.num_fail_per_sec,
}
@classmethod
def unserialize(cls, data):
obj = cls(None, data["name"], data["method"])
for key in [
"last_request_timestamp",
"start_time",
"num_requests",
"num_none_requests",
"num_failures",
"total_response_time",
"max_response_time",
"min_response_time",
"total_content_length",
"response_times",
"num_reqs_per_sec",
"num_fail_per_sec",
]:
setattr(obj, key, data[key])
return obj
def get_stripped_report(self):
"""
Return the serialized version of this StatsEntry, and then clear the current stats.
"""
report = self.serialize()
self.reset()
return report
def to_string(self, current=True):
"""
Return the stats as a string suitable for console output. If current is True, it'll show
the RPS and failure rait for the last 10 seconds. If it's false, it'll show the total stats
for the whole run.
"""
if current:
rps = self.current_rps
fail_per_sec = self.current_fail_per_sec
else:
rps = self.total_rps
fail_per_sec = self.total_fail_per_sec
return (" %-" + str(STATS_NAME_WIDTH) +
"s %7d %12s %7d %7d %7d | %7d %7.2f %7.2f") % (
(self.method and self.method + " " or "") + self.name,
self.num_requests,
"%d(%.2f%%)" % (self.num_failures, self.fail_ratio * 100),
self.avg_response_time,
self.min_response_time or 0,
self.max_response_time,
self.median_response_time or 0,
rps or 0,
fail_per_sec or 0,
)
def __str__(self):
return self.to_string(current=True)
def get_response_time_percentile(self, percent):
"""
Get the response time that a certain number of percent of the requests
finished within.
Percent specified in range: 0.0 - 1.0
"""
return calculate_response_time_percentile(self.response_times,
self.num_requests, percent)
def get_current_response_time_percentile(self, percent):
"""
Calculate the *current* response time for a certain percentile. We use a sliding
window of (approximately) the last 10 seconds (specified by CURRENT_RESPONSE_TIME_PERCENTILE_WINDOW)
when calculating this.
"""
if not self.use_response_times_cache:
raise ValueError(
"StatsEntry.use_response_times_cache must be set to True if we should be able to calculate the _current_ response time percentile"
)
# First, we want to determine which of the cached response_times dicts we should
# use to get response_times for approximately 10 seconds ago.
t = int(time.time())
# Since we can't be sure that the cache contains an entry for every second.
# We'll construct a list of timestamps which we consider acceptable keys to be used
# when trying to fetch the cached response_times. We construct this list in such a way
# that it's ordered by preference by starting to add t-10, then t-11, t-9, t-12, t-8,
# and so on
acceptable_timestamps = []
for i in range(9):
acceptable_timestamps.append(
t - CURRENT_RESPONSE_TIME_PERCENTILE_WINDOW - i)
acceptable_timestamps.append(
t - CURRENT_RESPONSE_TIME_PERCENTILE_WINDOW + i)
cached = None
for ts in acceptable_timestamps:
if ts in self.response_times_cache:
cached = self.response_times_cache[ts]
break
if cached:
# If we fond an acceptable cached response times, we'll calculate a new response
# times dict of the last 10 seconds (approximately) by diffing it with the current
# total response times. Then we'll use that to calculate a response time percentile
# for that timeframe
return calculate_response_time_percentile(
diff_response_time_dicts(self.response_times,
cached.response_times),
self.num_requests - cached.num_requests,
percent,
)
def percentile(self,
tpl=" %-" + str(STATS_TYPE_WIDTH) + "s %-" +
str(STATS_NAME_WIDTH) +
"s %8d %6d %6d %6d %6d %6d %6d %6d %6d %6d %6d %6d"):
if not self.num_requests:
raise ValueError(
"Can't calculate percentile on url with no successful requests"
)
return tpl % (self.method, self.name, self.num_requests,
self.get_response_time_percentile(0.5),
self.get_response_time_percentile(0.66),
self.get_response_time_percentile(0.75),
self.get_response_time_percentile(0.80),
self.get_response_time_percentile(0.90),
self.get_response_time_percentile(0.95),
self.get_response_time_percentile(0.98),
self.get_response_time_percentile(0.99),
self.get_response_time_percentile(0.999),
self.get_response_time_percentile(0.9999),
self.get_response_time_percentile(1.00))
def _cache_response_times(self, t):
self.response_times_cache[t] = CachedResponseTimes(
response_times=copy(self.response_times),
num_requests=self.num_requests,
)
# We'll use a cache size of CURRENT_RESPONSE_TIME_PERCENTILE_WINDOW + 10 since - in the extreme case -
# we might still use response times (from the cache) for t-CURRENT_RESPONSE_TIME_PERCENTILE_WINDOW-10
# to calculate the current response time percentile, if we're missing cached values for the subsequent
# 20 seconds
cache_size = CURRENT_RESPONSE_TIME_PERCENTILE_WINDOW + 10
if len(self.response_times_cache) > cache_size:
# only keep the latest 20 response_times dicts
for i in range(len(self.response_times_cache) - cache_size):
self.response_times_cache.popitem(last=False)
class LRU_cache:
def __init__(self, size, time_limit):
'''
Initialize the LRU cache
size: max number of elements allowed in cache
time_limit: how long an item is allowed to stay in the cache
'''
# Use Ordered Dictionary for cache
self.cache = OrderedDict()
# Capacity of cache
self.size = size
self.time_limit = time_limit
def get(self, key):
'''
Access item in cache and move it to the back and update the time
it was accessed at.
key: key of item to be retrieved
'''
if key in self.cache.keys():
value = self.cache.pop(key)[0]
self.cache[key] = (value, time.time())
return value
else:
return "Error: Item Not Found."
def put(self, key, value):
'''
Add item to cache or update the value of an existing key. This operation
clears all expired items
key: key of item to be retrieved
value: value of item
'''
if key in self.cache.keys():
self.cache.pop(key)
self.cache[key] = (value, time.time())
else:
if len(self.cache.keys()) >= self.size:
self.cache.popitem(last=False)
self.cache[key] = (value, time.time())
self.clear_expired()
def clear_expired(self):
'''
Clears all items in cache that have expired.
'''
expired = time.time() - self.time_limit
ex_flag = True
while ex_flag:
first = next(iter(self.cache))
if expired < self.cache[first][1]:
ex_flag = False
else:
self.cache.popitem(last=False)
def __str__(self):
'''
Prints all items in the cache as a dictionary.
'''
d = {}
for key, (value, time) in self.cache.items():
d[key] = value
return str(d)
class YTActions():
"""
Class responsible for searching in YouTube service and holding information about search results.
Attributes
----------
avail_files : OrderedDict
Contains tuples of following format:
avail_files = {
"token": (`adj_tokens`, `files`),
...
}
`adj_tokens` contains adjacent tokens, `files` contains files of given search.
(just as described below).
visible_files : dict
Current search results. Key is a movie name, value is a ``YTStor`` object for given movie.
adj_tokens : dict
Dictionary of tokens for adjacent search pages. Under ``False`` key the previous page is kept, under ``True`` -
the next. Other keys are not allowed.
vf_iter : obj
Here the ``YTActions`` obejct stores an iterator allowing for current directory content listing. Used by
``__iter__`` and ``__next__`` methods.
search_params : dict
Additional search params for __search.
yts_opts : dict
Custom options passed to YTStor objects created in this class.
api_key : str
YouTube API key.
preferences : dict
Current object preferences.
Parameters
----------
search_query : str
Currently used search phrase.
"""
api_key = "AIzaSyCPOg5HQfHayQH6mRu4m2PMGc3eHd5lllg"
preferences = {"metadata": {"desc": False, "thumb": False}}
def __init__(self, search_query):
if not isinstance(search_query, str):
raise ValueError("Expected str for 1st parameter (search_query).")
self.avail_files = OrderedDict()
self.visible_files = None
self.adj_tokens = {False: None, True: None}
self.vf_iter = None
self.search_params = {
"maxResults": 10,
"order": self.preferences["order"]
} # relevance by default
self.yts_opts = dict()
parsed = self.__searchParser(search_query)
# search params
self.search_params.update(parsed[0])
# YTa options
_pref = deepcopy(
self.preferences) # new object, just to not affect other intances.
if 'metadata' in parsed[1]:
try:
meta_list = parsed[1]['metadata'].split(',')
except AttributeError:
meta_list = []
if 'desc' in meta_list: _pref['metadata']['desc'] = True
else: _pref['metadata']['desc'] = False
if 'thumb' in meta_list: _pref['metadata']['thumb'] = True
else: _pref['metadata']['thumb'] = False
self.preferences = _pref
# YTs options
self.yts_opts = parsed[1]
if 'audio' in parsed[1] and 'video' not in parsed[1]:
self.yts_opts['video'] = False
if 'video' in parsed[1] and 'audio' not in parsed[1]:
self.yts_opts['audio'] = False
try:
self.__getChannelId()
except requests.exceptions.ConnectionError:
raise ConnectionError
if parsed[0].get("publishedBefore"):
self.search_params["publishedBefore"] += "T00:00:00Z"
if parsed[0].get("publishedAfter"):
self.search_params["publishedAfter"] += "T00:00:00Z"
def __getChannelId(self):
"""
Obtain channel id for channel name, if present in ``self.search_params``.
"""
if not self.search_params.get("channelId"):
return
api_fixed_url = "https://www.googleapis.com/youtube/v3/channels?part=id&maxResults=1&fields=items%2Fid&"
url = api_fixed_url + urlencode(
{
"key": self.api_key,
"forUsername": self.search_params["channelId"]
})
get = requests.get(url).json()
try:
self.search_params["channelId"] = get['items'][0]['id']
return # got it
except IndexError:
pass # try searching now...
api_fixed_url = "https://www.googleapis.com/youtube/v3/search?part=snippet&type=channel&fields=items%2Fid&"
url = api_fixed_url + urlencode({
"key": self.api_key,
"q": self.search_params['channelId']
})
get = requests.get(url).json()
try:
self.search_params["channelId"] = get['items'][0]['id'][
'channelId']
except IndexError:
del self.search_params["channelId"] # channel not found
def __searchParser(self, query):
"""
Parse `query` for advanced search options.
Parameters
----------
query : str
Search query to parse. Besides a search query, user can specify additional search parameters and YTFS
specific options. Syntax:
Additional search parameters: ``option:value``. if `value` contains spaces, then surround it with
parentheses; available parameters: `channel`, `max`, `before`, `after`, `order`.
YTFS options: specify options between ``[`` and ``]``; Available options: `a`, `v`, `f`, `P`, `s`, `m`.
If an option takes a parameter, then specify it beetween parentheses.
Examples: ``channel:foo search query``, ``my favourite music [a]``,
``channel:(the famous funny cats channel) [vf(240)P] funny cats max:20``.
Invalid parameters/options are ignored.
Returns
-------
params : tuple
Tuple: 0 - dictionary of url GET parameters; 1 - dictionary of YTStor options.
"""
ret = dict()
parse_params = True
buf = ""
ptr = ""
p_avail = ("channel", "max", "before", "after", "order")
opts = dict()
par_open = False
translate = {
'a': 'audio',
'v': 'video',
'f': 'format',
's': 'stream',
'P': 'stream',
'm': 'metadata',
'max': 'maxResults',
'channel': 'channelId',
'before': 'publishedBefore',
'after': 'publishedAfter',
'order': 'order',
'': 'q'
}
for i in query + ' ':
if parse_params:
if not par_open:
if i == ' ': # flush buf
try:
if ret.get(translate[ptr]):
ret[translate[ptr]] += ' '
else:
ret[translate[ptr]] = ''
ret[translate[ptr]] += buf
except KeyError:
pass
ptr = ""
buf = ""
elif i == ':' and buf in p_avail:
ptr = buf
buf = ""
elif not buf and i == '[': # buf must be empty
parse_params = False
ptr = ""
elif i != '(':
buf += i
elif not (par_open == 1 and i == ')'):
buf += i
if i == '(': par_open += 1
if par_open > 0 and i == ')': par_open -= 1
else:
if i == ']':
parse_params = True
par_open = False
ptr = ""
buf = ""
elif ptr and not par_open and i == '(':
par_open = True
elif par_open:
if i == ')':
try:
opts[translate[ptr]] = buf
except KeyError:
pass
par_open = False
buf = ""
else:
buf += i
elif i.isalpha():
ptr = i
try:
opts[translate[ptr]] = not i.isupper()
except KeyError:
pass
return (ret, opts)
def __search(self, pt=""):
"""
Method responsible for searching using YouTube API.
Parameters
----------
pt : str
Token of search results page. If ``None``, then the first page is downloaded.
Returns
-------
results : dict
Parsed JSON returned by YouTube API.
"""
if not self.search_params.get('q') and not self.search_params.get(
'channelId'):
return {'items': []} # no valid query - no results.
api_fixed_url = "https://www.googleapis.com/youtube/v3/search?part=snippet&type=video&fields=items(id%2Ckind%2Csnippet)%2CnextPageToken%2CprevPageToken&"
d = {"key": self.api_key, "pageToken": pt}
d.update(self.search_params)
url = api_fixed_url + urlencode(d)
try:
get = requests.get(url)
except requests.exceptions.ConnectionError:
raise ConnectionError
if get.status_code != 200:
return {'items': []} # no valid query - no results.
return get.json()
def __iter__(self):
"""
Create an iterator. Method allows - in a simple manner - for obtaining a generator which contains filenames.
The real generator is ``self.vf_iter``; YTActions object (used as iterator) is only a wrapper.
Returns
-------
self : YTActions
This very same object with ``self.vf_iter`` constructed and initialised.
"""
ctrl = []
if self.adj_tokens[False] is not None:
ctrl += [" prev"]
if self.adj_tokens[True] is not None:
ctrl += [" next"]
self.vf_iter = iter(
ctrl +
[e + self.visible_files[e].extension for e in self.visible_files])
return self
def __next__(self):
"""
``next()`` support. Returns next filename.
Returns
-------
file_name : str
Next filename from ``self.vf_iter.``
"""
return next(self.vf_iter) #easy, ain't it?
def __getitem__(self, key):
"""
Read elements from ``YTActions`` object by using `key`. One can use object like a dict (and like a boss, ofc).
Parameters
----------
key : str
The key (e.g. ``YTActions['Rick Astley - Never Gonna Give You Up.mp4']``).
Returns
-------
YTStor
``YTStor`` object associated with name `key`.
"""
_k = os.path.splitext(key)
if _k[1] not in ('.txt', '.jpg'):
key = _k[0]
return self.visible_files[key]
def __in__(self, arg):
"""
Check, if movie of name `arg` is present in the object.
Parameters
----------
arg : str
Filename.
"""
_a = os.path.splitext(arg)
if _a[1] not in ('txt', 'jpg'):
arg = _a[0]
return arg in self.visible_files or (
self.adj_tokens[0] is not None
and arg == " prev") or (self.adj_tokens[0] is None
and self.adj_tokens[1] is not None
and arg == " next")
def updateResults(self, forward=None):
"""
Reload search results or load another "page".
Parameters
----------
forward : bool or None, optional
Whether move forwards or backwards (``True`` or ``False``). If ``None``, then first page is loaded.
"""
# this choses data we need.
files = lambda x: {
i['snippet']['title'].replace('/', '\\'): YTStor(
{
'yid': i['id']['videoId'],
'pub_date': i['snippet']['publishedAt']
},
opts=self.yts_opts)
for i in x['items']
}
descs = lambda x: {(i['snippet']['title'].replace('/', '\\') + '.txt'):
YTMetaStor(
{
'title': i['snippet']['title'],
'yid': i['id']['videoId'],
'desc': i['snippet']['description'],
'channel': i['snippet']['channelTitle'],
'pub_date': i['snippet']['publishedAt']
},
opts=dict())
for i in x['items']}
thumbs = lambda x: {
(i['snippet']['title'].replace('/', '\\') + '.jpg'): YTMetaStor(
{
'url': i['snippet']['thumbnails']['high']['url'],
'pub_date': i['snippet']['publishedAt']
},
opts=dict())
for i in x['items']
}
try:
if self.adj_tokens[
forward] is None: # in case someone would somehow cross boundary.
forward = None
except KeyError:
pass
recv = None
try:
try:
data = self.avail_files[self.adj_tokens[
forward]] # maybe data is already available locally.
except KeyError:
recv = self.__search(
self.adj_tokens[forward]) # nope, we have to search.
except KeyError: # wrong index in adj_tokens
if forward is None:
recv = self.__search()
else:
raise ValueError(
"Valid values for forward are True, False or None (default)."
)
if recv is not None:
_d = files(recv)
if self.preferences['metadata']['desc']: _d.update(descs(recv))
if self.preferences['metadata']['thumb']: _d.update(thumbs(recv))
data = (None, _d) # little format unification.
if len(self.avail_files) > 4:
pop = self.avail_files.popitem(
False) # get rid of the oldest data.
for s in pop[1][1].values():
s.clean()
adj_t = deepcopy(
self.adj_tokens
) # this will we write to avail_files, now we update self.adj_tokens.
if data[0] is None: # get tokens from obtained results.
try:
self.adj_tokens[False] = recv['prevPageToken']
except KeyError:
self.adj_tokens[False] = None
try:
self.adj_tokens[True] = recv['nextPageToken']
except KeyError:
self.adj_tokens[True] = None
else: # already in avail_files.
self.adj_tokens = data[0]
if forward is not None:
# backup last results in avail_files:
self.avail_files[self.adj_tokens[not forward]] = (
adj_t, self.visible_files)
self.visible_files = data[1]
def clean(self):
"""Clear the data. For each ``YTStor`` object present in this object ``clean`` method is executed."""
for s in self.visible_files.values():
s.clean()
for s in [
sub[1][x] for sub in self.avail_files.values() for x in sub[1]
]: # Double list comprehensions aren't very
s.clean() # readable...
class alg:
def __repr__(self):
return "LIRS"
def __init__(self, c, **kwargs):
# c is cache size
self.c = c # Max
self.lirs = 0
self.hirs = 0
self.maxhirs = max(1, 0.01 * c)
self.maxlirs = c - self.maxhirs
self.S = OrderedDict(
) # Recently accessed pages (LIR pages and non-resident HIR pages), newest element is last
self.Q = OrderedDict() # Resident HIR pages
self.hitcount = 0
self.count = 0
def setup(self, reqlist):
# I'm an online algorithm :-)
pass
def get(self, key):
#if self.count % 100 == 0:
# sys.stderr.write("S: %d\n" % len(self.S))
self.countResident()
self.count += 1
if (key in self.S and self.S[key].resident) or key in self.Q:
#print "HIT for %s" % key
self.hitcount += 1
return 1
return 0
#print "MISS for %s" % key
def put(self, key, val=1):
if key in self.S:
e = self.S[key]
del self.S[key]
if e.flag == HIR:
# A HIR block with lower reuse distance than the bottom LIR element
if e.resident:
del self.Q[key]
self.hirs -= 1
assert self.oldestS().flag == LIR
assert self.hirs <= self.maxhirs
self.swap() # Change one LIR page to a HIR page
self.evict(
) # Keep the number of resident HIR pages <= self.maxhirs
self.lirs += 1
e.resident = True
e.flag = LIR
self.S[key] = e
self.prune() # If this was a LIR element at the bottom
elif key in self.Q:
# Resident HIR page which is not in S, reuse distance is large so we don't make it a LIR page
e = self.Q[key]
assert e.resident and e.flag == HIR
del self.Q[key]
self.Q[key] = e
self.S[key] = e
else:
# not in cache
# When a miss occurs and a free block is needed for replacement, we choose an HIR block that is resident in the cache
e = entry(key)
if self.lirs < self.maxlirs:
e.flag = LIR # Not using all the cache, make it a LIR page
self.lirs += 1
else:
# NOTE: Not changing the number of LIR blocks!
self.Q[key] = e
self.hirs += 1
self.evict() # Evicting if we have max number of HIR blocks
self.S[key] = e
assert self.oldestS().flag == LIR
self.countResident()
def countResident(self):
sm = 0
for key, e in self.S.items():
sm += 1 if e.resident else 0
for key, e in self.Q.items():
sm += 1 if key not in self.S else 0
if sm != self.lirs + self.hirs:
print "sm=%d, self.lirs=%d, self.hirs=%d" % (sm, self.lirs,
self.hirs)
assert sm == self.lirs + self.hirs
def evict(self):
if self.hirs > self.maxhirs: # Setting the size of Q
k, e_hir = self.Q.popitem(last=False)
assert e_hir.flag == HIR and e_hir.resident
e_hir.resident = False # It is maybe in S
self.hirs -= 1
if self.hirs > self.maxhirs:
print "hirs: %d maxhirs: %d" % (self.hirs, self.maxhirs)
assert self.hirs <= self.maxhirs
# TODO: If there are many nonresident entries in S, should we delete this from S?
def swap(self):
key, e = self.S.popitem(last=False)
assert e.flag == LIR and e.resident
e.flag = HIR
self.hirs += 1
self.lirs -= 1
self.Q[e.key] = e
# Switched this LIR block to HIR and put it in Q
# Important! We don't put it back in S
def prune(self):
while self.S:
oldest = self.oldestS()
if oldest.flag == LIR:
break
k, dele = self.S.popitem(last=False)
assert dele.flag == HIR
if dele.resident:
assert k in self.Q
else:
assert k not in self.Q
def oldestS(self):
return self.S.itervalues().next()
def oldestQ(self):
return self.Q.itervalues().next()
class World:
def __init__(self, server, name: str, path: str, chunk_cache_max: int) -> None:
self.server = server
self.name = name
self.path = path # should be "worlds/world_name_dim/" in production probably
self.data = None
self._chunk_cache_max = chunk_cache_max
self._chunk_cache = OrderedDict()
self._proper_name = None
self._dimension = None
@property
def proper_name(self):
if self._proper_name is None:
self._proper_name = list(self.server.worlds.keys())[list(self.server.worlds.values()).index(self)]
return self._proper_name
@property
def dimension(self):
if self._dimension is None:
self._dimension = "minecraft:" + self.proper_name.replace(self.name + "_", "")
return self._dimension
async def init(self):
self.data = await self.load_level_data()
return self # for fluent style chaining
async def load_level_data(self):
file = os.path.join(self.path, "level.dat")
if os.path.isfile(file):
async with aiofile.async_open(file, "rb") as level_data_file:
return nbt.TAG_Compound.unpack(Buffer(await level_data_file.read()))
return new_level_nbt((2586, self.server.meta.version, 19133), self.name, (0, 100, 0), self.server.conf["seed"])["Data"]
def cache_chunk(self, chunk: Chunk, key: tuple) -> Chunk:
self._chunk_cache[key] = chunk
if len(self.chunk_cache) > self._chunk_cache_max:
self._chunk_cache.popitem(False)
return chunk
async def fetch_chunk(self, chunk_x: int, chunk_z: int) -> Chunk:
key = (chunk_x, chunk_z)
try:
return self._chunk_cache[key]
except KeyError:
pass
try:
return self.cache_chunk(await self.server.chunkio.fetch_chunk_async(self.path, *key), key)
except FileNotFoundError:
chunk_data = self.server.generator.generate_chunk(self.data["RandomSeed"], self.dimension, chunk_x, chunk_z)
return chunk_data
class EmptyFeature(Sequence):
def __init__(self,
lookback,
raw_data_manager,
name="",
backfill=True,
history_lengh=None,
features=None):
raw_data_length = raw_data_manager.get_history()
# kind of spaghetti logic here to for feature history length but the gist is this:
# one may specify history length when instantiating or we can produce the maximum
# length given the data available
feath = 0
hh = raw_data_length
if features is not None:
feath = min([f.get_history_length() for f in features])
hh = feath
if history_lengh is None:
self.history_lengh = hh - lookback + 1
else:
self.history_lengh = history_lengh
self.lookback = lookback
self.raw_data_manager = raw_data_manager
# raw data must be able to fill history
assert (self.history_lengh + lookback - 1) <= raw_data_length
assert lookback <= raw_data_manager.get_lookback()
self.feature = OrderedDict()
self.feature_numpy = RingBuffer(capacity=self.history_lengh,
dtype=np.float64)
self.latest = {}
# a feature may contain multiple features
self.features = features
self.feature_df = raw_data_manager.get_backfill_df()
# trim df
drop_i = self.lookback
if self.features is not None:
if len(self.features) > 0:
drop_i = drop_i + max(
[f.get_lookback() for f in self.features]) - 1
self.feature_df = self.feature_df.iloc[(drop_i - 1):]
# must take into account history len;
if backfill:
self.backfill()
self.live_save_path = 'data_test/live/features/' + type(
self).__name__ + '.csv'
self.backtest_path = 'data_test/features/' + self.__str__(
) + name + '.csv'
# self.save_feature()
self.test_rounded = []
super().__init__()
def get_feature(self):
# assert len(self.feature) > 0
return self.feature
def get_history_length(self):
return self.history_lengh
# -> override this ...
def compute(self, data_dict):
return []
def populate_feature(self):
l = len(self.get_TS())
c = 0
for index, value in self.get_TS().items():
self.feature_numpy.append(value)
self.feature[index] = value
if c == l - 1:
self.latest = {'time': index, 'value': value}
c += 1
def update_feature(self, time, value):
# we remove first item in dict, add the new one, FIFO style
# maybe use hist length?
if len(self.feature) > self.history_lengh:
self.feature.popitem(last=False)
self.feature[time] = value
self.feature_numpy.append(value)
# assert len(self.feature) == self.history_lengh
# backfill history
def backfill(self):
data = self.raw_data_manager.get_backfill_data()
ff = self.compute(data)[-self.history_lengh:]
print(len(ff))
print(self.history_lengh)
assert len(ff) == self.history_lengh
self.feature_df[str(self)] = ff
self.populate_feature()
# live update
def update(self):
candle = self.raw_data_manager.get_live_candle()
time_stamp = candle.get('time')
# update sub-features
if self.features is not None:
for f in self.features:
f.update()
# check as to not duplicate update
if time_stamp in self.feature:
return
# print('Updating ... ' + type(self).__name__)
data = self.raw_data_manager.get_live_data()
new_val = self.compute(data)[-1]
self.latest = {'time': time_stamp, 'value': new_val}
self.update_feature(time_stamp, new_val)
# self.save_feature_live()
def __getitem__(self, i):
rounded_ii = self.get_rounded_index(i)
if rounded_ii in self.feature:
return self.feature[rounded_ii]
else:
print("Feature is... ")
print(type(self).__name__)
print("Actual index ")
print(i)
print("Rounded ")
print(rounded_ii)
print("First keys")
kk = list(self.feature.keys())
print(kk[0:10])
raise ValueError("Feature timestamp not found")
def get_last_timestamp(self):
return next(reversed(self.feature))
def get_rounded_index(self, index):
per_minutes = self.raw_data_manager.get_period_in_minutes()
return index - (index // 60 % per_minutes) * 60
def get_period(self):
return self.raw_data_manager.get_period()
def __len__(self):
return len(self.feature)
def __eq__(self, other_feature):
return isinstance(other_feature, EmptyFeature) and type(self).__name__ == type(other_feature).__name__ \
and self.raw_data_manager == other_feature.raw_data_manager
def __hash__(self):
return hash(str(self))
def __str__(self):
return type(self).__name__ + self.raw_data_manager.get_symbol(
) + self.get_period()
def get_history_start_time(self):
# return self.feature_df.index[0]
return next(iter(self.feature))
def get_DF(self):
return self.feature_df
def get_TS(self):
return self.feature_df[str(self)]
def get_numpy(self):
return self.feature_numpy
# Use this method especially ifm for example
# in an alpha that updates every 1 minute you need a feature that updates every 5m.
# If not used, feature[ii] will give a value error in that instance
def get_latest(self):
return self.latest
def get_lookback(self):
return self.lookback
def save_DF(self):
name = type(self).__name__ + '.csv'
self.feature_df.to_csv('data_test/features/' + name)
def save_feature(self):
if os.path.isfile(self.backtest_path):
return
feature_file = open(self.backtest_path, 'w+')
writer = csv.DictWriter(feature_file, fieldnames=['time', 'value'])
writer.writeheader()
for ts, val in self.feature.items():
writer.writerow({'time': ts, 'value': val})
def save_feature_live(self):
strategy_file = open(self.live_save_path, 'a')
writer = csv.DictWriter(strategy_file, fieldnames=['time', 'value'])
writer.writerow(self.latest)
for p in Point1(3, 25), Point1(7, 2):
print(p)
# making direct assignments
a = Point1.x.__doc__ = 15
b = Point1.y.__doc__ = 3
print(a * b)
print("\n")
# OrderedDict same as dictionaries but they order the items.
ordered_dict = OrderedDict()
for i in range(6):
ordered_dict[i] = i * 5
print(ordered_dict)
# we can popitem as well. If Last = True, then LIFO returns. If false, FIFO returns
print(ordered_dict.popitem(last=True))
print(ordered_dict.popitem(last=False))
print(ordered_dict)
# with this method, we can move a key and its pair to end. If last = False, then move it to the beginning
ordered_dict.move_to_end(1)
print(ordered_dict)
ordered_dict.move_to_end(1, last=False)
print(ordered_dict)
ordered_dict1 = {}
for i in range(6):
ordered_dict1[i] = i * 5
print(ordered_dict1)
# we cannot use this. Because popitem in dict has no arguments!
"""print(ordered_dict1.popitem(last=True))
print(ordered_dict1.popitem(last=False))"""
class _SlideCache(object):
def __init__(self, cache_size, dz_opts):
self.cache_size = cache_size
self.dz_opts = dz_opts
self._lock = Lock()
self._cache = OrderedDict()
def get(self, path, originalPath=None):
with self._lock:
if path in self._cache:
# Move to end of LRU
slide = self._cache.pop(path)
self._cache[path] = slide
return slide
osr = OpenSlide(path)
# try:
# osr = OpenSlide(path)
# except:
# osr = ImageSlide(path)
# Fix for 16 bits tiff files
# if osr._image.getextrema()[1] > 256:
# osr._image = osr._image.point(lambda i:i*(1./256)).convert('L')
slide = DeepZoomGenerator(osr, **self.dz_opts)
slide.osr = osr
slide.associated_images = {}
for name, image in slide.osr.associated_images.items():
slide.associated_images[name] = image
try:
mpp_x = osr.properties[openslide.PROPERTY_NAME_MPP_X]
mpp_y = osr.properties[openslide.PROPERTY_NAME_MPP_Y]
slide.properties = osr.properties
slide.mpp = (float(mpp_x) + float(mpp_y)) / 2
except (KeyError, ValueError):
try:
if osr.properties["tiff.ResolutionUnit"] == "centimetre":
numerator = 10000 # microns in CM
else:
numerator = 25400 # Microns in Inch
mpp_x = numerator / float(osr.properties["tiff.XResolution"])
mpp_y = numerator / float(osr.properties["tiff.YResolution"])
slide.properties = osr.properties
slide.mpp = (float(mpp_x) + float(mpp_y)) / 2
except:
slide.mpp = 0
try:
slide.properties = slide.properties
except:
slide.properties = osr.properties
slide.tileLock = Lock()
if originalPath:
slide.properties = {"Path": originalPath}
with self._lock:
if path not in self._cache:
while len(self._cache) >= self.cache_size:
self._cache.popitem(last=False)
self._cache[path] = slide
return slide
class ConnectionState:
def __init__(self, *, dispatch, chunker, syncer, http, loop, **options):
self.loop = loop
self.http = http
self.max_messages = max(options.get('max_messages', 5000), 100)
self.dispatch = dispatch
self.chunker = chunker
self.syncer = syncer
self.is_bot = None
self.shard_count = None
self._ready_task = None
self._fetch_offline = options.get('fetch_offline_members', True)
self.heartbeat_timeout = options.get('heartbeat_timeout', 60.0)
self._listeners = []
activity = options.get('activity', None)
if activity:
if not isinstance(activity, _ActivityTag):
raise TypeError(
'activity parameter must be one of Game, Streaming, or Activity.'
)
activity = activity.to_dict()
status = options.get('status', None)
if status:
if status is Status.offline:
status = 'invisible'
else:
status = str(status)
self._activity = activity
self._status = status
self.clear()
def clear(self):
self.user = None
self._users = weakref.WeakValueDictionary()
self._emojis = {}
self._calls = {}
self._guilds = {}
self._voice_clients = {}
# LRU of max size 128
self._private_channels = OrderedDict()
# extra dict to look up private channels by user id
self._private_channels_by_user = {}
self._messages = deque(maxlen=self.max_messages)
def process_listeners(self, listener_type, argument, result):
removed = []
for i, listener in enumerate(self._listeners):
if listener.type != listener_type:
continue
future = listener.future
if future.cancelled():
removed.append(i)
continue
try:
passed = listener.predicate(argument)
except Exception as e:
future.set_exception(e)
removed.append(i)
else:
if passed:
future.set_result(result)
removed.append(i)
if listener.type == ListenerType.chunk:
break
for index in reversed(removed):
del self._listeners[index]
@property
def self_id(self):
u = self.user
return u.id if u else None
@property
def voice_clients(self):
return list(self._voice_clients.values())
def _get_voice_client(self, guild_id):
return self._voice_clients.get(guild_id)
def _add_voice_client(self, guild_id, voice):
self._voice_clients[guild_id] = voice
def _remove_voice_client(self, guild_id):
self._voice_clients.pop(guild_id, None)
def _update_references(self, ws):
for vc in self.voice_clients:
vc.main_ws = ws
def store_user(self, data):
# this way is 300% faster than `dict.setdefault`.
user_id = int(data['id'])
try:
return self._users[user_id]
except KeyError:
user = User(state=self, data=data)
if user.discriminator != '0000':
self._users[user_id] = user
return user
def get_user(self, id):
return self._users.get(id)
def store_emoji(self, guild, data):
emoji_id = int(data['id'])
self._emojis[emoji_id] = emoji = Emoji(guild=guild,
state=self,
data=data)
return emoji
@property
def guilds(self):
return list(self._guilds.values())
def _get_guild(self, guild_id):
return self._guilds.get(guild_id)
def _add_guild(self, guild):
self._guilds[guild.id] = guild
def _remove_guild(self, guild):
self._guilds.pop(guild.id, None)
for emoji in guild.emojis:
self._emojis.pop(emoji.id, None)
del guild
@property
def emojis(self):
return list(self._emojis.values())
def get_emoji(self, emoji_id):
return self._emojis.get(emoji_id)
@property
def private_channels(self):
return list(self._private_channels.values())
def _get_private_channel(self, channel_id):
try:
value = self._private_channels[channel_id]
except KeyError:
return None
else:
self._private_channels.move_to_end(channel_id)
return value
def _get_private_channel_by_user(self, user_id):
return self._private_channels_by_user.get(user_id)
def _add_private_channel(self, channel):
channel_id = channel.id
self._private_channels[channel_id] = channel
if self.is_bot and len(self._private_channels) > 128:
_, to_remove = self._private_channels.popitem(last=False)
if isinstance(to_remove, DMChannel):
self._private_channels_by_user.pop(to_remove.recipient.id,
None)
if isinstance(channel, DMChannel):
self._private_channels_by_user[channel.recipient.id] = channel
def add_dm_channel(self, data):
channel = DMChannel(me=self.user, state=self, data=data)
self._add_private_channel(channel)
return channel
def _remove_private_channel(self, channel):
self._private_channels.pop(channel.id, None)
if isinstance(channel, DMChannel):
self._private_channels_by_user.pop(channel.recipient.id, None)
def _get_message(self, msg_id):
return utils.find(lambda m: m.id == msg_id, self._messages)
def _add_guild_from_data(self, guild):
guild = Guild(data=guild, state=self)
self._add_guild(guild)
return guild
def chunks_needed(self, guild):
for chunk in range(math.ceil(guild._member_count / 1000)):
yield self.receive_chunk(guild.id)
def _get_guild_channel(self, data):
try:
guild = self._get_guild(int(data['guild_id']))
except KeyError:
channel = self.get_channel(int(data['channel_id']))
guild = None
else:
channel = guild and guild.get_channel(int(data['channel_id']))
return channel, guild
async def request_offline_members(self, guilds):
# get all the chunks
chunks = []
for guild in guilds:
chunks.extend(self.chunks_needed(guild))
# we only want to request ~75 guilds per chunk request.
splits = [guilds[i:i + 75] for i in range(0, len(guilds), 75)]
for split in splits:
await self.chunker(split)
# wait for the chunks
if chunks:
try:
await utils.sane_wait_for(chunks,
timeout=len(chunks) * 30.0,
loop=self.loop)
except asyncio.TimeoutError:
log.info('Somehow timed out waiting for chunks.')
async def _delay_ready(self):
try:
launch = self._ready_state.launch
# only real bots wait for GUILD_CREATE streaming
if self.is_bot:
while not launch.is_set():
# this snippet of code is basically waiting 2 seconds
# until the last GUILD_CREATE was sent
launch.set()
await asyncio.sleep(2, loop=self.loop)
guilds = self._ready_state.guilds
if self._fetch_offline:
await self.request_offline_members(guilds)
# remove the state
try:
del self._ready_state
except AttributeError:
pass # already been deleted somehow
# call GUILD_SYNC after we're done chunking
if not self.is_bot:
log.info('Requesting GUILD_SYNC for %s guilds',
len(self.guilds))
await self.syncer([s.id for s in self.guilds])
except asyncio.CancelledError:
pass
else:
# dispatch the event
self.dispatch('ready')
finally:
self._ready_task = None
def parse_ready(self, data):
if self._ready_task is not None:
self._ready_task.cancel()
self._ready_state = ReadyState(launch=asyncio.Event(), guilds=[])
self.clear()
self.user = ClientUser(state=self, data=data['user'])
guilds = self._ready_state.guilds
for guild_data in data['guilds']:
guild = self._add_guild_from_data(guild_data)
if (not self.is_bot and not guild.unavailable) or guild.large:
guilds.append(guild)
for relationship in data.get('relationships', []):
try:
r_id = int(relationship['id'])
except KeyError:
continue
else:
self.user._relationships[r_id] = Relationship(
state=self, data=relationship)
for pm in data.get('private_channels', []):
factory, _ = _channel_factory(pm['type'])
self._add_private_channel(
factory(me=self.user, data=pm, state=self))
self.dispatch('connect')
self._ready_task = asyncio.ensure_future(self._delay_ready(),
loop=self.loop)
def parse_resumed(self, data):
self.dispatch('resumed')
def parse_message_create(self, data):
channel, _ = self._get_guild_channel(data)
message = Message(channel=channel, data=data, state=self)
self.dispatch('message', message)
self._messages.append(message)
def parse_message_delete(self, data):
raw = RawMessageDeleteEvent(data)
self.dispatch('raw_message_delete', raw)
found = self._get_message(raw.message_id)
if found is not None:
self.dispatch('message_delete', found)
self._messages.remove(found)
def parse_message_delete_bulk(self, data):
raw = RawBulkMessageDeleteEvent(data)
self.dispatch('raw_bulk_message_delete', raw)
to_be_deleted = [
message for message in self._messages
if message.id in raw.message_ids
]
for msg in to_be_deleted:
self.dispatch('message_delete', msg)
self._messages.remove(msg)
def parse_message_update(self, data):
raw = RawMessageUpdateEvent(data)
self.dispatch('raw_message_edit', raw)
message = self._get_message(raw.message_id)
if message is not None:
older_message = copy.copy(message)
if 'call' in data:
# call state message edit
message._handle_call(data['call'])
elif 'content' not in data:
# embed only edit
message.embeds = [Embed.from_data(d) for d in data['embeds']]
else:
message._update(channel=message.channel, data=data)
self.dispatch('message_edit', older_message, message)
def parse_message_reaction_add(self, data):
emoji_data = data['emoji']
emoji_id = utils._get_as_snowflake(emoji_data, 'id')
emoji = PartialEmoji(animated=emoji_data.get('animated', False),
id=emoji_id,
name=emoji_data['name'])
raw = RawReactionActionEvent(data, emoji)
self.dispatch('raw_reaction_add', raw)
# rich interface here
message = self._get_message(raw.message_id)
if message is not None:
emoji = self._upgrade_partial_emoji(emoji)
reaction = message._add_reaction(data, emoji, raw.user_id)
user = self._get_reaction_user(message.channel, raw.user_id)
if user:
self.dispatch('reaction_add', reaction, user)
def parse_message_reaction_remove_all(self, data):
raw = RawReactionClearEvent(data)
self.dispatch('raw_reaction_clear', raw)
message = self._get_message(raw.message_id)
if message is not None:
old_reactions = message.reactions.copy()
message.reactions.clear()
self.dispatch('reaction_clear', message, old_reactions)
def parse_message_reaction_remove(self, data):
emoji_data = data['emoji']
emoji_id = utils._get_as_snowflake(emoji_data, 'id')
emoji = PartialEmoji(animated=emoji_data.get('animated', False),
id=emoji_id,
name=emoji_data['name'])
raw = RawReactionActionEvent(data, emoji)
self.dispatch('raw_reaction_remove', raw)
message = self._get_message(raw.message_id)
if message is not None:
emoji = self._upgrade_partial_emoji(emoji)
try:
reaction = message._remove_reaction(data, emoji, raw.user_id)
except (AttributeError,
ValueError) as e: # eventual consistency lol
pass
else:
user = self._get_reaction_user(message.channel, raw.user_id)
if user:
self.dispatch('reaction_remove', reaction, user)
def parse_presence_update(self, data):
guild_id = utils._get_as_snowflake(data, 'guild_id')
guild = self._get_guild(guild_id)
if guild is None:
log.warning(
'PRESENCE_UPDATE referencing an unknown guild ID: %s. Discarding.',
guild_id)
return
user = data['user']
member_id = int(user['id'])
member = guild.get_member(member_id)
if member is None:
if 'username' not in user:
# sometimes we receive 'incomplete' member data post-removal.
# skip these useless cases.
return
member = Member(guild=guild, data=data, state=self)
guild._add_member(member)
old_member = member._copy()
member._presence_update(data=data, user=user)
self.dispatch('member_update', old_member, member)
def parse_user_update(self, data):
self.user = ClientUser(state=self, data=data)
def parse_channel_delete(self, data):
guild = self._get_guild(utils._get_as_snowflake(data, 'guild_id'))
channel_id = int(data['id'])
if guild is not None:
channel = guild.get_channel(channel_id)
if channel is not None:
guild._remove_channel(channel)
self.dispatch('guild_channel_delete', channel)
else:
# the reason we're doing this is so it's also removed from the
# private channel by user cache as well
channel = self._get_private_channel(channel_id)
if channel is not None:
self._remove_private_channel(channel)
self.dispatch('private_channel_delete', channel)
def parse_channel_update(self, data):
channel_type = try_enum(ChannelType, data.get('type'))
channel_id = int(data['id'])
if channel_type is ChannelType.group:
channel = self._get_private_channel(channel_id)
old_channel = copy.copy(channel)
channel._update_group(data)
self.dispatch('private_channel_update', old_channel, channel)
return
guild_id = utils._get_as_snowflake(data, 'guild_id')
guild = self._get_guild(guild_id)
if guild is not None:
channel = guild.get_channel(channel_id)
if channel is not None:
old_channel = copy.copy(channel)
channel._update(guild, data)
self.dispatch('guild_channel_update', old_channel, channel)
else:
log.warning(
'CHANNEL_UPDATE referencing an unknown channel ID: %s. Discarding.',
channel_id)
else:
log.warning(
'CHANNEL_UPDATE referencing an unknown guild ID: %s. Discarding.',
guild_id)
def parse_channel_create(self, data):
factory, ch_type = _channel_factory(data['type'])
if factory is None:
log.warning(
'CHANNEL_CREATE referencing an unknown channel type %s. Discarding.',
data['type'])
return
channel = None
if ch_type in (ChannelType.group, ChannelType.private):
channel_id = int(data['id'])
if self._get_private_channel(channel_id) is None:
channel = factory(me=self.user, data=data, state=self)
self._add_private_channel(channel)
self.dispatch('private_channel_create', channel)
else:
guild_id = utils._get_as_snowflake(data, 'guild_id')
guild = self._get_guild(guild_id)
if guild is not None:
channel = factory(guild=guild, state=self, data=data)
guild._add_channel(channel)
self.dispatch('guild_channel_create', channel)
else:
log.warning(
'CHANNEL_CREATE referencing an unknown guild ID: %s. Discarding.',
guild_id)
return
def parse_channel_pins_update(self, data):
channel_id = int(data['channel_id'])
channel = self.get_channel(channel_id)
if channel is None:
log.warning(
'CHANNEL_PINS_UPDATE referencing an unknown channel ID: %s. Discarding.',
channel_id)
return
last_pin = utils.parse_time(
data['last_pin_timestamp']) if data['last_pin_timestamp'] else None
try:
# I have not imported discord.abc in this file
# the isinstance check is also 2x slower than just checking this attribute
# so we're just gonna check it since it's easier and faster and lazier
channel.guild
except AttributeError:
self.dispatch('private_channel_pins_update', channel, last_pin)
else:
self.dispatch('guild_channel_pins_update', channel, last_pin)
def parse_channel_recipient_add(self, data):
channel = self._get_private_channel(int(data['channel_id']))
user = self.store_user(data['user'])
channel.recipients.append(user)
self.dispatch('group_join', channel, user)
def parse_channel_recipient_remove(self, data):
channel = self._get_private_channel(int(data['channel_id']))
user = self.store_user(data['user'])
try:
channel.recipients.remove(user)
except ValueError:
pass
else:
self.dispatch('group_remove', channel, user)
def parse_guild_member_add(self, data):
guild = self._get_guild(int(data['guild_id']))
if guild is None:
log.warning(
'GUILD_MEMBER_ADD referencing an unknown guild ID: %s. Discarding.',
data['guild_id'])
return
member = Member(guild=guild, data=data, state=self)
guild._add_member(member)
guild._member_count += 1
self.dispatch('member_join', member)
def parse_guild_member_remove(self, data):
guild = self._get_guild(int(data['guild_id']))
if guild is not None:
user_id = int(data['user']['id'])
member = guild.get_member(user_id)
if member is not None:
guild._remove_member(member)
guild._member_count -= 1
self.dispatch('member_remove', member)
else:
log.warning(
'GUILD_MEMBER_REMOVE referencing an unknown guild ID: %s. Discarding.',
data['guild_id'])
def parse_guild_member_update(self, data):
guild = self._get_guild(int(data['guild_id']))
user = data['user']
user_id = int(user['id'])
if guild is None:
log.warning(
'GUILD_MEMBER_UPDATE referencing an unknown guild ID: %s. Discarding.',
data['guild_id'])
return
member = guild.get_member(user_id)
if member is not None:
old_member = copy.copy(member)
member._update(data, user)
self.dispatch('member_update', old_member, member)
else:
log.warning(
'GUILD_MEMBER_UPDATE referencing an unknown member ID: %s. Discarding.',
user_id)
def parse_guild_emojis_update(self, data):
guild = self._get_guild(int(data['guild_id']))
if guild is None:
log.warning(
'GUILD_EMOJIS_UPDATE referencing an unknown guild ID: %s. Discarding.',
data['guild_id'])
return
before_emojis = guild.emojis
for emoji in before_emojis:
self._emojis.pop(emoji.id, None)
guild.emojis = tuple(
map(lambda d: self.store_emoji(guild, d), data['emojis']))
self.dispatch('guild_emojis_update', guild, before_emojis,
guild.emojis)
def _get_create_guild(self, data):
if data.get('unavailable') == False:
# GUILD_CREATE with unavailable in the response
# usually means that the guild has become available
# and is therefore in the cache
guild = self._get_guild(int(data['id']))
if guild is not None:
guild.unavailable = False
guild._from_data(data)
return guild
return self._add_guild_from_data(data)
async def _chunk_and_dispatch(self, guild, unavailable):
chunks = list(self.chunks_needed(guild))
await self.chunker(guild)
if chunks:
try:
await utils.sane_wait_for(chunks,
timeout=len(chunks),
loop=self.loop)
except asyncio.TimeoutError:
log.info('Somehow timed out waiting for chunks.')
if unavailable == False:
self.dispatch('guild_available', guild)
else:
self.dispatch('guild_join', guild)
def parse_guild_create(self, data):
unavailable = data.get('unavailable')
if unavailable == True:
# joined a guild with unavailable == True so..
return
guild = self._get_create_guild(data)
# check if it requires chunking
if guild.large:
if unavailable == False:
# check if we're waiting for 'useful' READY
# and if we are, we don't want to dispatch any
# event such as guild_join or guild_available
# because we're still in the 'READY' phase. Or
# so we say.
try:
state = self._ready_state
state.launch.clear()
state.guilds.append(guild)
except AttributeError:
# the _ready_state attribute is only there during
# processing of useful READY.
pass
else:
return
# since we're not waiting for 'useful' READY we'll just
# do the chunk request here if wanted
if self._fetch_offline:
asyncio.ensure_future(self._chunk_and_dispatch(
guild, unavailable),
loop=self.loop)
return
# Dispatch available if newly available
if unavailable == False:
self.dispatch('guild_available', guild)
else:
self.dispatch('guild_join', guild)
def parse_guild_sync(self, data):
guild = self._get_guild(int(data['id']))
guild._sync(data)
def parse_guild_update(self, data):
guild = self._get_guild(int(data['id']))
if guild is not None:
old_guild = copy.copy(guild)
guild._from_data(data)
self.dispatch('guild_update', old_guild, guild)
else:
log.warning(
'GUILD_UPDATE referencing an unknown guild ID: %s. Discarding.',
data['id'])
def parse_guild_delete(self, data):
guild = self._get_guild(int(data['id']))
if guild is None:
log.warning(
'GUILD_DELETE referencing an unknown guild ID: %s. Discarding.',
data['id'])
return
if data.get('unavailable', False) and guild is not None:
# GUILD_DELETE with unavailable being True means that the
# guild that was available is now currently unavailable
guild.unavailable = True
self.dispatch('guild_unavailable', guild)
return
# do a cleanup of the messages cache
self._messages = deque(
(msg for msg in self._messages if msg.guild != guild),
maxlen=self.max_messages)
self._remove_guild(guild)
self.dispatch('guild_remove', guild)
def parse_guild_ban_add(self, data):
# we make the assumption that GUILD_BAN_ADD is done
# before GUILD_MEMBER_REMOVE is called
# hence we don't remove it from cache or do anything
# strange with it, the main purpose of this event
# is mainly to dispatch to another event worth listening to for logging
guild = self._get_guild(int(data['guild_id']))
if guild is not None:
try:
user = User(data=data['user'], state=self)
except KeyError:
pass
else:
member = guild.get_member(user.id) or user
self.dispatch('member_ban', guild, member)
def parse_guild_ban_remove(self, data):
guild = self._get_guild(int(data['guild_id']))
if guild is not None:
if 'user' in data:
user = self.store_user(data['user'])
self.dispatch('member_unban', guild, user)
def parse_guild_role_create(self, data):
guild = self._get_guild(int(data['guild_id']))
if guild is None:
log.warning(
'GUILD_ROLE_CREATE referencing an unknown guild ID: %s. Discarding.',
data['guild_id'])
return
role_data = data['role']
role = Role(guild=guild, data=role_data, state=self)
guild._add_role(role)
self.dispatch('guild_role_create', role)
def parse_guild_role_delete(self, data):
guild = self._get_guild(int(data['guild_id']))
if guild is not None:
role_id = int(data['role_id'])
role = utils.find(lambda r: r.id == role_id, guild.roles)
try:
guild._remove_role(role)
except ValueError:
return
else:
self.dispatch('guild_role_delete', role)
else:
log.warning(
'GUILD_ROLE_DELETE referencing an unknown guild ID: %s. Discarding.',
data['guild_id'])
def parse_guild_role_update(self, data):
guild = self._get_guild(int(data['guild_id']))
if guild is not None:
role_data = data['role']
role_id = int(role_data['id'])
role = utils.find(lambda r: r.id == role_id, guild.roles)
if role is not None:
old_role = copy.copy(role)
role._update(role_data)
self.dispatch('guild_role_update', old_role, role)
else:
log.warning(
'GUILD_ROLE_UPDATE referencing an unknown guild ID: %s. Discarding.',
data['guild_id'])
def parse_guild_members_chunk(self, data):
guild_id = int(data['guild_id'])
guild = self._get_guild(guild_id)
members = data.get('members', [])
for member in members:
m = Member(guild=guild, data=member, state=self)
existing = guild.get_member(m.id)
if existing is None or existing.joined_at is None:
guild._add_member(m)
log.info('Processed a chunk for %s members in guild ID %s.',
len(members), guild_id)
self.process_listeners(ListenerType.chunk, guild, len(members))
def parse_voice_state_update(self, data):
guild = self._get_guild(utils._get_as_snowflake(data, 'guild_id'))
channel_id = utils._get_as_snowflake(data, 'channel_id')
if guild is not None:
if int(data['user_id']) == self.user.id:
voice = self._get_voice_client(guild.id)
if voice is not None:
ch = guild.get_channel(channel_id)
if ch is not None:
voice.channel = ch
member, before, after = guild._update_voice_state(data, channel_id)
if member is not None:
self.dispatch('voice_state_update', member, before, after)
else:
log.warning(
'VOICE_STATE_UPDATE referencing an unknown member ID: %s. Discarding.',
data['user_id'])
else:
# in here we're either at private or group calls
call = self._calls.get(channel_id)
if call is not None:
call._update_voice_state(data)
def parse_voice_server_update(self, data):
try:
key_id = int(data['guild_id'])
except KeyError:
key_id = int(data['channel_id'])
vc = self._get_voice_client(key_id)
if vc is not None:
asyncio.ensure_future(vc._create_socket(key_id, data))
def parse_typing_start(self, data):
channel, guild = self._get_guild_channel(data)
if channel is not None:
member = None
user_id = utils._get_as_snowflake(data, 'user_id')
if isinstance(channel, DMChannel):
member = channel.recipient
elif isinstance(channel, TextChannel):
member = guild.get_member(user_id)
elif isinstance(channel, GroupChannel):
member = utils.find(lambda x: x.id == user_id,
channel.recipients)
if member is not None:
timestamp = datetime.datetime.utcfromtimestamp(
data.get('timestamp'))
self.dispatch('typing', channel, member, timestamp)
def parse_relationship_add(self, data):
key = int(data['id'])
old = self.user.get_relationship(key)
new = Relationship(state=self, data=data)
self.user._relationships[key] = new
if old is not None:
self.dispatch('relationship_update', old, new)
else:
self.dispatch('relationship_add', new)
def parse_relationship_remove(self, data):
key = int(data['id'])
try:
old = self.user._relationships.pop(key)
except KeyError:
pass
else:
self.dispatch('relationship_remove', old)
def _get_reaction_user(self, channel, user_id):
if isinstance(channel, TextChannel):
return channel.guild.get_member(user_id)
return self.get_user(user_id)
def get_reaction_emoji(self, data):
emoji_id = utils._get_as_snowflake(data, 'id')
if not emoji_id:
return data['name']
try:
return self._emojis[emoji_id]
except KeyError:
return PartialEmoji(animated=data.get('animated', False),
id=emoji_id,
name=data['name'])
def _upgrade_partial_emoji(self, emoji):
emoji_id = emoji.id
if not emoji_id:
return emoji.name
try:
return self._emojis[emoji_id]
except KeyError:
return emoji
def get_channel(self, id):
if id is None:
return None
pm = self._get_private_channel(id)
if pm is not None:
return pm
for guild in self.guilds:
channel = guild.get_channel(id)
if channel is not None:
return channel
def create_message(self, *, channel, data):
return Message(state=self, channel=channel, data=data)
def receive_chunk(self, guild_id):
future = self.loop.create_future()
listener = Listener(ListenerType.chunk, future,
lambda s: s.id == guild_id)
self._listeners.append(listener)
return future
class FontConfig(object):
def __init__(self):
self._fontconfig = self._load_fontconfig_library()
self._search_cache = OrderedDict()
self._cache_size = 20
def dispose(self):
while len(self._search_cache) > 0:
self._search_cache.popitem().dispose()
self._fontconfig.FcFini()
self._fontconfig = None
def create_search_pattern(self):
return FontConfigSearchPattern(self._fontconfig)
def find_font(self, name, size=12, bold=False, italic=False):
result = self._get_from_search_cache(name, size, bold, italic)
if result:
return result
search_pattern = self.create_search_pattern()
search_pattern.name = name
search_pattern.size = size
search_pattern.bold = bold
search_pattern.italic = italic
result = search_pattern.match()
self._add_to_search_cache(search_pattern, result)
return result
def have_font(self, name):
result = self.find_font(name)
if result:
# Check the name matches, fontconfig can return a default
if name and result.name and result.name.lower() != name.lower():
return False
return True
else:
return False
def char_index(self, ft_face, character):
return self._fontconfig.FcFreeTypeCharIndex(ft_face, ord(character))
def _add_to_search_cache(self, search_pattern, result_pattern):
self._search_cache[(search_pattern.name, search_pattern.size,
search_pattern.bold,
search_pattern.italic)] = result_pattern
if len(self._search_cache) > self._cache_size:
self._search_cache.popitem(last=False)[1].dispose()
def _get_from_search_cache(self, name, size, bold, italic):
result = self._search_cache.get((name, size, bold, italic), None)
if result and result.is_valid:
return result
else:
return None
@staticmethod
def _load_fontconfig_library():
fontconfig = pyglet.lib.load_library('fontconfig')
fontconfig.FcInit()
fontconfig.FcPatternBuild.restype = c_void_p
fontconfig.FcPatternCreate.restype = c_void_p
fontconfig.FcFontMatch.restype = c_void_p
fontconfig.FcFreeTypeCharIndex.restype = c_uint
fontconfig.FcPatternAddDouble.argtypes = [c_void_p, c_char_p, c_double]
fontconfig.FcPatternAddInteger.argtypes = [c_void_p, c_char_p, c_int]
fontconfig.FcPatternAddString.argtypes = [c_void_p, c_char_p, c_char_p]
fontconfig.FcConfigSubstitute.argtypes = [c_void_p, c_void_p, c_int]
fontconfig.FcDefaultSubstitute.argtypes = [c_void_p]
fontconfig.FcFontMatch.argtypes = [c_void_p, c_void_p, c_void_p]
fontconfig.FcPatternDestroy.argtypes = [c_void_p]
fontconfig.FcPatternGetFTFace.argtypes = [
c_void_p, c_char_p, c_int, c_void_p
]
fontconfig.FcPatternGet.argtypes = [
c_void_p, c_char_p, c_int, c_void_p
]
return fontconfig
class FIFODictOrderedDict:
"""A simple FIFO mapping between keys and values. Implemented using an
OrderedDict. When the max. capacity is reached, the key/value pair that
has been in the dict the longest time is removed.
"""
def __init__(self, size, finalizer=None):
"""Create a FIFODictOrderedDict with the given maximum size.
Arguments:
- size: Determines the maximum size of the dict.
- finalizer: If finalizer is given, it must be a callable
f(key, value). It is then called, when a item is removed due to
the size of the dict reaching the maximum (finalizer is NOT called
when an item is explicitly deleted with del d[key] or when the
dict is cleared).
"""
if not isinstance(size, type(0)):
raise TypeError("size must be an int")
if not size > 0:
raise ValueError("size must be positive")
if finalizer is not None and not callable(finalizer):
raise TypeError("finalizer must be None or a callable")
self.__size = size
self.__data = OrderedDict()
self.__finalizer = finalizer
def add(self, key, val):
"""Add a key/value pair to the dict.
If a pair p with the same key already exists, p is replaced by the
new pair n, but n gets p's position in the FIFO dict and is deleted
when the old pair p would have been deleted. When the maximum
capacity is reached, the pair with the oldest key is deleted
from the dict.
The argument key is the key and the argument val is the value.
"""
if key in self.__data:
self.__data[key] = val # Replace old value
elif len(self.__data) < self.__size:
# The dict is not full yet. Just add the new pair.
self.__data[key] = val
else:
# The dict is full. We have to delete the oldest item first.
if self.__finalizer:
(delKey, delValue) = self.__data.popitem(last=False)
self.__finalizer(delKey, delValue)
else:
self.__data.popitem(last=False)
self.__data[key] = val
def get(self, key, default=None):
"""Find and return the element a given key maps to.
Look for the given key in the dict and return the associated value
if found. If not found, the value of default is returned.
"""
return self.__data.get(key, default)
def clear(self):
"""Delete all key/value pairs from the dict"""
self.__data = OrderedDict()
def __setitem__(self, key, item):
self.add(key, item)
def __getitem__(self, key):
return self.__data[key]
def __len__(self):
return len(self.__data)
def __str__(self):
allitems = []
for key in self.__data:
val = self.__data[key]
item = "%s: %s" % (str(key), str(val))
allitems.append(item)
return "{%s}" % ", ".join(allitems)
def __contains__(self, item):
return (item in self.__data)
def __delitem__(self, item):
if item not in self.__data:
raise KeyError(item)
del self.__data[item]
def __iter__(self):
for k in self.__data:
yield k
class Cache(object):
def __init__(self, name, max_entries=1000, keylen=1, lru=False):
if lru:
self.cache = LruCache(max_size=max_entries)
self.max_entries = None
else:
self.cache = OrderedDict()
self.max_entries = max_entries
self.name = name
self.keylen = keylen
self.sequence = 0
self.thread = None
caches_by_name[name] = self.cache
def check_thread(self):
expected_thread = self.thread
if expected_thread is None:
self.thread = threading.current_thread()
else:
if expected_thread is not threading.current_thread():
raise ValueError(
"Cache objects can only be accessed from the main thread")
def get(self, *keyargs):
if len(keyargs) != self.keylen:
raise ValueError("Expected a key to have %d items", self.keylen)
if keyargs in self.cache:
cache_counter.inc_hits(self.name)
return self.cache[keyargs]
cache_counter.inc_misses(self.name)
raise KeyError()
def update(self, sequence, *args):
self.check_thread()
if self.sequence == sequence:
# Only update the cache if the caches sequence number matches the
# number that the cache had before the SELECT was started (SYN-369)
self.prefill(*args)
def prefill(self, *args): # because I can't *keyargs, value
keyargs = args[:-1]
value = args[-1]
if len(keyargs) != self.keylen:
raise ValueError("Expected a key to have %d items", self.keylen)
if self.max_entries is not None:
while len(self.cache) >= self.max_entries:
self.cache.popitem(last=False)
self.cache[keyargs] = value
def invalidate(self, *keyargs):
self.check_thread()
if len(keyargs) != self.keylen:
raise ValueError("Expected a key to have %d items", self.keylen)
# Increment the sequence number so that any SELECT statements that
# raced with the INSERT don't update the cache (SYN-369)
self.sequence += 1
self.cache.pop(keyargs, None)
def invalidate_all(self):
self.check_thread()
self.sequence += 1
self.cache.clear()
class Domains(object):
def __init__(self, master):
self.master = master
self.log = master.log
self.domain = master.mongo.domain
self.catecory = master.mongo.catecory
self.template = master.mongo.template
self.article = master.mongo.spider_article
self.exc_article = master.mongo.spider_exc
self.html_file = master.mongo.html_file
self.url_redis = redis.Redis(**master.conf.redis_url)
self.domains = PQDict(key=lambda x: x.value.domain['_id'])
self.fetching = dict()
self.waiting = OrderedDict()
self.doing = dict()
self.sync_round = 1800
self.last = -1
master.fetch_adapter.register(['cate','art','page','img'],
self.fetch_get, self.fetch, self.fetch_cancel)
master.handle_adapter.register(['cate'],
self.handle_get, self.handle, self.handle_cancel)
@property
def counter(self):
res = {
'fetch':len(self.fetching),
'domain_wait':len(self.waiting),
'domain_doing':len(self.doing),
'fetch_cate':0,
'fetch_art':0,
'fetch_img':0,
'fetch_page':0,
}
for cmd, _ in self.fetching.itervalues():
res['fetch_' + cmd] += 1
return res
def is_fetching(self, key):
return key in self.fetching
def is_doing(self, key):
return key in self.fetching \
or key in self.waiting \
or key in self.doing
def sync(self, quit=False):
self.sync_last(quit)
self.sync_other(quit)
def sync_last(self, quit):
if not quit and self.sync_round > time.time() - self.last:
return
last = time.time()
domains, tpls = {}, {}
for domain in self.domains:
domains.update(domain.back(self.last, last))
tpls.update(domain.tpls.back(self.last, last))
if not quit:
self.load_domains(self.last, last)
self.load_tpls(self.last, last)
for domain in domains.itervalues():
self.domain.save(domain)
for tpl in tpls.itervalues():
self.template.save(tpl)
self.last = last
def sync_other(self, quit):
for domain in self.domains:
domain.sync(quit)
self.domains.heapify()
def load_domains(self, start, end):
doc = {'status':'common', 'last':{'$gt':start, '$lte':end}}
domains = self.domain.find(doc)
for domain in domains:
if domain['_id'] not in self.domains:
xdomain = Domain(self, domain)
self.domains.put(xdomain)
if start > 0:
xdomain.load()
else:
self.domains[domain['_id']].update(domain)
if domains.count() > 0:
self.log.info('load %d domains.' % domains.count())
def load_tpls(self, start, end):
doc = {'status':'common', 'last':{'$gt':start, '$lte':end}}
tpls = self.template.find(doc)
for tpl in tpls:
with self.domains.get2do(tpl['domain']) as domain:
domain.tpls.put(tpl)
if tpls.count > 0:
self.log.info('load %d tpls.' % tpls.count())
def add_url(self, key, domain):
return self.url_redis.sadd(domain, key)
def has_url(self, key, domain):
return self.url_redis.sismember(domain, key)
def selector(self, domain, tpl):
if domain in self.domains:
return self.domains[domain].tpls.selector(tpl)
return {}
def new_arts(self, article, urls):
with self.domains.get2do(article['domain']) as domain:
return domain.new_arts(urls, 'art', article)
return 0
def fetch_page(self, page):
with self.domains.get2do(page['domain']) as domain:
return domain.pages.put(page)
return False
def fetch_img(self, img):
with self.domains.get2do(img['domain']) as domain:
return domain.imgs.put(img)
return False
def new_handle(self, key, value):
self.waiting[key] = value
def quit(self):
for cmd, task in self.fetching.itervalues():
self.domains[task['domain']].cancel(cmd, task)
self.fetching.clear()
for cmd, task, _ in self.waiting.itervalues():
self.domains[task['domain']].cancel(cmd, task)
self.waiting.clear()
for cmd, task in self.doing.itervalues():
self.domains[task['domain']].cancel(cmd, task)
self.doing.clear()
def fetch_get(self, count):
tasks = []
if not self.domains:
return tasks
null = 0
for _ in xrange(count):
cmd, task = None, None
with self.domains.get2do() as domain:
cmd, task = domain.get()
if not task:
null += 1
if null >= 5:
break
self.domains.heapify()
continue
tasks.append({'key':task['_id'], 'cmd':cmd, 'info':task})
self.fetching[task['_id']] = (cmd, task)
return tasks
def fetch_cancel(self, key, cmd):
if key in self.fetching:
cmd, task = self.fetching.pop(key)
with self.domains.get2do(task['domain']) as domain:
domain.cancel(cmd, task)
def fetch(self, key, cmd, res):
if key in self.fetching:
cmd, task = self.fetching.pop(key)
with self.domains.get2do(task['domain']) as domain:
domain.fetch(cmd, task, res)
def handle_get(self, count, **kwargs):
tasks = []
for _ in xrange(min(len(self.waiting), count)):
cmd, task, ext = self.waiting.popitem()[1]
tasks.append({'key':task['_id'], 'cmd':cmd, 'info':task, 'ext':ext})
self.doing[task['_id']] = (cmd, task)
return tasks
def handle_cancel(self, key, cmd):
if key in self.doing:
cmd, task = self.doing.pop(key)
with self.domains.get2do(task['domain']) as domain:
domain.cancel(cmd, task)
def handle(self, key, cmd, res):
if key in self.doing:
cmd, task = self.doing.pop(key)
with self.domains.get2do(task['domain']) as domain:
domain.handle(cmd, task, res)
class UVar(object):
"""
A class to hold a variable associated with the UGrid. Data can be on the
nodes, edges, etc. -- "UGrid Variable"
It holds an array of the data, as well as the attributes associated
with that data -- this is mapped to a netcdf variable with
attributes(attributes get stored in the netcdf file)
"""
def __init__(self, name, location, data=None, attributes=None):
"""
create a UVar object
:param name: the name of the variable (depth, u_velocity, etc.)
:type name: string
:param location: the type of grid element the data is associated with:
'node', 'edge', or 'face'
:param data: The data itself
:type data: 1-d numpy array or array-like object ().
If you have a list or tuple, it should be something that can be
converted to a numpy array (list, etc.)
"""
self.name = name
if location not in ['node', 'edge', 'face', 'boundary']:
raise ValueError("location must be one of: "
"'node', 'edge', 'face', 'boundary'")
self.location = location
if data is None:
# Could be any data type, but we'll default to float
self._data = np.zeros((0, ), dtype=np.float64)
else:
self._data = asarraylike(data)
# FixMe: we need a separate attribute dict -- we really do'nt want all this
# getting mixed up with the python object attributes
self.attributes = {} if attributes is None else attributes
# if the data is a netcdf variable, pull the attributes from there
try:
for attr in data.ncattrs():
self.attributes[attr] = data.getncattr(attr)
except AttributeError: # must not be a netcdf variable
pass
self._cache = OrderedDict()
# def update_attrs(self, attrs):
# """
# update the attributes of the UVar object
# :param attr: Dict containing attributes to be added to the object
# """
# for key, val in attrs.items():
# setattr(self, key, val)
@property
def data(self):
"""
The data associated with the UVar -- a numpy array-like object with the actual data
"""
return self._data
@data.setter
def data(self, data):
self._data = asarraylike(data)
@data.deleter
def data(self):
self._data = self._data = np.zeros((0, ), dtype=np.float64)
@property
def shape(self):
"""
The shape of the data array associated with the UVar.
"""
return self.data.shape
@property
def max(self):
""" maximum value of the variable """
return np.max(self._data)
@property
def min(self):
""" minimum value of the variable """
return np.min(self._data)
@property
def dtype(self):
""" data type of the variable """
return self.data.dtype
@property
def ndim(self):
""" number of dimensions of the variable """
return self.data.ndim
def __getitem__(self, item):
"""
Transfers responsibility to the data's __getitem__ if not cached
"""
rv = None
if str(item) in self._cache:
rv = self._cache[str(item)]
else:
rv = self._data.__getitem__(item)
self._cache[str(item)] = rv
if len(self._cache) > 3:
self._cache.popitem(last=False)
return rv
def __str__(self):
print("in __str__, data is:", self.data)
msg = ("UVar object: {0:s}, on the {1:s}s, and {2:d} data "
"points\nAttributes: {3}").format
return msg(self.name, self.location, len(self.data), self.attributes)
def __len__(self):
return len(self.data)
__author__ = 'rakesh'
#What is orderedDict
#An OrderedDict is a dict that remembers the order that keys were first inserted. If a new entry overwrites an existing entry, the original insertion position is left unchanged. Deleting an entry and reinserting it will move it to the end.
from collections import OrderedDict
LRUCache = OrderedDict()
count = 1
for i in [2, 4, 5, 6, 7, 9, 10]:
LRUCache[i] = count
count += 1
print LRUCache
LRUCache.popitem(last=False)
print LRUCache
# print LRUCache
#
# # del LRUCache[2] #if you delete the old entry then you ened
# # LRUCache[2] = ""
# # print LRUCache
#
# #The best thing about orderedDict is that it remembers the order in which the keys are inserted. If the new entry overwrites and existing entry, the original insertion position will remain unchanged. Deleting an entry and reinseeting it will move to the end. This is really good for LRU Cache will deletes the entry
#
# #dictionary sorted by key
#
# LRUCache = sorted(LRUCache.items(), key= lambda t: t[1])
# print LRUCache #It is really amazing that you can sort your LRUCache
#
# #there are lots of operation that you can perform
# # >>> # regular unsorted dictionary
class OrderedSet(object):
def __init__(self, iterable=[]):
self._dict = OrderedDict([(elem, True) for elem in iterable])
def add(self, elem):
self._dict[elem] = True
def remove(self, elem):
del self._dict[elem]
def discard(self, elem):
self._dict.pop(elem, None)
def pop(self, last=True):
return self._dict.popitem(last)[0]
def clear(self):
self._dict.clear()
def copy(self):
tmp = OrderedSet()
tmp._dict = self._dict.copy()
return tmp
#--------------------------------------------------------------------------
# Boolean set functions
#--------------------------------------------------------------------------
def isdisjoint(self, other):
for elem in self._dict:
if elem in other: return False
for elem in other:
if elem in self._dict: return False
return True
def issubset(self, other):
for elem in self._dict:
if elem not in other: return False
return True
def issuperset(self, other):
for elem in other:
if elem not in self._dict: return False
return True
# Since __eq__ will return False for two OrderedSets with same elements but
# a different order so provide a separate function.
def isequal(self, other):
if not isinstance(other, self.__class__): return NotImplemented
if len(self) != len(other): return False
for elem in self._dict.keys():
if elem not in other._dict: return False
return True
#--------------------------------------------------------------------------
# Set operations
#--------------------------------------------------------------------------
def union(self, *others):
tmp = self.copy()
for other in others:
if isinstance(other, self.__class__):
tmp._dict.update(other._dict)
else:
tmp._dict.update({key: True for key in other})
return tmp
def intersection(self, *others):
tmp = self.copy()
if not others: return tmp
tmp2 = set(self._dict.keys()).intersection(*others)
for key in self._dict.keys():
if key not in tmp2: tmp._dict.pop(key, None)
return tmp
def difference(self, *others):
tmp = self.copy()
if not others: return tmp
tmp2 = set()
tmp2.update(*others)
for key in tmp2:
tmp._dict.pop(key, None)
return tmp
def symmetric_difference(self, other):
tmp = set(self._dict.keys())
tmp.intersection_update(other)
tmp2 = self.union(other)
for key in tmp:
del tmp2._dict[key]
return tmp2
def update(self, *others):
for other in others:
if isinstance(other, self.__class__):
self._dict.update(other._dict)
else:
self._dict.update({key: True for key in other})
def intersection_update(self, *others):
if not others: return
tmp = set(self._dict.keys())
tmp2 = set(tmp).intersection(*others)
for key in tmp:
if key not in tmp2: del self._dict[key]
def difference_update(self, *others):
if not others: return
tmp2 = set()
tmp2.update(*others)
for key in tmp2:
self._dict.pop(key, None)
def symmetric_difference_update(self, other):
tmp = set(self._dict.keys())
tmp.intersection_update(other)
self.update(other)
for key in tmp:
del self._dict[key]
#--------------------------------------------------------------------------
# Special functions to support set and container operations
#--------------------------------------------------------------------------
def __contains__(self, elem):
"""Implemement set 'in' operator."""
return elem in self._dict
def __bool__(self):
"""Implemement set bool operator."""
return bool(self._dict)
def __len__(self):
return len(self._dict)
def __iter__(self):
for k, v in self._dict.items():
yield k
def __eq__(self, other):
# Not sure why this shouldn't raise a TypeError for set but I want the
# behaviour to be consistent with standard set.
"""Overloaded boolean operator."""
if isinstance(other, self.__class__):
return self._dict == other._dict
elif isinstance(other, set):
if len(self) != len(other): return False
if not self.issubset(other): return False
return self.issuperset(other)
return NotImplemented
def __ne__(self, other):
"""Overloaded boolean operator."""
result = self.__eq__(other)
if result is NotImplemented: return NotImplemented
return not result
def __lt__(self, other):
"""Implemement set < operator."""
if not isinstance(other, self.__class__) and \
not isinstance(other, set):
return NotImplemented
return self <= other and self != other
def __le__(self, other):
"""Implemement set <= operator."""
if not isinstance(other, self.__class__) and \
not isinstance(other, set):
return NotImplemented
return self.issubset(other)
def __gt__(self, other):
"""Implemement set > operator."""
if not isinstance(other, self.__class__) and \
not isinstance(other, set):
return NotImplemented
return self >= other and self != other
def __ge__(self, other):
"""Implemement set >= operator."""
if not isinstance(other, self.__class__) and \
not isinstance(other, set):
return NotImplemented
return self.issuperset(other)
def __or__(self, other):
"""Implemement set | operator."""
return self.union(other)
def __and__(self, other):
"""Implemement set & operator."""
return self.intersection(other)
def __sub__(self, other):
"""Implemement set - operator."""
return self.difference(other)
def __xor__(self, other):
"""Implemement set ^ operator."""
return self.symmetric_difference(other)
def __ior__(self, other):
"""Implemement set |= operator."""
self.update(other)
return self
def __iand__(self, other):
"""Implemement set &= operator."""
self.intersection_update(other)
return self
def __isub__(self, other):
"""Implemement set -= operator."""
self.difference_update(other)
return self
def __ixor__(self, other):
"""Implemement set ^= operator."""
self.symmetric_difference_update(other)
return self
#--------------------------------------------------------------------------
# String representation
#--------------------------------------------------------------------------
def __str__(self):
if not self: return "set()"
return "{" + ", ".join([repr(e) for e in self]) + "}"
def __repr__(self):
if not self: return "{}()".format(type(self).__name__)
return self.__str__()
if previous is not None and previous != data or len(tmp.keys()) == 0:
f = SlmCcdbEntry(runStart, run - 1, previous)
if len(tmp.keys()) == 0:
f.runMax = None
runStart = run
print(f)
f.writeFile(directory='./slm-data')
fcmd.write(f.getCommand() + '\n')
previous = data
fcmd.close()
###################################################################
###################################################################
fcmd = open('hwp2ccdb.sh', 'w')
runStart, hwpStart = hwpData.popitem(False)
while len(hwpData) > 0:
run, hwp = hwpData.popitem(False)
if hwp != hwpStart or len(hwpData) == 0:
if hwp != hwpStart:
runEnd = run - 1
else:
runEnd = run
entry = HwpCcdbEntry(runStart, runEnd, data={'hwp': hwpStart})
print(entry)
entry.writeFile(directory='./hwp-data')
fcmd.write(entry.getCommand() + '\n')
runStart, hwpStart = run, hwp
fcmd.close()
print('')
class ConsistencyTestClass(object):
"""
This class contains all the Fortran consistency check tests.
The logs dictionary contains a deque object with a list of all source lines
that fail a particular test (given as the logs key)
"""
def __init__(self):
self.logs = OrderedDict()
##############
def process(self):
"""
Check results from all tests that have been run:
1. For each test:
i. log (as info) test description and number of lines of code that fail the test
ii. log (as info) all lines of code that do not conform to standards
2. return total number of errors across all tests
"""
tot_err_cnt = 0
while self.logs:
desc, log = self.logs.popitem(last=False)
err_cnt = len(log)
LOGGER.info("* %s: %d error(s) found", desc, err_cnt)
while log:
msg = log.popleft()
LOGGER.info(" %s", msg)
tot_err_cnt += err_cnt
return tot_err_cnt
##############
def init_log(self, description):
"""
If self.logs does not already have a key for description, set
self.logs[description] to an empty deque list.
"""
if description not in self.logs.keys():
self.logs[description] = deque([])
##############
def check_for_hard_tabs(self, file_and_line, line):
"""
Log any lines containing hard tabs
"""
test_desc = 'Check for hard tabs'
self.init_log(test_desc)
if "\t" in line:
self.logs[test_desc].append(
"%s: %s" % (file_and_line, line.replace("\t", 2 * UNIBLOCK)))
##############
def check_for_trailing_whitespace(self, file_and_line, line):
"""
Log any lines containing trailing whitespace
"""
test_desc = 'Check for trailing white space'
self.init_log(test_desc)
full_line_len = len(line)
no_trailing_space_len = len(line.rstrip(" "))
if no_trailing_space_len < full_line_len:
self.logs[test_desc].append(
"%s: %s" %
(file_and_line, line.rstrip(" ") +
(full_line_len - no_trailing_space_len) * UNIBLOCK))
##############
def check_line_length(self,
file_and_line,
line,
comment_char="!",
max_len=132):
"""
Log any lines exceeding max_len
Currently ignores comment characters / anything following
"""
test_desc = 'Check length of lines'
self.init_log(test_desc)
line_len = len(line.split(comment_char)[0].rstrip(" "))
if line_len > max_len:
self.logs[test_desc].append("%s: %s" %
(file_and_line, line[:max_len] +
(line_len - max_len) * UNIBLOCK))
##############
def check_case_in_module_statements(self,
file_and_line,
line,
comment_char="!"):
"""
The following module statements should be all lowercase:
* implicit none
* public
* private
* save
Note that at some point we may want to remove "save" altogether,
since it is implicit in a module
"""
test_desc = 'Check for case sensitive statements'
self.init_log(test_desc)
statements = ["implicit none", "public", "private", "save"]
# ignore comments, and strip all white space
line_without_comments = line.split(comment_char)[0].strip(" ")
if line_without_comments.lower() in statements:
if line_without_comments not in statements:
self.logs[test_desc].append("%s: %s" % (file_and_line, line))
##############
def check_for_spaces(self, file_and_line, line, comment_char="!"):
"""
The following statements should all include spaces:
* else if
* end if
* else where
* end where
* end do
* end module
"""
test_desc = 'Check for spaces in statements'
self.init_log(test_desc)
statements = [
"elseif", "endif", "elsewhere", "endwhere", "enddo", "endmodule"
]
# ignore comments, and strip all white space
line_without_comments = line.split(comment_char)[0].strip(" ")
for bad_statement in statements:
if line_without_comments.lower().startswith(bad_statement):
self.logs[test_desc].append(
"%s: %s" % (file_and_line, line_without_comments))
break
##############
def check_for_double_quotes(self, file_and_line, line):
"""
All Fortran strings should appear as 'string', not "string"
"""
test_desc = 'Check for double quotes in statements'
self.init_log(test_desc)
if '"' in line:
self.logs[test_desc].append("%s: %s" % (file_and_line, line))
##############
def check_logical_statements(self, file_and_line, line):
"""
Use symbols, not words, for logical operators:
* >=, not .ge.
* >, not .gt.
* <=, not .le.
* <, not .lt.
* ==, not .eq.
* /=, not .ne.
"""
test_desc = 'Check for unwanted logical operators'
self.init_log(test_desc)
operators = ['.ge.', '.gt.', '.le.', '.lt.', '.eq.', '.ne.']
for operator in operators:
if operator in line:
self.logs[test_desc].append("%s: %s" % (file_and_line, line))
break
##############
def check_r8_settings(self, file_and_line, line, comment_char="!"):
"""
Make sure all real numbers are cast as r8
"""
test_desc = 'Check for r8'
self.init_log(test_desc)
import re
# Looking for decimal numbers that do not end in _r8
# Edge cases:
# 1. ignore decimals in comments
# 2. Ignore decimals in format strings (e.g. E24.16)
# 3. Allow 1.0e-2_r8
line_without_comments = line.split(comment_char)[0].strip(" ")
# Regex notes
# 1. (?<!\w) -- do not match numbers immediately following a letter,
# such as E24 or I0 (these are Fortran format strings)
# [regex refers to this as a negative lookbehind]
# 2. \d+\. -- match N consecutive decimal digits (for N>=1) followed
# by a decimal point
# 3. (\d+[eE]([+-])?)? -- Optionally match a number followed by either
# e or E (optionally followed by + or -)
# 4. \d+ -- match N consecutive decimal digits (for N>=1)
# 5. (?!\d+|_|[eE]) -- do not match a decimal, an underscore, an e,
# or an E
# [regex refers to this as a negative lookahead]
regex = r'(?<!\w)\d+\.(\d+[eE]([+-])?)?\d+(?!\d+|_|[eE])'
valid = re.compile(regex)
if valid.search(line_without_comments):
self.logs[test_desc].append("%s: %s" % (file_and_line, line))
class UMVar(object):
"""
A class to group multiple UVars (or other data sources) and retrieve common information.
All the variables grouped in this class must have the same shape, location, and unique
names.
TODO: Add attribues that all grouped variables have in common to the UMVar?
"""
def __init__(self, name, location='none', data=None, attributes=None):
"""
:param name: the name of the data (depth, u_velocity, etc.)
:type name: string
:param location: the type of grid element the data is associated with:
'node', 'edge', or 'face' the data is assigned to
:param data: the data
:type data: list-like of data sources that satisfy the conditions of util.asarraylike.
All data sources must have the same shape.
Examples: netCDF Dataset, numpy array
"""
self.name = name
if location not in ['node', 'edge', 'face', 'boundary', 'none']:
raise ValueError(
"location must be one of: 'node', 'edge', 'face', 'boundary', or 'none'"
)
self.location = location
if len(data) == 1:
raise ValueError(
"UMVar need at least 2 data sources of the same size and shape"
)
if not all([isarraylike(d) for d in data]):
raise ValueError("Data must satisfy isarraylike or be a UVar")
self.shape = data[0].shape
if not all([d.shape == self.shape for d in data]):
raise ValueError(
"All data sources must be the same size and shape")
for d in data:
setattr(self, d.name, d)
self.variables = [d.name for d in data]
self._cache = OrderedDict()
def add_var(self, var):
if var.shape != self.shape:
raise ValueError('Variable {0} has incorrect shape {1}'.format(
var.name, var.shape))
if var.name in self.variables:
raise ValueError('Variable {0} already exists in UMVar'.format(
var.name))
self.variables.append(var.name)
setattr(self, var.name, var)
def __getitem__(self, item):
if str(item) in self._cache:
return self._cache[str(item)]
else:
rv = np.ma.column_stack([
self.__getattribute__(var).__getitem__(item)
for var in self.variables
])
self._cache[str(item)] = rv
if len(self._cache) > 3:
self._cache.popitem(last=False)
return rv
class LRUQueue(object):
"""
LRUQueue is a data structure that orders objects by
their insertion time, and supports an update/touch operation
that resets an item to the newest slot.
This is an example of a priority queue, ordered by
insertion time, with explicit support for "touch".
"""
def __init__(self, key=lambda n: n):
"""
The `key` parameter may be provided if the
items in the queue are complex.
The `key` parameter should select a unique "id" field from
each item.
"""
super(LRUQueue, self).__init__()
self._q = OrderedDict()
self._key = key
def push(self, v):
k = self._key(v)
self._q[k] = v
def pop(self):
return self._q.popitem(last=False)[1]
def touch(self, v):
"""
Reset the given value back to the newest slot.
"""
k = self._key(v)
del self._q[k]
self._q[k] = v
def size(self):
return len(self._q)
def __len__(self):
return self.size()
@staticmethod
def test():
q = LRUQueue()
assert q.size() == 0
assert len(q) == 0
q.push(0)
assert q.size() == 1
assert len(q) == 1
assert q.pop() == 0
assert q.size() == 0
assert len(q) == 0
q.push(0)
q.push(1)
assert q.pop() == 0
assert q.pop() == 1
q.push(0)
q.push(1)
q.touch(0)
assert q.pop() == 1
assert q.pop() == 0
q = LRUQueue(key=lambda n: n[0])
q.push([0])
assert q.pop() == [0]
q.push([0])
q.push([1])
assert q.pop() == [0]
assert q.pop() == [1]
return True
class PositionsManager(Initializable):
MAX_POSITIONS_COUNT = 2000
def __init__(self, config, trader, exchange_manager):
super().__init__()
self.logger = get_logger(self.__class__.__name__)
self.config, self.trader, self.exchange_manager = config, trader, exchange_manager
self.positions_initialized = False # TODO
self.positions = OrderedDict()
async def initialize_impl(self):
self._reset_positions()
def get_open_positions(self, symbol=None, since=-1, limit=-1):
return self._select_positions(True, symbol, since, limit)
def get_closed_positions(self, symbol=None, since=-1, limit=-1):
return self._select_positions(False, symbol, since, limit)
def upsert_position(self, position_id, raw_position) -> bool:
if position_id not in self.positions:
self.positions[position_id] = self._create_position_from_raw(
raw_position)
self._check_positions_size()
return True
return self._update_position_from_raw(self.positions[position_id],
raw_position)
def upsert_position_instance(self, position) -> bool:
if position.position_id not in self.positions:
self.positions[position.position_id] = position
self._check_positions_size()
return True
# TODO
return False
# private
def _check_positions_size(self):
if len(self.positions) > self.MAX_POSITIONS_COUNT:
self._remove_oldest_positions(int(self.MAX_POSITIONS_COUNT / 2))
def _create_position_from_raw(self, raw_position):
position = Position(self.trader)
position.update_position_from_raw(raw_position)
return position
def _update_position_from_raw(self, position, raw_position):
return position.update_position_from_raw(raw_position)
def _select_positions(self,
status=PositionStatus.OPEN,
symbol=None,
since=-1,
limit=-1):
positions = [
position for position in self.positions.values()
if (position.status == status and (
symbol is None or (symbol and position.symbol == symbol)) and (
since == -1 or (since and position.timestamp < since)))
]
return positions if limit == -1 else positions[0:limit]
def _reset_positions(self):
self.positions_initialized = False
self.positions = OrderedDict()
def _remove_oldest_positions(self, nb_to_remove):
for _ in range(nb_to_remove):
self.positions.popitem(last=False)
def wordCount(file, input):
tel_sorted_by_value = OrderedDict(
sorted(file.items(), key=lambda x: x[1], reverse=True))
for x in xrange(input):
print tel_sorted_by_value.popitem(last=False)
class LRUCache(HasStrictTraits):
""" A least-recently used cache.
Items older than `size()` accesses are dropped from the cache.
"""
size = Int
# Called with the key and value that was dropped from the cache
cache_drop_callback = Callable
# This event contains the set of cached cell keys whenever it changes
updated = Event()
_lock = Instance(RLock, args=())
_cache = Instance(OrderedDict)
def __init__(self, size, **traits):
self.size = size
self._initialize_cache()
super(LRUCache, self).__init__(**traits)
def _initialize_cache(self):
with self._lock:
if self._cache is None:
self._cache = OrderedDict()
else:
self._cache.clear()
def _renew(self, key):
with self._lock:
r = self._cache.pop(key)
self._cache[key] = r
return r
# -------------------------------------------------------------------------
# LRUCache interface
# -------------------------------------------------------------------------
def __contains__(self, key):
with self._lock:
return key in self._cache
def __len__(self):
with self._lock:
return len(self._cache)
def __getitem__(self, key):
with self._lock:
return self._renew(key)
def __setitem__(self, key, result):
try:
dropped = None
with self._lock:
self._cache[key] = result
self._renew(key)
if self.size < len(self._cache):
dropped = self._cache.popitem(last=False)
if dropped and self.cache_drop_callback is not None:
self.cache_drop_callback(*dropped)
finally:
self.updated = list(self.keys())
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
def items(self):
with self._lock:
return list(self._cache.items())
def keys(self):
with self._lock:
return list(self._cache.keys())
def values(self):
with self._lock:
return list(self._cache.values())
def clear(self):
with self._lock:
self._initialize_cache()
self.updated = []
print(q)
from collections import defaultdict
dd = defaultdict(lambda: 'Null')
dd['kay1'] = '123'
print(dd['kay1'], dd['kay2'])
from collections import OrderedDict
d = dict([('a', 1), ('b', 2), ('c', 3)])
print(d)
od = OrderedDict([('b', 2), ('a', 1), ('c', 3)])
print(od, od.keys()) # order by insert order
od['e'] = 4
od['e'] = 7
print(od)
od.popitem()
print(od)
od.popitem(last=False)
print(od)
class LastUpdateOrderDict(OrderedDict):
def __init__(self, capacity):
super(LastUpdateOrderDict, self).__init__()
self.capacity = capacity
def __setitem__(self, key, value):
containsKey = 1 if key in self else 0
if len(self) - containsKey >= self.capacity:
last = self.popitem(last=False)
print('remove:', last)
class PyPIEvents:
"""
When treated as an iterator, this class yields (package, version,
timestamp, action) tuples indicating new packages or package versions
registered on PyPI where action is one of 'create', 'source', or 'remove'.
A small attempt is made to avoid duplicate reports, but we don't attempt to
avoid reporting stuff already in the database (it's simpler to just start
from the beginning of PyPI's log and work through it).
The iterator only retrieves a small batch of entries at a time as PyPI
(very sensibly) limits the number of entries in a single query (to 50,000
at the time of writing), so the instance will need repeated querying to
retrieve all rows (this works in our favour though as it means there's an
obvious place to poll for control events between batches).
:param str pypi_root:
The web address at which to find the PyPI XML-RPC server.
:param int serial:
The serial number of the event from which to start reading.
:param int retries:
The number of retries the class may attempt if the HTTP connection
fails.
:param int cache_size:
The size of the internal cache used to attempt to avoid duplicate
reports.
"""
# pylint: disable=too-few-public-methods
add_file_re = re.compile(r'^add ([^ ]+) file')
create_re = re.compile(r'^create$')
remove_re = re.compile(r'^remove(?: (?:package|release))?')
def __init__(self,
pypi_xmlrpc='https://pypi.org/pypi',
serial=0,
retries=3,
cache_size=1000):
self.retries = retries
self.next_read = datetime.now(tz=UTC)
self.serial = serial
# Keep a list of the last cache_size (package, version) tuples so we
# can make a vague attempt at reducing duplicate reports
self.cache = OrderedDict()
self.cache_size = cache_size
self.transport = PiWheelsTransport()
self.client = xmlrpc.client.ServerProxy(pypi_xmlrpc, self.transport)
def _get_events(self):
# On rare occasions we get some form of HTTP improper state, or DNS
# lookups fail. In this case just return an empty list and try again
# later. If we get a protocol error with error 5xx it's a server-side
# problem, so we again return an empty list and try later
try:
return self.client.changelog_since_serial(self.serial)
except (OSError, http.client.ImproperConnectionState):
return []
except xmlrpc.client.ProtocolError as exc:
if exc.errcode >= 500:
return []
else:
raise
def __iter__(self):
# The next_read flag is used to delay reads to PyPI once we get to the
# end of the event log entries
if datetime.now(tz=UTC) > self.next_read:
events = self._get_events()
if events:
for (package, version, timestamp, action, serial) in events:
timestamp = datetime.fromtimestamp(timestamp, tz=UTC)
match = self.add_file_re.search(action)
if match is not None:
action = ('source'
if match.group(1) == 'source' else 'create')
try:
self.cache.move_to_end((package, version))
except KeyError:
self.cache[(package, version)] = (timestamp,
action)
yield (package, version, timestamp, action)
else:
(last_timestamp,
last_action) = self.cache[(package, version)]
if (last_action, action) == ('create', 'source'):
self.cache[(package,
version)] = (last_timestamp,
action)
yield (package, version, last_timestamp,
action)
while len(self.cache) > self.cache_size:
self.cache.popitem(last=False)
elif self.create_re.search(action) is not None:
yield (package, None, timestamp, 'create')
elif self.remove_re.search(action) is not None:
# If version is None here, indicating package deletion
# we could search and remove all corresponding versions
# from the cache but, frankly, it's not worth it
if version is not None:
self.cache.pop((package, version), None)
yield (package, version, timestamp, 'remove')
self.serial = serial
else:
# If the read is empty we've reached the end of the event log
# or an error has occurred; make sure we don't bother PyPI for
# another 10 seconds
self.next_read = datetime.now(tz=UTC) + timedelta(seconds=10)
class CachedStore(BaseStore):
"""
A thread-safe cached persistent store of the subscriber database.
Prerequisite: persistent_store need to be thread safe
"""
def __init__(self, persistent_store, cache_capacity=512, loop=None):
self._lock = threading.Lock()
self._cache = OrderedDict()
self._cache_capacity = cache_capacity
self._persistent_store = persistent_store
self._on_ready = OnDataReady(loop=loop)
def add_subscriber(self, subscriber_data):
"""
Method that adds the subscriber.
"""
sid = SIDUtils.to_str(subscriber_data.sid)
with self._lock:
if sid in self._cache:
raise DuplicateSubscriberError(sid)
self._persistent_store.add_subscriber(subscriber_data)
self._cache_put(sid, subscriber_data)
self._on_ready.add_subscriber(subscriber_data)
@contextmanager
def edit_subscriber(self, subscriber_id):
"""
Context manager to modify the subscriber data.
"""
with self._lock:
if subscriber_id in self._cache:
data = self._cache_get(subscriber_id)
subscriber_data = copy.deepcopy(data)
else:
subscriber_data = \
self._persistent_store.get_subscriber_data(subscriber_id)
yield subscriber_data
self._persistent_store.update_subscriber(subscriber_data)
self._cache_put(subscriber_id, subscriber_data)
def delete_subscriber(self, subscriber_id):
"""
Method that deletes a subscriber, if present.
"""
with self._lock:
if subscriber_id in self._cache:
del self._cache[subscriber_id]
self._persistent_store.delete_subscriber(subscriber_id)
def delete_all_subscribers(self):
"""
Method that removes all the subscribers from the store
"""
with self._lock:
self._cache_clear()
self._persistent_store.delete_all_subscribers()
def resync(self, subscribers):
"""
Method that should resync the store with the mentioned list of
subscribers. The resync leaves the current state of subscribers
intact.
Args:
subscribers - list of subscribers to be in the store.
"""
with self._lock:
self._cache_clear()
self._persistent_store.resync(subscribers)
self._on_ready.resync(subscribers)
def get_subscriber_data(self, subscriber_id):
"""
Method that returns the subscriber data for the subscriber.
"""
with self._lock:
if subscriber_id in self._cache:
return self._cache_get(subscriber_id)
else:
subscriber_data = \
self._persistent_store.get_subscriber_data(subscriber_id)
self._cache_put(subscriber_id, subscriber_data)
return subscriber_data
def list_subscribers(self):
"""
Method that returns the list of subscribers stored.
Note: this method is not cached since it's easier to get the whole list
from persistent store
"""
return self._persistent_store.list_subscribers()
async def on_ready(self):
return await self._on_ready.event.wait()
def _cache_get(self, k):
"""
Get from the LRU cache. Move the last hit entry to the end.
"""
self._cache.move_to_end(k)
return self._cache[k]
def _cache_put(self, k, v):
"""
Put to the LRU cache. Evict the first item if full.
"""
if self._cache_capacity == len(self._cache):
self._cache.popitem(last=False)
self._cache[k] = v
def _cache_list(self):
return list(self._cache.keys())
def _cache_clear(self):
self._cache.clear()
def parseTypeDefinitions(self, outname, xmlDescription):
def typeReady(element, types, xmlNamespaces):
"Are all member types defined?"
parentname = type_aliases.get(
element.get("Name"), element.get("Name")
) # If a type contains itself, declare that type as available
for child in element:
if child.tag == "{http://opcfoundation.org/BinarySchema/}Field":
childname = get_type_name(child.get("TypeName"))[1]
if childname != "Bit" and childname != parentname:
try:
get_type_for_name(child.get("TypeName"), types,
xmlNamespaces)
except TypeNotDefinedException:
# Type is using other types which are not yet loaded, try later
return False
return True
def unknownTypes(element, types, xmlNamespaces):
"Return all unknown types (for debugging)"
unknowns = []
for child in element:
if child.tag == "{http://opcfoundation.org/BinarySchema/}Field":
try:
get_type_for_name(child.get("TypeName"), types,
xmlNamespaces)
except TypeNotDefinedException:
# Type is using other types which are not yet loaded, try later
unknowns.append(child.get("TypeName"))
return unknowns
def structWithOptionalFields(element):
"Is this a structure with optional fields?"
opt_fields = []
for child in element:
if child.tag != "{http://opcfoundation.org/BinarySchema/}Field":
continue
typename = child.get("TypeName")
if typename and get_type_name(typename)[1] == "Bit":
if re.match(re.compile('.+Specified'), child.get("Name")):
opt_fields.append(child.get("Name"))
elif child.get("Name") == "Reserved1":
if len(opt_fields) + int(child.get("Length")) != 32:
return False
else:
break
else:
return False
else:
return False
for child in element:
switchfield = child.get("SwitchField")
if switchfield and switchfield in opt_fields:
opt_fields.remove(switchfield)
return len(opt_fields) == 0
def structWithBitFields(element):
"Is this a structure with bitfields?"
for child in element:
typename = child.get("TypeName")
if typename and get_type_name(typename)[1] == "Bit":
return True
return False
snippets = OrderedDict()
xmlDoc = etree.iterparse(xmlDescription, events=['start-ns'])
xmlNamespaces = dict([node for _, node in xmlDoc])
if xmlDoc.root.get("TargetNamespace"):
targetNamespace = xmlDoc.root.get("TargetNamespace")
if not targetNamespace in self.namespaceIndexMap:
raise RuntimeError(
"TargetNamespace '{targetNs}' is not listed in namespace index mapping. "
"Use the following option to define the mapping (can be used multiple times). "
"--namespaceMap=X:{targetNs} where X is the resulting namespace index in the server."
.format(targetNs=targetNamespace))
else:
raise RuntimeError(
"TargetNamespace Attribute not defined in BSD file.")
for typeXml in xmlDoc.root:
if not typeXml.get("Name"):
continue
name = typeXml.get("Name")
snippets[name] = typeXml
detectLoop = len(snippets) + 1
while len(snippets) > 0:
if detectLoop == len(snippets):
name, typeXml = snippets.popitem()
raise RuntimeError(
"Infinite loop detected or type not found while processing types "
+ name + ": unknonwn subtype " +
str(unknownTypes(typeXml, self.types, xmlNamespaces)) +
". If the unknown subtype is 'Bit', then maybe a struct with "
+
"optional fields is defined wrong in the .bsd-file. If not, maybe "
+
"you need to import additional types with the --import flag. "
+
"E.g. '--import==UA_TYPES#/path/to/deps/ua-nodeset/Schema/"
+ "Opc.Ua.Types.bsd'")
detectLoop = len(snippets)
for name, typeXml in list(snippets.items()):
if (targetNamespace in self.types
and name in self.types[targetNamespace]
) or name in excluded_types:
del snippets[name]
continue
if not typeReady(typeXml, self.types, xmlNamespaces):
continue
if structWithBitFields(
typeXml) and not structWithOptionalFields(typeXml):
continue
if name in builtin_types:
new_type = BuiltinType(name)
elif typeXml.tag == "{http://opcfoundation.org/BinarySchema/}EnumeratedType":
new_type = EnumerationType(outname, typeXml,
targetNamespace)
elif typeXml.tag == "{http://opcfoundation.org/BinarySchema/}OpaqueType":
new_type = OpaqueType(
outname, typeXml, targetNamespace,
get_base_type_for_opaque(name)['name'])
elif typeXml.tag == "{http://opcfoundation.org/BinarySchema/}StructuredType":
try:
new_type = StructType(outname, typeXml,
targetNamespace, self.types,
xmlNamespaces)
except TypeNotDefinedException:
# Type is using other types which are not yet loaded, try later
continue
else:
raise Exception("Type not known")
self.insert_type(new_type)
del snippets[name]
def run_epoch(game_obj,
mcts_operator_p,
mcts_operator_q,
p_configs,
q_configs,
max_game_length=62,
prior_nodes_repo_size=300):
# report the new board
_state = game_obj.states[-1][0]
json_state = json.dumps(_state.tolist())
UDPSock_BOARD.send(json_state)
# read parameters
tree_depth_p = p_configs['tree_depth']
memory_p = p_configs['memory']
allowed_time_p = p_configs['allowed_time']
tree_depth_q = q_configs['tree_depth']
memory_q = q_configs['memory']
allowed_time_q = q_configs['allowed_time']
# start
trace = []
DATA_p = []
# DATA_q = []
# in-simulation simluation re-use
#prior_nodes_repo = OrderedDict(enumerate(range(prior_nodes_repo_size)))
prior_nodes_repo = OrderedDict()
# actual play
_iter = 0
det_pass = {1: False, -1: False} # check if any move is possible or not
while True:
########################################################################
########################################################################
########################################################################
# emergency stop
_iter += 1
if _iter > max_game_length:
break
# end if
rootLogger.info("===============\n")
rootLogger.info(" Game turn: %d\n" % _iter)
rootLogger.info("===============")
rootLogger.info("Player B turn\n")
# mcts_operator_p's turn
# (1) select last n steps
last_states = game_obj.states[-memory_p:]
# (2) feed to agent
mcts_operator_p.feed(last_states)
# (2.5) find if node has been sampled before
# rootLogger.info('last_states')
# rootLogger.info(last_states)
_key = make_node_key(last_states)
prior_node = prior_nodes_repo.get(_key)
# (3) agent run mcts
rootLogger.info(" = start MCTS\n")
results = mcts_operator_p.start_mcts(tree_depth=tree_depth_p,
allowed_time=allowed_time_p,
prior_node=prior_node,
logger=logger_p) # end start
updated_node, sampled_nodes = results
# (3.5) store sampled nodes for re-use
for _node in sampled_nodes:
key = make_node_key(_node.data['states'])
prior_nodes_repo.update({key: _node})
# end for
# control the size of repo
excess_n = len(prior_nodes_repo) - prior_nodes_repo_size
for _counter in range(excess_n):
prior_nodes_repo.popitem(last=False)
# end for
# (4) record the MCTS run (first layer)
rootLogger.info(' = collect MCTS data\n')
mcts_probs = mcts_operator_p.get_mcts_probs()
DATA_p.append({'states': last_states, 'probs': mcts_probs})
# (5) select move
move_node = mcts_operator_p.select_move()
# check surrender or pass
if move_node == (-1, 0): # surrender
winner = -cur_player
break
elif move_node == (-1, 1): # pass
rootLogger.info(' - PASS\n')
det_pass[cur_player] = True
if det_pass[
-cur_player] == True: # opponent also no move, end game
break
# end if
# end if
# (6) record move
cur_player = last_states[-1][1]
move = move_node.data['move']
_confidence = mcts_operator_p.max_confidence
rootLogger.info(' - PLAYS %s (confidence: %4.1f%%)\n' %
(str(move), _confidence * 100))
# send confidence value
UDPSock_MISC.send(_confidence)
trace.append({
'player': cur_player,
'move': move,
'confidence': _confidence
})
# actually transit into the next state
move_state = move_node.data['states'][-1]
game_obj.set_state(move_state)
# report the new board
_state = game_obj.states[-1][0]
json_state = json.dumps(_state.tolist())
UDPSock_BOARD.send(json_state)
det_pass[cur_player] = False
det_pass[-cur_player] = False
########################################################################
########################################################################
########################################################################
# emergency stop
_iter += 1
if _iter > max_game_length:
break
# end if
rootLogger.info("===============\n")
rootLogger.info(" Game turn: %d\n" % _iter)
rootLogger.info("===============\n")
rootLogger.info("Player W turn\n")
# mcts_operator_q's turn
# (1) select last n steps
last_states = game_obj.states[-memory_q:]
# (2.5) find if node has been sampled before
_key = make_node_key(last_states)
prior_node = prior_nodes_repo.get(_key)
# start background run
control_Q = queue.Queue()
sample_Q = queue.LifoQueue(
) # keep only last two run (in fact need last one only)
sample_Q.put({})
args = (
control_Q,
sample_Q,
mcts_operator_q,
last_states,
prior_node,
make_node_key,
tree_depth_q,
allowed_time_q,
logger_q,
)
prc_mcts = Process(target=mcts_background_run, args=args)
prc_mcts.start()
rootLogger.info(' = start background MCTS process\n')
# ===== USER INTERACTION HERE ==== #
# find all legal moves
nexts = game_obj.get_all_legal_moves(last_states)
if len(nexts) == 0:
rootLogger.info('No legal moves for player\n')
rootLogger.info(
'No more possible moves. Click any empty cell to continue.\n')
(user_input, addr) = UDPSock_IN.recvfrom(buf)
move_node = (-1, 1)
input_move = (-1, 1)
else:
legal_moves = {item['move']: item for item in nexts}
# read input from player
while True:
try:
rootLogger.info(' = legal moves: %s\n' %
str(list(legal_moves.keys())))
rootLogger.info(' = waiting for player to respond\n')
# user_input = input('Enter your move: ')
while True:
(user_input, addr) = UDPSock_IN.recvfrom(buf)
print(' = receive input:', user_input)
break
# end while
input_move = tuple(map(int, user_input.split()[:2]))
if input_move not in legal_moves:
err_msg = 'Invalid move'
UDPSock.sendto(err_msg.encode(),
('localhost', output_socket_post))
raise (Exception(err_msg))
# end if
move_state = legal_moves[input_move]['state']
break
except Exception as e:
rootLogger.info('Error: %s\n' % str(e))
# end try
# end while
# end if
# get the samples from background run and put to repo
_repo = sample_Q.get_nowait()
prior_nodes_repo.update(_repo)
# send signal through queue to end the background process
control_Q.put('END')
# control_Q.put('END')
# prc_mcts.terminate()
# for i in range(20):
# rootLogger.info ('killed background process')
# control the size of repo
excess_n = len(prior_nodes_repo) - prior_nodes_repo_size
for _counter in range(excess_n):
prior_nodes_repo.popitem(last=False)
# end for
# (4) record the MCTS run (first layer)
# note: optional in interactive play
rootLogger.info(' = collect MCTS data\n')
# mcts_probs = mcts_operator_q.get_mcts_probs()
# DATA_q.append({'states': last_states, 'probs': mcts_probs})
# (5) select move
# move_node = mcts_operator_q.select_move()
# (6) record move
cur_player = last_states[-1][1]
# move = move_node.data['move']
# confidence is undefined in interactive mode
# _confidence = mcts_operator_q.max_confidence
# rootLogger.info(' - PLAYS %s (confidence: %4.1f%%)' % (str(move), _confidence*100))
# trace.append({'player': cur_player, 'move': move, 'confidence': _confidence})
trace.append({'player': cur_player, 'move': input_move})
# check surrender or pass
if move_node == (-1, 0): # surrender
winner = -cur_player
break
elif move_node == (-1, 1): # pass
rootLogger.info(' - PASS\n')
det_pass[cur_player] = True
if det_pass[
-cur_player] == True: # opponent also no move, end game
break
# end if
# end if
# actually transit into the next state
# move_state = move_node.data['states'][-1]
game_obj.set_state(move_state)
# report the new board
_state = game_obj.states[-1][0]
json_state = json.dumps(_state.tolist())
UDPSock_BOARD.send(json_state)
det_pass[cur_player] = False
det_pass[-cur_player] = False
# wnd while
# determine winner
final_state = game_obj.states[-1]
winner = game_obj.get_winner(final_state)
# decorate the data
for item in DATA_p:
item['winner'] = winner
# end for
# for item in DATA_q:
# item['winner'] = winner
# # end for
# return {'data_p': DATA_p, 'data_q': DATA_q, 'trace': trace, 'game_obj': game_obj}
return {'data_p': DATA_p, 'trace': trace, 'game_obj': game_obj}
class TileValue(Layer):
"""A wrapper for rendering an image of uint8, where you can choose a colour for each, or show through the background if you want. Has a tile cache, to make it suitably fast to render."""
def __init__(self, values, tile_size=256, cache_size = 1024):
"""Initalises the value viewer; you can use None if you would rather initialise it later."""
# Store various bits of info...
self.tile_size = tile_size
self.cache_size = cache_size
self.visible = True
# Setup the cache and stroe the values...
self.cache = OrderedDict()
self.cache_key = 0 # Incrimented each time the contents is replaced, and included in the tile key - avoids memory churn.
self.set_values(values)
# Indicates if we are using interpolation or not...
self.interpolate = False
# The colours to use for the mask channels, with None to let the background show through...
# (If we index outside the array then that is None.)
self.colour = []
# A temporary, used when creating tiles...
self.temp = cairo.ImageSurface(cairo.FORMAT_ARGB32, tile_size, tile_size)
self.temp_ctx = cairo.Context(self.temp)
def set_values(self, values):
"""Replaces the current values array with a new one."""
# Default values for if None is provided...
if values==None: values = numpy.zeros((480, 640), dtype=numpy.uint8)
# Some shared setup stuff...
stride = cairo.ImageSurface.format_stride_for_width(cairo.FORMAT_A8, values.shape[1])
mask = numpy.empty((values.shape[0], stride), dtype=numpy.uint8)
temp = cairo.ImageSurface.create_for_data(mask, cairo.FORMAT_A8, mask.shape[1], mask.shape[0], stride)
# For each value create an alpha mask...
self.values = []
for i in xrange(values.max()+1):
# Create a mask for this layer...
mask[:,:] = 0
mask[values==i] = 255
# Copy the mask into perminant storage...
ms = cairo.ImageSurface(cairo.FORMAT_A8, mask.shape[1], mask.shape[0])
ctx = cairo.Context(ms)
ctx.set_source_surface(temp, 0, 0)
ctx.paint()
self.values.append(ms)
# Clean up temp - its dependent on the memory in mask...
del temp
# Change the cache key, so we recycle old tiles rather than use them...
self.cache_key += 1
def get_size(self):
"""Returns the tuple (height, width) for the original size."""
if len(self.values)==0: return (480, 640)
return (self.values[0].get_height(), self.values[0].get_width())
def get_interpolate(self):
"""Returns True if the mask is interpolated, False if it is not."""
return self.interpolate
def set_interpolate(self, interpolate):
"""True to interpolate, False to not. Defaults to True"""
self.interpolate = interpolate
def get_colour(self, value):
"""Returns the colour used for the given part of the mask, as 4 tuple with the last element alpha, or None if it is transparent."""
if value < len(self.colour): return self.colour[value]
else: return None
def set_colour(self, value, col = None):
"""Sets the colour for a value, as a 4-tuple (r,g,b,a), or None for totally transparent."""
if len(self.colour)<=value: self.colour += [None] * (value + 1 - len(self.colour))
self.colour[value] = col
self.cache_key += 1
def fetch_tile(self, scale, bx, by, extra = 0):
"""Fetches a tile, given a scale (Multiplication of original dimensions.) and base coordinates (bx, by) - the tile will be the size provided on class initialisation. This request is going through a caching layer, which will hopefuly have the needed tile. If not it will be generated and the tile that has been used the least recently thrown away."""
# Instance the key for this tile and check if it is in the cache, if so grab it, rearrange it to the top of the queue and return it...
key = (scale, bx, by, self.interpolate, self.cache_key)
if key in self.cache:
ret = self.cache[key]
del self.cache[key]
self.cache[key] = ret
return ret
# Select a new tile to render into - either create one or recycle whatever is going to die at the end of the cache...
if len(self.cache)>=(self.cache_size-extra):
ret = self.cache.popitem()[1]
else:
ret = cairo.ImageSurface(cairo.FORMAT_ARGB32, self.tile_size, self.tile_size)
# Render the tile, with the appropriate scaling...
ctx = cairo.Context(ret)
ctx.save()
ctx.set_operator(cairo.OPERATOR_SOURCE)
ctx.set_source_rgba(0.0, 0.0, 0.0, 0.0)
ctx.paint()
ctx.restore()
if scale<8.0:
self.temp_ctx.save()
self.temp_ctx.translate(-bx, -by)
self.temp_ctx.scale(scale, scale)
for i, mask in enumerate(self.values[:len(self.colour)]):
col = self.colour[i]
if col!=None:
self.temp_ctx.set_source_rgba(col[0], col[1], col[2], col[3])
self.temp_ctx.set_operator(cairo.OPERATOR_SOURCE)
self.temp_ctx.paint()
oasp = cairo.SurfacePattern(mask)
oasp.set_filter(cairo.FILTER_BILINEAR if self.interpolate else cairo.FILTER_NEAREST)
self.temp_ctx.set_source(oasp)
self.temp_ctx.set_operator(cairo.OPERATOR_DEST_IN)
self.temp_ctx.paint()
ctx.set_source_surface(self.temp)
ctx.paint()
self.temp_ctx.restore()
else:
# Cairo has an annoying habit of failing if you push the scale too far - this makes it recursive for such situations...
power = math.log(scale) / math.log(2.0)
child_scale = 2**(int(math.floor(power))-2)
remain_scale = scale / child_scale
child_bx = int(math.floor((bx-1) / (remain_scale*0.5*self.tile_size))) * self.tile_size//2
child_by = int(math.floor((by-1) / (remain_scale*0.5*self.tile_size))) * self.tile_size//2
child = self.fetch_tile(child_scale, child_bx, child_by, extra+1)
oasp = cairo.SurfacePattern(child)
oasp.set_filter(cairo.FILTER_BILINEAR if self.interpolate else cairo.FILTER_NEAREST)
ctx.translate(-(bx - child_bx*remain_scale), -(by - child_by*remain_scale))
ctx.scale(remain_scale, remain_scale)
ctx.set_source(oasp)
ctx.paint()
# Store the tile in the cache and return...
self.cache[key] = ret
return ret
def draw(self, ctx, vp):
"""Draws the image into the provided context, matching the provided viewport."""
if not self.visible:
return
# Calculate the scale, base coordinates and number of tiles in each dimension...
scale = vp.width / float(vp.end_x - vp.start_x)
base_x = int(math.floor(vp.start_x * scale))
base_y = int(math.floor(vp.start_y * scale))
tile_bx = int(math.floor(base_x / float(self.tile_size))) * self.tile_size
tile_by = int(math.floor(base_y / float(self.tile_size))) * self.tile_size
# Loop and draw the required tiles, with the correct offset...
y = tile_by
while y < int(base_y+vp.height+1):
x = tile_bx
while x < int(base_x+vp.width+1):
tile = self.fetch_tile(scale, x, y)
ctx.set_source_surface(tile, x - base_x, y - base_y)
ctx.rectangle(x - base_x, y - base_y, self.tile_size, self.tile_size)
ctx.fill()
x += self.tile_size
y += self.tile_size
