def test_keys_appended_to_lru_cache_are_in_correct_order_after_get(self):
context = LRUCache(capacity=5)
context.set("Michael", "Jordan")
context.set("Scotty", "Pippen")
context.get("Michael")
expected = OrderedDict([("Scotty", "Pippen"), ("Michael", "Jordan")])
assert context.cache == expected
def test_get(self):
c = LRUCache(3)
c.put(4, 2)
c.put(1, 1)
c.put(2, 3)
# get the first key
self.assertEqual(c.get(2), 3)
self.assertEqual(c.cache(), {2: 3, 1: 1, 4: 2})
self.assertEqual(c.size(), 3)
# get the second key
self.assertEqual(c.get(1), 1)
self.assertEqual(c.cache(), {1: 1, 2: 3, 4: 2})
self.assertEqual(c.size(), 3)
# get the last key
self.assertEqual(c.get(4), 2)
self.assertEqual(c.cache(), {4: 2, 2: 3, 1: 1})
self.assertEqual(c.size(), 3)
# get a non-existing key
self.assertEqual(c.get(40), -1)
self.assertEqual(c.cache(), {4: 2, 2: 3, 1: 1})
self.assertEqual(c.size(), 3)
def test_read_counter_is_reset_when_read_limit_reached(self):
context = LRUCache(capacity=1)
context.set("Michael", "Jordan")
for _ in range(11):
context.get("Michael")
context.set("Michael", "Jordan")
context.get("Michael")
assert context.get("Michael") == OrderedDict([("Michael", "Jordan")])
def test_key_is_removed_from_lru_cache_after_accessed_10_times(self):
context = LRUCache(capacity=5)
context.set("Michael", "Jordan")
context.set("Scotty", "Pippen")
for _ in range(11):
context.get("Michael")
expected = OrderedDict([("Scotty", "Pippen")])
assert context.cache == expected
def test_set_move_front(self):
cache = LRUCache(2)
cache.set('foo', 10)
cache.set('bar', 20)
cache.set('foo', 40)
cache.set('baz', 30)
self.assertEqual(cache.get('foo'), 40)
self.assertEqual(cache.get('bar'), None)
self.assertEqual(cache.get('baz'), 30)
def test_get_element(self):
cache = LRUCache(3)
cache[1] = '1'
cache[2] = '2'
cache[3] = '3'
self.assertEqual('test', cache.get(11, 'test'))
self.assertEqual("[(1, '1'), (2, '2'), (3, '3')]", str(cache))
self.assertEqual('1', cache.get(1, 'test'))
self.assertEqual("[(2, '2'), (3, '3'), (1, '1')]", str(cache))
self.assertEqual(3, len(cache))
def test_instanciation_without_timeout(self):
cache = LRUCache(5)
cache.put('1', '2')
cache.put('2', '3')
cache.put('3', '4')
cache.put('4', '5')
cache.put('first', 'value')
self.assertEqual(cache.get('1'), '2')
self.assertEqual(cache.get('2'), '3')
self.assertEqual(cache.get('3'), '4')
self.assertEqual(cache.get('4'), '5')
self.assertEqual(cache.get('5'), -1)
def test_insert_over_capacity():
cache = LRUCache(3)
cache.put('key1', 'val1')
cache.put('key2', 'val2')
cache.put('key3', 'val3')
cache.put('key4', 'val4')
with pytest.raises(CacheMissException) as ex:
cache.get('key1')
assert 'key1' in str(ex)
assert cache.get('key2') == 'val2'
assert cache.get('key3') == 'val3'
assert cache.get('key4') == 'val4'
def test_basics(self):
cache = LRUCache(5, 60)
self.assertEqual(cache.capacity, 5)
sample = {"test": True, "foo": "bar"}
cache.set("a", sample)
result = cache.get("a")
self.assertEqual(result, sample)
result = cache.get("a")
self.assertDictEqual(result, sample)
cache.evict("a")
result = cache.get("a")
self.assertEqual(result, None)
def test_stale(self):
cache = LRUCache(5, 1)
sample = {"test": True, "foo": "bar"}
key = "a"
cache.set(key, sample)
self.assertFalse(cache.is_stale(key))
time.sleep(1.2)
self.assertTrue(cache.is_stale(key))
self.assertEqual(cache.get("a"), None)
def test_all(self):
c = LRUCache(5)
c.put(1, 'one')
c.put(2, 'two')
c.put(3, 'three')
c.delete(3)
self.assertEqual(c.size(), 2)
self.assertEqual(c.cache(), {2: 'two', 1: 'one'})
c.put(4, 'four')
c.put(6, 'six')
self.assertEqual(c.get(1), 'one')
self.assertEqual(c.size(), 4)
self.assertEqual(c.cache(), {1: 'one', 6: 'six', 4: 'four', 2: 'two'})
self.assertEqual(c.get(3), -1)
c.put(1, 'five')
self.assertEqual(c.cache(), {1: 'five', 6: 'six', 4: 'four', 2: 'two'})
c.put(12, 'twelve')
self.assertEqual(c.cache(), {
12: 'twelve',
1: 'five',
6: 'six',
4: 'four',
2: 'two'
})
c.put(5, 'five')
self.assertEqual(c.cache(), {
5: 'five',
12: 'twelve',
1: 'five',
6: 'six',
4: 'four'
})
self.assertEqual(c.get(2), -1)
c.reset()
self.assertEqual(c.size(), 0)
self.assertEqual(c.cache(), {})
def main():
testLRU = LRUCache(3)
testLRU.put(1, "Hyd")
testLRU.put(2, "Mumbai")
testLRU.put(3, "Delhi")
assert testLRU.cache == {1: "Hyd", 2: "Mumbai", 3: "Delhi"}
print("Testcase1 passed")
assert testLRU.get(2) == "Mumbai"
print("Testcase2 passed")
assert testLRU.cache == {1: "Hyd", 3: "Delhi", 2: "Mumbai"}
print("Testcase3 passed")
assert testLRU.get(5) == -1
print("Testcase4 passed")
testLRU.put(4, "Chennai")
assert testLRU.get_cache() == {3: "Delhi", 2: "Mumbai", 4: "Chennai"}
print("Testcase5 passed")
print("All test cases passed")
def test_keys(self):
cache = LRUCache(5, 60)
samples = {
"a": {"a": 123, "b": 456},
"bc": {"z": 156},
"foo": 223,
"bar": "adada"}
for key in samples:
cache.set(key, samples[key])
self.assertListEqual(cache.keys(), samples.keys())
for key in samples:
self.assertTrue(cache.has_key(key))
self.assertEqual(cache.get(key), samples[key])
def test_get_empty(self):
cache = LRUCache()
self.assertEqual(cache.get('foo'), None)
def test_update_one():
cache = LRUCache(3)
cache.put('key', 'val')
cache.put('key', 'new val')
assert cache.get('key') == 'new val'
def test_set_full(self):
cache = LRUCache(1)
cache.set('foo', 10)
cache.set('bar', 20)
self.assertEqual(cache.get('foo'), None)
self.assertEqual(cache.get('bar'), 20)
def test_set_replace(self):
cache = LRUCache()
cache.set('foo', 10)
cache.set('foo', 20)
self.assertEqual(cache.get('foo'), 20)
def test_insert_many():
cache = LRUCache(3)
cache.put('key', 'val')
cache.put('key2', 'val2')
cache.put('key3', 'val3')
assert cache.get('key3') == 'val3'
def test_insert_one():
cache = LRUCache(3)
cache.put('key', 'val')
assert cache.get('key') == 'val'
class CoroutineProcessor(object):
name = None
log_rate = 1000
log_rate_skip = 50
def __init__(self, func, targets=None, **kwargs):
if targets is None:
targets = []
elif not hasattr(targets, '__iter__'):
targets = [targets]
self.targets = targets
self.func = func
# Raise errors by default.
self.error = kwargs.pop('error', None)#'raise')
self.name = (
kwargs.get('name') or self.name or
(
self.func.__class__.__name__
if hasattr(func, '__class__') else self.func.__name__
)
)
# self.ids = set()
self.ids = LRUCache(max_size=10000)
self.idx = 0
self.idx_skip = 0
# Make sure we clean up if the program exits before the finalize
# functions are called.
if hasattr(self.func, 'finalize'):
atexit.register(self.finalize)
@coroutine
def run(self, *args, **kwargs):
self.targets = [ t.run(*args, **kwargs) for t in self.targets ]
# =============================================================
def send(res):
if isinstance(res, dict) or not hasattr(res, '__iter__'):
res = [res]
for item in res:
###############
LOGGER.debug(pformat(item))
###############
_id = item.get('id')
# ----------------------------------------
if self.idx % self.log_rate == 0:
LOGGER.debug(
"[{0}] ({1}) {2}".format(self.idx, self.name, _id)
)
# ----------------------------------------
self.idx += 1
if _id:
exists = self.ids.get(_id)
# if _id in self.ids:
if exists:
LOGGER.warning(
"[{0}] ({1}) Document already processed!".format(
_id, self.name
))
else:
# self.ids.add(_id)
self.ids[_id] = 1
# Stop processing this document?
meta = item.get('__meta__', {}) or {}
procs = set(meta.get('processors', []) or [])
procs.add(self.name)
meta['processors'] = list(procs)
item['__meta__'] = meta
if meta.get('skip'):
# -----------------------------------
if self.idx_skip % self.log_rate_skip == 0:
LOGGER.info("[{0}] ({1}) Skipped ({2}). Current: {3} ".format(
self.idx_skip, self.name, _id, pformat(item['__meta__'])
))
# -----------------------------------
self.idx_skip += 1
continue
# Pass the item to the next stage
# in the pipeline.
for target in self.targets:
target.send(item)
# =============================================================
try:
while True:
doc = (yield)
###############
LOGGER.debug(pformat(doc))
###############
if doc is StopIteration:
raise StopIteration
try:
res = self.func(doc, *args, **kwargs)
except Exception as e:
if self.error == 'raise':
raise
else:
msg = """
Error processing document:
Task: {}
Exception: {}
Traceback: {}
""".format(self.name, e, traceback.format_exc())
LOGGER.error(textwrap.dedent(msg))
if DEBUG:
self.to_pickle(doc)
else:
if res:
if res is StopIteration:
raise StopIteration
else:
send(res)
except StopIteration as e:
res = self.finalize()
if res:
# Process final batch, if any.
send(res)
for target in self.targets:
try:
target.send(StopIteration)
except StopIteration:
pass
except GeneratorExit:
LOGGER.warning("[{0}] Quitting...".format(self.name))
def to_pickle(self, doc):
with open(os.path.join(self.path_pickle, self.name or 'UNK'), 'ab') as f:
pickle.dump(doc, f)
def get_state(self):
try:
self.func.get_state()
except AttributeError as e:
raise NotImplementedError
def to_df(self):
try:
self.func.to_df()
except AttributeError as e:
raise NotImplementedError
def to_csv(self, _id=None):
try:
self.func.to_csv(_id=_id)
except AttributeError as e:
raise NotImplementedError
def to_json(self):
try:
self.func.to_json()
except AttributeError as e:
raise NotImplementedError
def finalize(self):
try:
res = self.func.finalize()
except AttributeError as e:
########################
LOGGER.error(
"""
Processor doesn't have a 'finalize' method: {}
""".format(
self.func.name if hasattr(self.func, 'name')
else self.func.__name__
)
)
########################
raise NotImplementedError
except Exception as e:
msg = """
Error finalizing task:
Processor: {}
Exception: {}
Traceback: {}
""".format(self.name, e, traceback.format_exc())
LOGGER.error(textwrap.dedent(msg))
else:
return res
def __eq__(self, other):
eq = (
other and
self.__class__ is other.__class__ and
self.__dict__ == other.__dict__
)
return eq
def __or__(self, other):
if isinstance(other, CoroutineProcessor):
return CoroutineProcessor(
self.func, self.targets + [other], error=self.error
)
elif callable(other):
return CoroutineProcessor(
self.func,
self.targets + [CoroutineProcessor(other, error=self.error)]
)
else:
raise Exception(
"{0} is not composable with {0}".format(self, other)
)
def __getitem__(self, item):
return self.targets.__getitem__(item)
def __len__(self):
return len(self.targets)
def __repr__(self):
return "{0}({1})".format(
self.__class__.__name__,
", ".join(repr(prs) for prs in self.targets)
)
def __call__(self, *args, **kwargs):
return self.run(*args, **kwargs)
# for reusing the same port without waiting for TIME_WAIT to expire
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((server_name, port_number))
# Will listen to only one connection at a time
s.listen( number_of_connections )
LRU = LRUCache( size_of_cache )
while True:
conn, addr = s.accept()
data = conn.recv(1024)
print "DEBUG: " + data
# convert the received data into JSON
json_data = json.loads(data)
"""
fetch the request method
GET denotes a GET cache request to check for hit/miss
POST denotes a SAVE request once a miss happens
"""
request_method = json_data["method"]
if request_method == "GET":
lru_response = LRU.get(json_data["username"])
conn.send(str(lru_response))
elif request_method == "POST":
LRU.set(json_data["username"], json_data["userdata"])
conn.close()
class EDRCmdrs(object):
#TODO these should be player and/or squadron specific
EDR_CMDRS_CACHE = utils2to3.abspathmaker(__file__, 'cache', 'cmdrs.v7.p')
EDR_INARA_CACHE = utils2to3.abspathmaker(__file__, 'cache', 'inara.v7.p')
EDR_SQDRDEX_CACHE = utils2to3.abspathmaker(__file__, 'cache',
'sqdrdex.v2.p')
def __init__(self, edrserver):
self.server = edrserver
self._player = EDPlayerOne()
self.heartbeat_timestamp = None
edr_config = EDRConfig()
self._edr_heartbeat = edr_config.edr_heartbeat()
try:
with open(self.EDR_CMDRS_CACHE, 'rb') as handle:
self.cmdrs_cache = pickle.load(handle)
except:
self.cmdrs_cache = LRUCache(edr_config.lru_max_size(),
edr_config.cmdrs_max_age())
try:
with open(self.EDR_INARA_CACHE, 'rb') as handle:
self.inara_cache = pickle.load(handle)
except:
self.inara_cache = LRUCache(edr_config.lru_max_size(),
edr_config.inara_max_age())
try:
with open(self.EDR_SQDRDEX_CACHE, 'rb') as handle:
self.sqdrdex_cache = pickle.load(handle)
except:
self.sqdrdex_cache = LRUCache(edr_config.lru_max_size(),
edr_config.sqdrdex_max_age())
@property
def player(self):
return self._player
def player_name(self):
return self._player.name
def set_player_name(self, new_player_name):
if (new_player_name != self._player.name):
self._player.force_new_name(new_player_name)
self.__update_squadron_info(force_update=True)
def player_pledged_to(self, power, time_pledged=0):
edr_config = EDRConfig()
delta = time_pledged - self._player.time_pledged if self._player.time_pledged else time_pledged
if power == self._player.power and delta <= edr_config.noteworthy_pledge_threshold(
):
EDRLOG.log(
u"Skipping pledged_to (not noteworthy): current vs. proposed {} vs. {}; {} vs {}"
.format(self._player.power, power, self._player.time_pledged,
time_pledged), "DEBUG")
return False
self._player.pledged_to(power, time_pledged)
since = self._player.pledged_since()
return self.server.pledged_to(power, since)
def __squadron_id(self):
self.__update_squadron_info()
info = self._player.squadron_info()
return info["squadronId"] if info else None
def __update_squadron_info(self, force_update=False):
if self.server.is_anonymous():
return
mark_twain_flag = int(
(EDTime.js_epoch_now() - self.heartbeat_timestamp) /
1000) >= self._edr_heartbeat if self.heartbeat_timestamp else True
if force_update or mark_twain_flag:
info = self.server.heartbeat()
if info:
self.heartbeat_timestamp = info[
"heartbeat"] if "heartbeat" in info else EDTime.js_epoch_now(
)
self._player.squadron_member(
info) if "squadronId" in info else self._player.lone_wolf(
)
else:
self.heartbeat_timestamp = EDTime.js_epoch_now()
self._player.lone_wolf()
def persist(self):
with open(self.EDR_CMDRS_CACHE, 'wb') as handle:
pickle.dump(self.cmdrs_cache,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
with open(self.EDR_INARA_CACHE, 'wb') as handle:
pickle.dump(self.inara_cache,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
with open(self.EDR_SQDRDEX_CACHE, 'wb') as handle:
pickle.dump(self.sqdrdex_cache,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
def evict(self, cmdr):
try:
del self.cmdrs_cache[cmdr.lower()]
except KeyError:
pass
try:
del self.inara_cache[cmdr.lower()]
except KeyError:
pass
try:
sqdr_id = self.__squadron_id()
if sqdr_id:
sq_cmdr_key = u"{}:{}".format(sqdr_id, cmdr.lower())
del self.sqdrdex_cache[sq_cmdr_key]
except KeyError:
pass
def __edr_cmdr(self, cmdr_name, autocreate):
profile = self.cmdrs_cache.get(cmdr_name.lower())
cached = self.cmdrs_cache.has_key(cmdr_name.lower())
if cached or profile:
EDRLOG.log(
u"Cmdr {cmdr} is in the EDR cache with id={cid}".format(
cmdr=cmdr_name, cid=profile.cid if profile else 'N/A'),
"DEBUG")
return profile
profile = self.server.cmdr(cmdr_name, autocreate)
if not profile:
self.cmdrs_cache.set(cmdr_name.lower(), None)
EDRLOG.log(
u"No match on EDR. Temporary entry to be nice on EDR's server.",
"DEBUG")
return None
dex_profile = self.server.cmdrdex(profile.cid)
if dex_profile:
EDRLOG.log(
u"EDR CmdrDex entry found for {cmdr}: {id}".format(
cmdr=cmdr_name, id=profile.cid), "DEBUG")
profile.dex(dex_profile)
self.cmdrs_cache.set(cmdr_name.lower(), profile)
EDRLOG.log(
u"Cached EDR profile {cmdr}: {id}".format(cmdr=cmdr_name,
id=profile.cid), "DEBUG")
return profile
def __edr_sqdrdex(self, cmdr_name, autocreate):
sqdr_id = self.__squadron_id()
if not sqdr_id:
return None
key = u"{}:{}".format(sqdr_id, cmdr_name.lower())
profile = self.sqdrdex_cache.get(key)
if profile:
EDRLOG.log(
u"Cmdr {cmdr} is in the EDR IFF cache for squadron {sqid} with key {key}"
.format(cmdr=cmdr_name, sqid=sqdr_id, key=key), "DEBUG")
return profile
profile = self.__edr_cmdr(cmdr_name, autocreate)
if not profile:
return None
sqdrdex_dict = self.server.sqdrdex(sqdr_id, profile.cid)
if sqdrdex_dict:
EDRLOG.log(
u"EDR SqdrDex {sqid} entry found for {cmdr}@{cid}".format(
sqid=sqdr_id, cmdr=cmdr_name, cid=profile.cid), "DEBUG")
profile.sqdrdex(sqdrdex_dict)
self.sqdrdex_cache.set(u"{}:{}".format(sqdr_id, cmdr_name.lower()),
profile)
EDRLOG.log(
u"Cached EDR SqdrDex {sqid} entry for {cmdr}@{cid}".format(
sqid=sqdr_id, cmdr=cmdr_name, cid=profile.cid), "DEBUG")
return profile.sqdrdex_profile
def __inara_cmdr(self, cmdr_name, check_inara_server):
inara_profile = None
stale = self.inara_cache.is_stale(cmdr_name.lower())
cached = self.inara_cache.has_key(cmdr_name.lower())
if cached and not stale:
inara_profile = self.inara_cache.get(cmdr_name.lower())
EDRLOG.log(
u"Cmdr {} is in the Inara cache (name={})".format(
cmdr_name, inara_profile.name if inara_profile else 'N/A'),
"DEBUG")
elif check_inara_server:
EDRLOG.log(
u"Stale {} or not cached {} in Inara cache. Inara API call for {}."
.format(stale, cached, cmdr_name), "INFO")
inara_profile = self.server.inara_cmdr(cmdr_name)
if inara_profile and inara_profile.name.lower() == cmdr_name.lower(
):
self.inara_cache.set(cmdr_name.lower(), inara_profile)
EDRLOG.log(
u"Cached Inara profile {}: {},{},{},{}".format(
cmdr_name, inara_profile.name, inara_profile.squadron,
inara_profile.role, inara_profile.powerplay), "DEBUG")
elif self.inara_cache.has_key(cmdr_name.lower()):
inara_profile = self.inara_cache.peek(cmdr_name.lower())
self.inara_cache.refresh(cmdr_name.lower())
EDRLOG.log(u"Refresh and re-use stale match in Inara cache.",
"INFO")
else:
inara_profile = None
self.inara_cache.set(cmdr_name.lower(), None)
EDRLOG.log(
u"No match on Inara. Temporary entry to be nice on Inara's server.",
"INFO")
return inara_profile
def cmdr(self, cmdr_name, autocreate=True, check_inara_server=False):
profile = self.__edr_cmdr(cmdr_name, autocreate)
inara_profile = self.__inara_cmdr(cmdr_name, check_inara_server)
if profile is None:
if inara_profile is None:
EDRLOG.log(
u"Failed to retrieve/create cmdr {}".format(cmdr_name),
"ERROR")
return None
else:
return inara_profile
if inara_profile:
EDRLOG.log(
u"Combining info from EDR and Inara for cmdr {}".format(
cmdr_name), "INFO")
profile.complement(inara_profile)
squadron_profile = self.__edr_sqdrdex(cmdr_name, autocreate)
if squadron_profile:
EDRLOG.log(
u"Combining info from Squadron for cmdr {}".format(cmdr_name),
"INFO")
profile.sqdrdex(squadron_profile.sqdrdex_dict())
return profile
def is_friend(self, cmdr_name):
profile = self.__edr_cmdr(cmdr_name, False)
if profile is None:
return False
return profile.is_friend()
def is_ally(self, cmdr_name):
sqdr_id = self.__squadron_id()
if not sqdr_id:
return False
profile = self.__edr_sqdrdex(cmdr_name, False)
if profile:
return profile.is_ally()
return False
def tag_cmdr(self, cmdr_name, tag):
if tag in ["enemy", "ally"]:
return self.__squadron_tag_cmdr(cmdr_name, tag)
else:
return self.__tag_cmdr(cmdr_name, tag)
def contracts(self):
return self.server.contracts()
def contract_for(self, cmdr_name):
if not cmdr_name:
return False
profile = self.cmdr(cmdr_name)
if not profile:
return False
return self.server.contract_for(profile.cid)
def place_contract(self, cmdr_name, reward):
if not cmdr_name:
return False
if reward <= 0:
return self.remove_contract(cmdr_name)
profile = self.cmdr(cmdr_name)
if not profile:
return False
return self.server.place_contract(profile.cid, {
"cname": cmdr_name.lower(),
"reward": reward
})
def remove_contract(self, cmdr_name):
if not cmdr_name:
return False
profile = self.cmdr(cmdr_name)
if not profile:
return False
return self.server.remove_contract(profile.cid)
def __tag_cmdr(self, cmdr_name, tag):
EDRLOG.log(u"Tagging {} with {}".format(cmdr_name, tag), "DEBUG")
profile = self.__edr_cmdr(cmdr_name, False)
if profile is None:
EDRLOG.log(u"Couldn't find a profile for {}.".format(cmdr_name),
"DEBUG")
return False
tagged = profile.tag(tag)
if not tagged:
EDRLOG.log(
u"Couldn't tag {} with {} (e.g. already tagged)".format(
cmdr_name, tag), "DEBUG")
self.evict(cmdr_name)
return False
dex_dict = profile.dex_dict()
EDRLOG.log(u"New dex state: {}".format(dex_dict), "DEBUG")
success = self.server.update_cmdrdex(profile.cid, dex_dict)
self.evict(cmdr_name)
return success
def __squadron_tag_cmdr(self, cmdr_name, tag):
sqdr_id = self.__squadron_id()
if not sqdr_id:
EDRLOG.log(u"Can't tag: not a member of a squadron", "DEBUG")
return False
EDRLOG.log(u"Tagging {} with {} for squadron".format(cmdr_name, tag),
"DEBUG")
profile = self.__edr_sqdrdex(cmdr_name, False)
if profile is None:
EDRLOG.log(
u"Couldn't find a squadron profile for {}.".format(cmdr_name),
"DEBUG")
return False
tagged = profile.tag(tag)
if not tagged:
EDRLOG.log(
u"Couldn't tag {} with {} (e.g. already tagged)".format(
cmdr_name, tag), "DEBUG")
self.evict(cmdr_name)
return False
sqdrdex_dict = profile.sqdrdex_dict()
EDRLOG.log(u"New dex state: {}".format(sqdrdex_dict), "DEBUG")
augmented_sqdrdex_dict = sqdrdex_dict
augmented_sqdrdex_dict["level"] = self._player.squadron_info(
)["squadronLevel"]
augmented_sqdrdex_dict["by"] = self._player.name
success = self.server.update_sqdrdex(sqdr_id, profile.cid,
augmented_sqdrdex_dict)
self.evict(cmdr_name)
return success
def memo_cmdr(self, cmdr_name, memo):
if memo is None:
return self.clear_memo_cmdr(cmdr_name)
EDRLOG.log(u"Writing a note about {}: {}".format(memo, cmdr_name),
"DEBUG")
profile = self.__edr_cmdr(cmdr_name, False)
if profile is None:
EDRLOG.log(u"Couldn't find a profile for {}.".format(cmdr_name),
"DEBUG")
return False
noted = profile.memo(memo)
if not noted:
EDRLOG.log(u"Couldn't write a note about {}".format(cmdr_name),
"DEBUG")
self.evict(cmdr_name)
return False
dex_dict = profile.dex_dict()
success = self.server.update_cmdrdex(profile.cid, dex_dict)
self.evict(cmdr_name)
return success
def clear_memo_cmdr(self, cmdr_name):
EDRLOG.log(u"Removing a note from {}".format(cmdr_name), "DEBUG")
profile = self.__edr_cmdr(cmdr_name, False)
if profile is None:
EDRLOG.log(u"Couldn't find a profile for {}.".format(cmdr_name),
"DEBUG")
return False
noted = profile.remove_memo()
if not noted:
EDRLOG.log(u"Couldn't remove a note from {}".format(cmdr_name),
"DEBUG")
self.evict(cmdr_name)
return False
dex_dict = profile.dex_dict()
success = self.server.update_cmdrdex(profile.cid, dex_dict)
self.evict(cmdr_name)
return success
def untag_cmdr(self, cmdr_name, tag):
if tag in ["enemy", "ally"]:
return self.__squadron_untag_cmdr(cmdr_name, tag)
else:
return self.__untag_cmdr(cmdr_name, tag)
def __untag_cmdr(self, cmdr_name, tag):
EDRLOG.log(u"Removing {} tag from {}".format(tag, cmdr_name), "DEBUG")
profile = self.__edr_cmdr(cmdr_name, False)
if profile is None:
EDRLOG.log(u"Couldn't find a profile for {}.".format(cmdr_name),
"DEBUG")
return False
untagged = profile.untag(tag)
if not untagged:
EDRLOG.log(
u"Couldn't untag {} (e.g. tag not present)".format(cmdr_name),
"DEBUG")
self.evict(cmdr_name)
return False
dex_dict = profile.dex_dict()
EDRLOG.log(u"New dex state: {}".format(dex_dict), "DEBUG")
success = self.server.update_cmdrdex(profile.cid, dex_dict)
self.evict(cmdr_name)
return success
def __squadron_untag_cmdr(self, cmdr_name, tag):
sqdr_id = self.__squadron_id()
if not sqdr_id:
EDRLOG.log(u"Can't untag: not a member of a squadron", "DEBUG")
return False
EDRLOG.log(u"Removing {} tag from {}".format(tag, cmdr_name), "DEBUG")
profile = self.__edr_cmdr(cmdr_name, False)
if profile is None:
EDRLOG.log(u"Couldn't find a profile for {}.".format(cmdr_name),
"DEBUG")
return False
untagged = profile.untag(tag)
if not untagged:
EDRLOG.log(
u"Couldn't untag {} (e.g. tag not present)".format(cmdr_name),
"DEBUG")
self.evict(cmdr_name)
return False
sqdrdex_dict = profile.sqdrdex_dict()
EDRLOG.log(u"New dex state: {}".format(sqdrdex_dict), "DEBUG")
augmented_sqdrdex_dict = sqdrdex_dict
augmented_sqdrdex_dict["level"] = self._player.squadron_info(
)["squadronLevel"]
augmented_sqdrdex_dict["by"] = self._player.name
success = self.server.update_sqdrdex(sqdr_id, profile.cid,
augmented_sqdrdex_dict)
self.evict(cmdr_name)
return success
class EDRBountyHuntingStats(object):
def __init__(self):
self.max = 0
self.previous_max = 0
self.min = 100.0
self.previous_min = 0
self.sum_scanned = 0
self.sum_awarded = 0
self.distribution = {"last_index": 0, "bins": [0]*25}
self.scanned_nb = 0
self.awarded_nb = 0
self.awarded_bounties = deque(maxlen=20)
self.scans = deque(maxlen=20)
self.efficiency = deque(maxlen=20)
self.max_efficiency = 5000000
self.max_normal_bounty = 350000 * 4 # appears to be the highest bounty per faction for NPC and the 4x boost from Nov 30th 2020
now = EDTime.py_epoch_now()
self.start = now
self.current = now
edr_config = EDRConfig()
self.scans_cache = LRUCache(edr_config.lru_max_size(), edr_config.blips_max_age())
self.last = {"timestamp": now, "bounty": None}
def reset(self):
self.max = 0
self.previous_max = 0
self.min = 100.0
self.previous_min = 0
self.sum_scanned = 0
self.sum_awarded = 0
self.distribution = {"last_index": 0, "bins": [0]*25}
self.scanned_nb = 0
self.awarded_nb = 0
self.awarded_bounties = deque(maxlen=20)
self.scans = deque(maxlen=20)
self.efficiency = deque(maxlen=20)
self.max_efficiency = 1000000
now = EDTime.py_epoch_now()
self.start = now
self.current = now
edr_config = EDRConfig()
self.scans_cache = LRUCache(edr_config.lru_max_size(), edr_config.blips_max_age())
self.last = {"timestamp": now, "bounty": None, "name": None, "distribution_index": 0}
def scanned(self, entry):
if entry.get("event", None) != "ShipTargeted":
return False
if entry.get("ScanStage", 0) < 3:
return False
raw_pilot_name = entry.get("PilotName", None)
if not raw_pilot_name:
return False
bounty = entry.get("Bounty", 0)
now = EDTime.py_epoch_now()
index = min(int(round(bounty/self.max_normal_bounty * (len(self.distribution["bins"])-1), 0)), len(self.distribution["bins"])-1)
self.last = {
"timestamp": now,
"bounty": bounty,
"name": entry.get("PilotName_Localised", ""),
"distribution_index": index
}
if self.__probably_previously_scanned(entry):
return False
self.scans_cache.set(raw_pilot_name, entry)
self.current = now
if bounty <= 0:
return False
self.scanned_nb += 1
self.__update_efficiency()
self.sum_scanned += bounty
self.previous_max = self.max
self.previous_min = self.min
self.max = max(self.max, bounty)
self.min = min(self.min, bounty)
index = min(int(round(bounty/self.max_normal_bounty * (len(self.distribution["bins"])-1), 0)), len(self.distribution["bins"])-1)
self.distribution["last_index"] = index
self.distribution["bins"][index] += 1
self.scans.append((now, bounty))
def __probably_previously_scanned(self, entry):
raw_pilot_name = entry.get("PilotName", None)
if not raw_pilot_name:
return False
last_scan = self.scans_cache.get(raw_pilot_name)
if not last_scan:
return False
return (entry["LegalStatus"] == last_scan["LegalStatus"]) and (entry.get("Bounty", 0) == last_scan.get("Bounty",0))
def awarded(self, entry):
if entry.get("event", None) != "Bounty":
return
now = EDTime.py_epoch_now()
self.current = now
total_rewards = 0
rewards = entry.get("Rewards", [])
for reward in rewards:
total_rewards += reward.get("Reward", 0)
self.sum_awarded += total_rewards
self.awarded_nb += 1
self.awarded_bounties.append((now, total_rewards))
self.__update_efficiency()
def credits_per_hour(self):
now = EDTime.py_epoch_now()
self.current = now
elapsed_time = self.current - self.start
if elapsed_time:
return self.sum_awarded / (elapsed_time / 3600.0)
return 0
def bounty_average(self):
return (self.sum_scanned / self.scanned_nb) if self.scanned_nb else 0.0
def reward_average(self):
return (self.sum_awarded / self.awarded_nb) if self.scanned_nb else 0.0
def __update_efficiency(self):
now = EDTime.py_epoch_now()
efficiency = self.credits_per_hour()
self.efficiency.append((now, efficiency))
self.max_efficiency = max(self.max_efficiency, efficiency)
def __repr__(self):
return str(self.__dict__)
class EDRLegalRecords(object):
EDR_LEGAL_RECORDS_CACHE = utils2to3.abspathmaker(__file__, 'cache', 'legal_records.v3.p')
def __init__(self, server):
self.server = server
self.timespan = None
self.records_check_interval = None
config = EDRConfig()
try:
with open(self.EDR_LEGAL_RECORDS_CACHE, 'rb') as handle:
self.records = pickle.load(handle)
except:
self.records = LRUCache(config.lru_max_size(), config.legal_records_max_age())
self.timespan = config.legal_records_recent_threshold()
self.records_check_interval = config.legal_records_check_interval()
def persist(self):
with open(self.EDR_LEGAL_RECORDS_CACHE, 'wb') as handle:
pickle.dump(self.records, handle, protocol=pickle.HIGHEST_PROTOCOL)
def summarize(self, cmdr_id):
if not cmdr_id:
EDRLOG.log(u"No cmdr_id, no records for {}".format(cmdr_id), "INFO")
return None
self.__update_records_if_stale(cmdr_id)
records = self.records.get(cmdr_id)["records"] if self.records.has_key(cmdr_id) else None
if not records:
EDRLOG.log(u"No legal records for {}".format(cmdr_id), "INFO")
return None
EDRLOG.log(u"Got legal records for {}".format(cmdr_id), "INFO")
overview = None
(clean, wanted, bounties, recent_stats) = self.__process(records)
timespan = EDTime.pretty_print_timespan(self.timespan, short=True, verbose=True)
maxB = u""
lastB = u""
if recent_stats["maxBounty"]:
max_bounty = EDFineOrBounty(recent_stats["maxBounty"]).pretty_print()
maxB = _(u", max={} cr").format(max_bounty)
if "last" in recent_stats and recent_stats["last"].get("value", None) and (recent_stats["last"].get("starSystem", "") not in ["", "unknown", "Unknown"]):
tminus = EDTime.t_minus(recent_stats["last"]["timestamp"], short=True)
last_bounty = EDFineOrBounty(recent_stats["last"]["value"]).pretty_print()
lastB = _(u", last: {} cr in {} {}").format(last_bounty, recent_stats["last"]["starSystem"], tminus)
# Translators: this is an overview of a cmdr's recent legal history for the 'last {}' days, number of clean and wanted scans, and optionally max and last bounties
overview = _(u"[Past {}] clean:{} / wanted:{}{}{}").format(timespan, recent_stats["clean"], recent_stats["wanted"], maxB, lastB)
return {"overview": overview, "clean": clean, "wanted": wanted, "bounties": bounties}
def __are_records_stale_for_cmdr(self, cmdr_id):
if self.records.get(cmdr_id) is None:
return True
last_updated = self.records.get(cmdr_id)["last_updated"]
now = datetime.datetime.now()
epoch_now = time.mktime(now.timetuple())
epoch_updated = time.mktime(last_updated.timetuple())
return (epoch_now - epoch_updated) > self.records_check_interval
def __update_records_if_stale(self, cmdr_id):
updated = False
if self.__are_records_stale_for_cmdr(cmdr_id):
now = datetime.datetime.now()
records = self.server.legal_stats(cmdr_id)
self.records.set(cmdr_id, {"last_updated": now, "records": records})
updated = True
return updated
def __process(self, legal_stats):
last = self.__emptyMonthlyBag()
clean = []
wanted = []
bounties = []
recent_stats = {"clean": 0, "wanted": 0, "maxBounty": 0, "last": {"value": 0, "timestamp": None, "starSystem": None}}
now_date = datetime.datetime.now()
currentYear = now_date.year
currentMonth = now_date.month
orderly = self.__orderlyMonthNo()
monthSpan = int(min(12, round(1 + (self.timespan / (60*60*24) - now_date.day)/30)))
for m in orderly:
if (m not in legal_stats):
clean.append(0)
wanted.append(0)
bounties.append(0)
continue
wayTooOld = int(legal_stats[m]["year"]) < currentYear-1
tooOld = (int(legal_stats[m]["year"]) == currentYear-1) and int(m) <= currentMonth
if (wayTooOld or tooOld):
clean.append(0)
wanted.append(0)
bounties.append(0)
continue
if (m in orderly[12-monthSpan:]):
recent_stats["clean"] += legal_stats[m]["clean"]
recent_stats["wanted"] += legal_stats[m]["wanted"]
if legal_stats[m]["max"]:
recent_stats["maxBounty"] = max(recent_stats["maxBounty"], legal_stats[m]["max"].get("value", 0))
if legal_stats[m]["last"] and legal_stats[m]["last"].get("value", 0) >= recent_stats["last"]["value"]:
recent_stats["last"] = legal_stats[m]["last"]
clean.append(legal_stats[m]["clean"])
wanted.append(legal_stats[m]["wanted"])
last[m] = legal_stats[m]["last"]
bounties.append(legal_stats[m]["max"]["value"])
return (clean, wanted, bounties, recent_stats)
@staticmethod
def __orderlyMonthNo():
currentMonthIDX0 = datetime.datetime.now().month-1
return [ str((((currentMonthIDX0 - i) %12) + 12)%12) for i in range(11,-1,-1)]
@staticmethod
def __emptyMonthlyBag():
return {
'0': None,
'1': None,
'2': None,
'3': None,
'4': None,
'5': None,
'6': None,
'7': None,
'8': None,
'9': None,
'10': None,
'11': None
}
class EDRFactions(object):
EDR_FACTIONS_CACHE = utils2to3.abspathmaker(__file__, 'cache',
'edr_factions.v1.p')
def __init__(self):
edr_config = EDRConfig()
try:
with open(self.EDR_FACTIONS_CACHE, 'rb') as handle:
self.factions_cache = pickle.load(handle)
except:
self.factions_cache = LRUCache(edr_config.lru_max_size(),
edr_config.factions_max_age())
def persist(self):
with open(self.EDR_FACTIONS_CACHE, 'wb') as handle:
pickle.dump(self.factions_cache,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
def process(self, factions, star_system):
factions_in_system = {}
for faction in factions:
factions_in_system[faction["Name"].lower()] = EDRFaction(faction)
self.factions_cache.set(star_system.lower(), factions_in_system)
def get(self, name, star_system):
factions_in_system = self.get_all(star_system)
if factions_in_system:
return factions_in_system.get(name.lower(), None)
return None
def get_all(self, star_system):
return self.factions_cache.get(star_system.lower())
def assess(self, star_system, security, population):
factions_in_system = self.get_all(star_system)
assessments = {}
for faction in factions_in_system:
assessments[faction] = factions_in_system[faction].assess(
security, population)
return assessments
def summarize_yields(self, star_system, security, population, inventory):
assessment = self.assess(star_system, security, population)
if not assessment:
return None
yields = {}
for faction_name in assessment:
if not assessment[faction_name]:
continue
faction = self.get(faction_name, star_system)
faction_chance = faction.influence
state_chance = 1.0 / len(
faction.active_states) if faction.active_states else 0.0
chance = faction_chance * state_chance
outcomes = assessment[faction_name].outcomes
for material in outcomes:
if yields.get(material, None) is None:
yields[material] = 0
yields[material] += chance
return [
u"{:.0f}%: {}".format(chance * 100.0,
inventory.oneliner(material.title()))
for (material, chance
) in sorted(yields.items(), key=lambda x: x[1], reverse=True)
]
def test_invalidate(self):
cache = LRUCache(2)
cache.set('foo', 10)
self.assertEqual(cache.get('foo'), 10)
self.assertEqual(cache.invalidate('foo'), 10)
self.assertEqual(cache.get('foo'), None)
def test_set_empty(self):
cache = LRUCache()
cache.set('foo', 10)
self.assertEqual(cache.get('foo'), 10)
# for reusing the same port without waiting for TIME_WAIT to expire
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((server_name, port_number))
# Will listen to only one connection at a time
s.listen(number_of_connections)
LRU = LRUCache(size_of_cache)
while True:
conn, addr = s.accept()
data = conn.recv(1024)
print "DEBUG: " + data
# convert the received data into JSON
json_data = json.loads(data)
"""
fetch the request method
GET denotes a GET cache request to check for hit/miss
POST denotes a SAVE request once a miss happens
"""
request_method = json_data["method"]
if request_method == "GET":
lru_response = LRU.get(json_data["username"])
conn.send(str(lru_response))
elif request_method == "POST":
LRU.set(json_data["username"], json_data["userdata"])
conn.close()