def fix_no_speed(addr, mm_loaded=False): print("fix_no_speed():", addr, 'mm_loaded', mm_loaded) bugNoSpeedFn = mypydir + "/cache/nospeed-%s.txt" % addr if On_Cluster: mm_tmp_nid2spd = Mem({ "num": 30, "prefix": "~fx_n2spd~", "expire": 86400 * 30 }) else: mm_tmp_nid2spd = Mem({ "use_ips": ['localhost'], "prefix": "~fx_n2spd~", "expire": 86400 * 30 }) aleady_mm_loaded = mm_loaded correctSpdFn = DirOSM + os.sep + addr + "/%s-nids-to-speed.txt" % addr if not aleady_mm_loaded: cnt = 0 pthresh = 1 print('Loading ' + correctSpdFn) with open(correctSpdFn, "r") as f: for l in f: st = l.split(",") if len(st) < 3: continue mm_tmp_nid2spd.set((int(st[0]), int(st[1])), float(st[2])) cnt += 1 if cnt >= pthresh: print('fix_no_speed load cnt', cnt) pthresh *= 2 print("correct spd tup len=", cnt) mm_nid2latlng = CacheManager( overwrite_prefix=True) # will use existing config mm_nid2neighbor = CacheManager(overwrite_prefix=True) mm_nid2latlng.use_cache( meta_file_name="osm/cache-%s-nodeid-to-lat-lng.txt" % addr) mm_nid2neighbor.use_cache( meta_file_name="osm/cache-%s-nodeid-to-neighbor-nid.txt" % addr) n1n2s = {} print('Reading ' + bugNoSpeedFn) with open(bugNoSpeedFn, "r") as f: for l in f: st = l.split(",") if len(st) < 2: continue st = [int(x) for x in st] tup = (st[0], st[1]) if tup not in n1n2s: n1n2s[tup] = -1 print("no_speed nid tup len=", len(n1n2s)) lastBugCnt = None while True: bugcnt = 0 for tup in n1n2s.keys(): if n1n2s[tup] > 0: continue n1 = tup[0] n2 = tup[1] hd1 = get_bear_given_nid12(n1, n2, mm_nid2latlng) if hd1 is None: continue nblist = mm_nid2neighbor.get(n1) # ?->n1->n2 mindiff = 1e10 fixed = 0 if nblist: for nbn in nblist: hdn = get_bear_given_nid12(nbn, n1, mm_nid2latlng) if hdn is None: continue angle = min_angle_diff(hd1, hdn) if (nbn, n1) in n1n2s and n1n2s[(nbn, n1)] > 0: spdn = n1n2s[(nbn, n1)] else: spdn = mm_tmp_nid2spd.get((nbn, n1)) if angle < mindiff and spdn is not None: mindiff = angle n1n2s[tup] = spdn fixed = 1 if fixed: continue nblist = mm_nid2neighbor.get(n2) # n1->n2->? mindiff = 1e10 if nblist: for nbn in nblist: hdn = get_bear_given_nid12(n2, nbn, mm_nid2latlng) if hdn is None: continue angle = min_angle_diff(hd1, hdn) if (n2, nbn) in n1n2s and n1n2s[(n2, nbn)] > 0: spdn = n1n2s[(n2, nbn)] else: spdn = mm_tmp_nid2spd.get((n2, nbn)) if angle < mindiff and spdn is not None: mindiff = angle n1n2s[tup] = spdn fixed = 1 if fixed == 0: bugcnt += 1 if bugcnt == 0: break print("bugcnt", bugcnt) if lastBugCnt is not None: if lastBugCnt == bugcnt: print("Give up #", bugcnt) break lastBugCnt = bugcnt with open(correctSpdFn, "a") as f: for tup in n1n2s.keys(): if n1n2s[tup] < 0: continue print("%d,%d,%.2f" % (tup[0], tup[1], n1n2s[tup])) f.write("%d,%d,%.2f\n" % (tup[0], tup[1], n1n2s[tup])) print("Give up #", bugcnt)
class CacheManager: def __init__(self, **kwargs): self.my_ip = get_my_ip() self.my_home_dir = os.path.expanduser("~") #'/root' self.whoami = getpass.getuser() #'root' self.my_platform = get_platform() # 'mac','ubuntu','centos' assert not " " in self.my_ip, "[ CM ] my_ip cannot multi interface!" self.my_meta_dir = self.my_home_dir + CACHE_META_DIR # run which func if redis return None? params follow the format: func_if_get_none(self.meta, key, *args , **kwargs) and return val (->key) self.func_if_get_none = None # shorten the prefix, like TinyURL. self.overwrite_prefix = kwargs.get("overwrite_prefix", False) # for Mem use. Use recent host ranking? self.rt_servers = kwargs.get("rt_servers", False) # overwrite redis cluster config? need to reload mem ! self.overwrite_redis_servers = kwargs.get("overwrite_redis_servers", False) if self.overwrite_redis_servers: print('[ CM ] overwrite_redis_servers !!! Reload !') def set_meta_file_name(self, fname): """Check and set self.meta_file_name, self.meta_file_abs_path. :param fname: = self.meta_file_name. str """ assert fname != "", "Please provide meta_file_name (cannot be empty)!" assert not fname.endswith( os.sep), "set_meta_file_name cannot endswith(os.sep)!" assert not ".." in fname, "meta_file_name cannot have '../' !" self.meta_file_name = fname self.meta_file_abs_path = self.my_meta_dir + os.sep + fname def exist_cache(self, meta_file_name): self.set_meta_file_name(meta_file_name) return os.path.exists(self.meta_file_abs_path) def create_cache(self, **kwargs): """Create cache meta and file. :param meta_file_name: relative path to meta file. str :param overwrite_meta_file: overwrite meta_file? bool :param overwrite_cache_file: overwrite cache file? bool :param overwrite_memory: overwrite contents in cache memory? bool :param gen_cache_file_cmd: linux command for generating cache file on host. str :param gen_host_ip: the host ip who generated cache file. str :param gen_username: the user who generated cache file. str :param cache_file_abs_path: where to store generated cache file? str :param store_allKeys: if store all keys as list in 'allKeys' :param engine: redis or TODO. str :param params: a dict, params for redis. dict :param yield_func: if not cache-file, yield on the fly, give yield-in-file as well :param yield_args: yield-in-file CUT/+ other args """ self.set_meta_file_name(kwargs.get("meta_file_name", "")) overwrite_meta_file = kwargs.get("overwrite_meta_file", False) overwrite_cache_file = kwargs.get("overwrite_cache_file", False) if not self.overwrite_prefix: self.overwrite_prefix = kwargs.get("overwrite_prefix", False) if os.path.exists(self.meta_file_abs_path) and not overwrite_meta_file: raise Exception( "Already exists locally: " + self.meta_file_abs_path + ", overwrite_meta_file=True,? or use_cache() instead of create_cache()" ) gen_cache_file_cmd = kwargs.get("gen_cache_file_cmd", "") assert gen_cache_file_cmd != "", "gen_cache_file_cmd cannot be empty!" gen_host_ip = kwargs.get("gen_host_ip", self.my_ip) if gen_host_ip == "127.0.0.1" or gen_host_ip == "localhost": gen_host_ip = self.my_ip gen_username = kwargs.get("gen_username", self.whoami) self.cache_file_abs_path = kwargs.get("cache_file_abs_path", "") self.store_allKeys = kwargs.get("store_allKeys", False) assert self.cache_file_abs_path != "", "cache_file_abs_path cannot be empty!" engine = kwargs.get("engine", "redis") params = kwargs.get("params", {}) params["prefix"] = self.meta_file_name if self.overwrite_prefix: params[ "overwrite_prefix"] = True # ask Mem() to translate pref mapping params["engine"] = engine params["overwrite_memory"] = kwargs.get("overwrite_memory", False) yield_func = kwargs.get("yield_func", None) # gen k,v on the fly, avoid cache file yield_args = kwargs.get("yield_args", None) # inputs to yield func kv_action_type = kwargs.get( "kv_action_type", None) # yield func set key-value append/direct set loading_msg = kwargs.get("loading_msg", "") meta = {} meta["meta_file_name"] = self.meta_file_name meta["meta_file_abs_path"] = self.meta_file_abs_path meta["gen_cache_file_cmd"] = gen_cache_file_cmd meta["gen_host_ip"] = gen_host_ip meta["gen_username"] = gen_username meta["cache_file_abs_path"] = self.cache_file_abs_path meta["engine"] = engine meta["params"] = pickle.dumps(params) meta["create_time"] = unix2datetime(time.time()) meta["yield_func"] = dill.dumps(yield_func) # it is a function in 3.py meta["yield_args"] = yield_args meta["kv_action_type"] = kv_action_type if self.my_platform != "centos" and not overwrite_meta_file: # check if already exists on tarekc: r = requests.get(url=FILE_SERVER_GET + "cache_meta", params={"meta_file_name": self.meta_file_name}) data = r.json() if not "not-found" in data['status']: raise Exception( "Already exists on Centos: " + self.meta_file_name + ", overwrite_meta_file=True,? or use_cache() instead of create_cache()" ) # auto set content type to application/json: r = requests.post(FILE_SERVER_POST + "cache_meta", json=meta) # TODO not using json if r.status_code != 200: raise Exception("create_cache post fail status_code: %d" % r.status_code) if gen_host_ip == self.my_ip or self.my_platform == "centos": if yield_func is None or yield_args is None: if overwrite_cache_file: gen_cache_file_cmd += " overwrite=true" if iprint: print("RUN " + gen_cache_file_cmd) subprocess.call(gen_cache_file_cmd.split(" ")) tmpdir = os.sep.join(self.meta_file_abs_path.split(os.sep)[0:-1]) if not os.path.exists(tmpdir): os.makedirs(tmpdir) if True: pickle.dump(meta, open(self.meta_file_abs_path, 'wb')) else: # TODO, no longer using json with open(self.meta_file_abs_path, 'wb') as f: json.dump(meta, codecs.getwriter('utf-8')(f), ensure_ascii=False) if iprint: print("create_cache(): Saved meta_file " + self.meta_file_abs_path) else: if iprint: print("Gen cmd not on local host: " + gen_cache_file_cmd) meta["params"] = pickle.loads(str(meta["params"])) if self.rt_servers: meta["params"]["rt_servers"] = True if self.overwrite_redis_servers: meta["params"]["overwrite_servers"] = True ret = self.activate_mem(meta["params"]) if ret == "invalid" or params["overwrite_memory"]: if yield_func is None or yield_args is None: if iprint: print("RUN load_cache_file_into_mem() ...") self.load_cache_file_into_mem(meta["cache_file_abs_path"], meta["params"], self.store_allKeys, msg=loading_msg) else: if iprint: print("RUN yield_func() " + yield_args) self.yield_file_into_mem(yield_func, yield_args, meta["params"], kv_action_type=kv_action_type, msg=loading_msg) print(self.activate_mem(meta["params"])) self.meta = meta def use_cache(self, **kwargs): """Use mem according to cache meta. :param meta_file_name: relative path to meta file. str :param overwrite_meta_by_centos: overwrite meta_file by that on centos? bool :param store_allKeys: upon re-do, if store all keys as list in 'allKeys' - overwrite_memory: force reload into memory. - ignore_invalid_mem: non-block call, ignore not in cache. - set_to_be_valid: overwrite flag=1 to regard as valid. - ignore_lock: no access lock on load mm func. - overwrite_prefix: use tiny short prefix. - check_pref_is_short: check if overwrite_prefix is SET. """ self.set_meta_file_name(kwargs.get("meta_file_name", "")) overwrite_meta_by_centos = kwargs.get("overwrite_meta_by_centos", False) self.store_allKeys = kwargs.get("store_allKeys", False) self.ignore_invalid_mem = kwargs.get("ignore_invalid_mem", False) overwrite_memory = kwargs.get("overwrite_memory", False) set_to_be_valid = kwargs.get("set_to_be_valid", False) ignore_lock = kwargs.get("ignore_lock", False) check_pref_is_short = kwargs.get("check_pref_is_short", True) if not self.overwrite_prefix: self.overwrite_prefix = kwargs.get("overwrite_prefix", False) loading_msg = kwargs.get("loading_msg", "") params = kwargs.get("params", {}) # if you want to change params if "params" in kwargs: print("[CM] params update these:", params) if not os.path.exists(self.meta_file_abs_path): if self.my_platform != "centos": r = requests.get( url=FILE_SERVER_GET + "cache_meta", params={"meta_file_name": self.meta_file_name}) meta = r.json() # TODO not using json, changed syncdir/file-server/app.py: get_file() if "not-found" in meta['status']: raise Exception("Not exists? " + self.meta_file_name) meta.pop('status') tmpdir = os.sep.join( self.meta_file_abs_path.split(os.sep)[0:-1]) if not os.path.exists(tmpdir): os.makedirs(tmpdir) with open(self.meta_file_abs_path, 'wb') as f: # TODO not using json json.dump(meta, codecs.getwriter('utf-8')(f), ensure_ascii=False) if iprint: print("Copied from centos: " + self.meta_file_abs_path) else: raise Exception("Not exists? " + self.meta_file_name) else: # already exists locally: if self.my_platform != "centos" and overwrite_meta_by_centos: r = requests.get( url=FILE_SERVER_GET + "cache_meta", params={"meta_file_name": self.meta_file_name}) meta = r.json() if not "not-found" in meta['status']: meta.pop('status') with open(self.meta_file_abs_path, 'wb') as f: # TODO not using json json.dump(meta, codecs.getwriter('utf-8')(f), ensure_ascii=False) if iprint: print("Copied from centos: " + self.meta_file_abs_path) try: meta = pickle.load(open(self.meta_file_abs_path, 'rb')) except: # TODO not using json meta = json.load(open(self.meta_file_abs_path, 'rb')) meta["params"] = pickle.loads(str(meta["params"])) meta["params"].update(params) if self.overwrite_prefix: meta["params"]['overwrite_prefix'] = True meta["params"]["overwrite_memory"] = overwrite_memory meta["params"]["ignore_lock"] = ignore_lock if self.rt_servers: meta["params"]["rt_servers"] = True if self.overwrite_redis_servers: meta["params"]["overwrite_servers"] = True if set_to_be_valid: print("force set to be valid !") self.set_mem_valid(meta["params"]) return ret = self.activate_mem(meta["params"]) if (ret == "invalid" and not self.ignore_invalid_mem) or overwrite_memory: yield_func = dill.loads( meta["yield_func"]) if "yield_func" in meta else None yield_args = meta["yield_args"] if "yield_args" in meta else None if yield_func is None or yield_args is None: if iprint: print("Redo load_cache_file_into_mem() ...") self.load_cache_file_into_mem(meta["cache_file_abs_path"], meta["params"], self.store_allKeys, msg=loading_msg) else: if iprint: print("RUN yield_func() " + yield_args) kv_action_type = meta[ "kv_action_type"] if "kv_action_type" in meta else None self.yield_file_into_mem(yield_func, yield_args, meta["params"], kv_action_type=kv_action_type, msg=loading_msg) print(self.activate_mem(meta["params"])) self.meta = meta if check_pref_is_short: if len(self.mm.prefix) > 4: # support more prefixes if > more. print(self.mm.prefix) print(" :prefix too long, not overwrite_prefix?") sys.exit(0) # You Edit def activate_mem(self, params): if params["engine"] == "redis": if iprint >= 3: print("Redis params", params) self.mm = Mem(params) tmp = self.mm.get(Valid_Flag_Key) tmp2 = self.mm.get(End_Flag_Key) if (tmp is None or tmp == 0) or (tmp2 is None or tmp2 == 0): if iprint >= 2: print("[ CM ] Invalid activation") return "invalid" return 'ok' return "unknown-engine" def set_mem_valid(self, params): # force set to be valid if params["engine"] == "redis": self.mm = Mem(params) self.mm.set(Valid_Flag_Key, 1) self.mm.set(End_Flag_Key, 1) def load_cache_file_into_mem(self, cache_file_abs_path, params, store_allKeys=False, msg=''): '''cache_file assumed to be a dict of k:list, with header, dtype, keyPos info. ''' if self.my_platform == "mac": single_server = True # not on cluster else: single_server = False assert params["engine"] == "redis" Force_Redo = ("overwrite_memory" in params and params["overwrite_memory"]) with AccessRestrictionContext(prefix=cache_file_abs_path, no_restriction=single_server) as lock: lock.Access_Or_Wait_And_Skip("load_cache_file_into_mem") if iprint: print("load_cache_file_into_mem params", params) self.mm = Mem(params) tmp = self.mm.get(Valid_Flag_Key) tmp2 = self.mm.get(End_Flag_Key) if tmp == 1 and tmp2 == 1 and not Force_Redo: if iprint: print("Redis already cached, skip...") return print("pickle.load %s ..." % cache_file_abs_path) da = pickle.load(open(cache_file_abs_path, "rb")) if iprint >= 2: try: print(" - Info - ", da["header"], da["dtype"], da["keyPos"]) except: print(" - missing keyPos or header or dtype -") if not Force_Redo: self.mm.set(End_Flag_Key, 0) self.mm.set(Valid_Flag_Key, 1) time.sleep(0.1) # redis checks expired at freq 10hz. allKeys = [] if "header" in da: header = da.pop("header") self.mm.set("header", header) if "dtype" in da: dtype = da.pop("dtype") self.mm.set("dtype", dtype) if "keyPos" in da: keyPos = da.pop("keyPos") self.mm.set("keyPos", keyPos) cnt, thresh = 0, 1024 * 8 for k, v in da.items(): self.mm.set(k, v) cnt += 1 if store_allKeys: allKeys.append(k) if iprint and cnt >= thresh: thresh *= 2 print("mm key cnt %d. %s" % (cnt, msg)) if store_allKeys: self.mm.set("allKeys", allKeys) self.mm.set(End_Flag_Key, 1) if iprint: print("loaded cache key num: %d" % cnt) def yield_file_into_mem(self, yield_func, yield_args, params, kv_action_type=None, msg=''): if self.my_platform == "mac": single_server = True # not on cluster else: single_server = False assert params["engine"] == "redis" Force_Redo = ("overwrite_memory" in params and params["overwrite_memory"]) ignore_lock = ("ignore_lock" in params and params["ignore_lock"]) with AccessRestrictionContext( prefix=yield_args + self.meta_file_name, no_restriction=(ignore_lock or single_server)) as lock: lock.Access_Or_Wait_And_Skip("yield_file_into_mem") self.mm = Mem(params) if iprint: print("yield_file_into_mem params", params) print('mm.prefix', self.mm.prefix) tmp = self.mm.get(Valid_Flag_Key) tmp2 = self.mm.get(End_Flag_Key) if tmp == 1 and tmp2 == 1 and not Force_Redo: if iprint: print("Redis already cached, skip...") return if not Force_Redo: self.mm.set(End_Flag_Key, 0) self.mm.set(Valid_Flag_Key, 1) time.sleep(0.1) cnt, thresh = 0, 1024 * 8 for k, v in yield_func(yield_args): if kv_action_type is None: self.mm.set(k, v) elif kv_action_type == 1: # append to vlist tmp = self.mm.get(k) if tmp is None: tmp = [] if v not in tmp: tmp.append(v) self.mm.set(k, tmp) cnt += 1 if iprint and cnt >= thresh: thresh *= 2 print("mm key cnt %d. %s" % (cnt, msg)) self.mm.set(End_Flag_Key, 1) if iprint: print("yielded key num: %d" % cnt) ''' ---- mm.get: what to do if redis returns null? perhaps low memory and you reload ?''' def get(self, key, *args, **kwargs): val = self.mm.get(key, *args, **kwargs) if val is None: if self.func_if_get_none is not None: val = self.func_if_get_none(self.meta, key, *args, **kwargs) if val is not None: self.mm.set(key, val, *args, **kwargs) return val def set(self, key, val, *args, **kwargs): # self.set = self.mm.set set only allowed in load_cache_file_into_mem ? _val = self.mm.get(key, *args, **kwargs) if _val is None: print("CacheManager SET !!", key, val) self.mm.set(key, val, *args, **kwargs) return True return False def get_id(self, ): try: return self.meta_file_name except AttributeError: return None def invalidate_mem( self): # Mark as invalid. force reload/re-yield k,v into mem try: meta = pickle.load(open(self.meta_file_abs_path, 'rb')) except: # TODO not using json meta = json.load(open(self.meta_file_abs_path, 'rb')) params = pickle.loads(str(meta["params"])) assert params["engine"] == "redis" self.mm = Mem(params) self.mm.set(End_Flag_Key, 0) self.mm.set(Valid_Flag_Key, 0)
mypydir = os.path.abspath( os.path.dirname(inspect.getfile(inspect.currentframe()))) addpath = mypydir + Mytools_Rel_Dir from mem import Mem config = None traci = None # Redis cache on port 6380 mm_sumo = Mem({ "prefix": "s!", "expire": 864000, "use_ips": [region2serverIP[R['addr']]['host']] }) mm_sumo.set("test", 1) _mm_is_good = 0 if mm_sumo.get('test') == 1: print("Mem good on " + region2serverIP[R['addr']]['host']) _mm_is_good = 1 else: raise Exception("Mem fail ... " + region2serverIP[R['addr']]['host']) def construct_edge(eid, isInternal, turn="", allViaLanes=None, tlsid=None, internalNode=None): if not isInternal: