def __init__(self, fname, tag="", maxsize=10000): # max log size KB self.fname = HomeDir + os.sep + "errorlogger" + os.sep + fname self.tag = fname if tag == "" else tag if _iprint >= 2: print("ErrorLogger created at: " + self.fname + ", TAG " + self.tag) self.maxsize = maxsize self.fd_list = [] tmp = glob.glob(self.fname) for fn in tmp: try: sz = float(os.stat(fn).st_size) / 1024 if sz >= maxsize: os.remove(fn) except: pass tmpdir = os.path.dirname(self.fname) if not os.path.exists(tmpdir): os.makedirs(tmpdir) fd = open(self.fname, "a") self.fd_list.append(fd) self.my_ip = get_my_ip() self.create_time = ( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) i = 0 self.fd_list[i].close()
def __init__(self, tag=""): self.lg_index = 0 self.my_ip = get_my_ip() if not os.path.exists(folder): os.makedirs(folder) try: self.my_tname = tag + ip2tarekc[self.my_ip] except: self.my_tname = tag + self.my_ip.split(".", 2)[-1] if iprint >= 2: print(self.my_tname) self.fd_list = [] self.fnames = [] self.freshness = 0 # latest log always named 0: log-tarekc-`0`-date-txt tmp = glob.glob(folder + "log*") # log/log* for fn in tmp: try: st = fn.replace(fnamePrefix + self.my_tname + "-", "").split("-", 1) ind = int(st[0]) + 1 newfn = fnamePrefix + self.my_tname + "-%d-" % ind + st[-1] movefile(fn, newfn) if iprint >= 2: print(fn, newfn) except: pass fmain = fnamePrefix + self.my_tname + "-0-" + datetime.datetime.now( ).strftime("%Y-%m-%d-%H-%M-%S") + ".txt" if iprint >= 1: print("create", fmain) fd = open(fmain, "w") self.fd_list.append(fd) self.fnames.append(fmain) self.lg(self.my_tname) self.lg(self.my_ip) self.lg(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) self.lg(time.time()) self.lg("\n")
def __init__(self, fname, maxsize=1000, overwrite=False): self.fname = fname self.maxsize = maxsize self.fd_list = [] tmp = glob.glob(self.fname) for fn in tmp: try: sz = float(os.stat(fn).st_size) / 1024 if sz >= maxsize: os.remove(fn) except: pass tmpdir = os.path.dirname(self.fname) if not os.path.exists(tmpdir): os.makedirs(tmpdir) if not overwrite: fd = open(self.fname, "a") else: fd = open(self.fname, "w") self.fd_list.append(fd) self.my_ip = get_my_ip() self.lg("\n") self.lg(self.my_ip) self.lg(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) self.lg(time.time()) i = 0 self.fd_list[i].close() self.history = {}
def __init__(self, **kwargs): self.my_ip = get_my_ip() self.my_home_dir = os.path.expanduser("~") #'/root' self.whoami = getpass.getuser() #'root' self.my_platform = get_platform() # 'mac','ubuntu','centos' assert not " " in self.my_ip, "[ CM ] my_ip cannot multi interface!" self.my_meta_dir = self.my_home_dir + CACHE_META_DIR # run which func if redis return None? params follow the format: func_if_get_none(self.meta, key, *args , **kwargs) and return val (->key) self.func_if_get_none = None # shorten the prefix, like TinyURL. self.overwrite_prefix = kwargs.get("overwrite_prefix", False) # for Mem use. Use recent host ranking? self.rt_servers = kwargs.get("rt_servers", False) # overwrite redis cluster config? need to reload mem ! self.overwrite_redis_servers = kwargs.get("overwrite_redis_servers", False) if self.overwrite_redis_servers: print('[ CM ] overwrite_redis_servers !!! Reload !')
def __init__(self): self.lg_index = 0 self.my_ip = get_my_ip() try: self.my_tname = ip2tarekc[self.my_ip] except: self.my_tname = self.my_ip self.fd_list = [] self.fnames = [] fmain = fnamePrefix + self.my_tname + ".txt" fd = open(fmain, "w") self.fd_list.append(fd) self.fnames.append(fmain) self.lg(self.my_tname) self.lg(self.my_ip) self.lg(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) self.lg(time.time()) self.lg("\n")
def __init__(self ): self.lg_index=0 self.my_ip = get_my_ip() try: self.my_tname = ip2tarekc[self.my_ip] except: self.my_tname = self.my_ip self.fd_list=[] self.fnames =[] fmain = fnamePrefix+self.my_tname+".txt" fd = open(fmain,"w") self.fd_list.append(fd) self.fnames.append(fmain) self.lg(self.my_tname) self.lg(self.my_ip) self.lg(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) self.lg(time.time()) self.lg("\n")
#!/usr/bin/env python import os, sys import subprocess import random,time import inspect mypydir =os.path.abspath(os.path.dirname(inspect.getfile(inspect.currentframe()))) sys.path.append(mypydir+"/mytools") from readconf import get_conf,get_conf_int import collections,math from random import randint from namehostip import get_my_ip MYIP = get_my_ip() print("my IP= "+MYIP) try: SUMOPATH=str(os.environ.get("SUMO_HOME")) if SUMOPATH.endswith('/'): sys.path.append(os.path.join(SUMOPATH, "tools")) # */sumo/ else: sys.path.append(os.path.join(SUMOPATH+'/', "tools")) # has to end with '/' from sumolib import checkBinary except ImportError: sys.exit("$SUMO_HOME/tools not defined...") random.seed(1) iprint = 1 cfg = "conf.txt" nf=get_conf(cfg,"node-files")
numDataSets = len(useDataSets) ind2dataName = { 0: 'bci3d3a', 1: 'bci3d4a', 2: 'bci3d4c', 3: 'bci3d5', 4: 'bci4d2a', } dataName2ind = {} for k, v in ind2dataName.items(): dataName2ind[v] = k configfile = "conf.txt" confs = "conf/conf-%s.txt" My_IP = get_my_ip() lg = Logger() lg.lg_list([__file__]) from_conf = get_conf_int(configfile, "from_conf") from_tasklist = get_conf_int(configfile, "from_tasklist") use_cache = get_conf_int(configfile, "use_cache") if use_cache: try: import memcache # https://www.sinacloud.com/doc/_static/memcache.html except: use_cache = 0 if use_cache: CACHE_SIZE = 100 * 1024 * 1024 * 1024 # 10G CACHE_TIME = 0
from geo import get_bearing_latlng2, min_angle_diff, convert_line_to_dic, get_dist_meters_latlng, latlng_to_city_state_country, get_osm_file_quote_given_file from CacheManager import CacheManager from osmutil import osmPipeline from mem import AccessRestrictionContext, Synchronizer, Mem from myosrm import crawl_nid_to_latlng configfile = "conf.txt" HomeDir = os.path.expanduser("~") DirData = get_conf(configfile, "DirData") #~/greendrive/proc DirOSM = get_conf(configfile, "DirOSM") #~/greendrive/osmdata DirData = replace_user_home(DirData) DirOSM = replace_user_home(DirOSM) gpsfolder = "gps" obdfolder = "obd" combinefolder = "combine" MyIp = get_my_ip() My_Platform = get_platform() # "centos" means cluster On_Cluster = False if My_Platform == 'centos': On_Cluster = True iprint = 2 err = ErrorLogger("allerror.txt", py_fname(__file__, False)) lg = SimpleAppendLogger("./logs/" + py_fname(__file__, False)) mm = CacheManager() # using redis on cluster EXT = get_conf(configfile, "EXT") # .gz CUT = get_conf(configfile, "CUT") # ~| EQU = get_conf(configfile, "EQU", delimiter=":") KeyUserEmail = get_conf(configfile, "KeyUserEmail") KeyUserName = get_conf(configfile, "KeyUserName")
import inspect mypydir = os.path.abspath( os.path.dirname(inspect.getfile(inspect.currentframe()))) sys.path.append(mypydir) from os.path import expanduser from hostip import ip2tarekc, get_username from readconf import get_conf from namehostip import get_my_ip iprint = 1 h2score = {} h2mem = {} h2cpu = {} MYIP = get_my_ip() def invoke_mem(cmd, host): global h2mem h2mem[host] = 0 try: output = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE).communicate()[0] h2mem[host] = int(output.split(":")[1].strip().split(' ')[0]) if iprint: print(host + " " + str(h2mem[host])) except: if iprint: print(host + " die") pass
WORKDIR = os.path.expanduser("~") + "/eeg" configfile = "conf.txt" import memcache # https://www.sinacloud.com/doc/_static/memcache.html CACHE_TIME = 0 # or unix 1497190272, if larger than 30 days. CACHE_SIZE = 100 * 1024 * 1024 * 1024 # 100G, no use, use etc/config use_cache = get_conf_int(configfile, "use_cache") if not use_cache: print( __file__.split("/")[-1], "did not plan to use cache, exit, edit conf") sys.exit(0) cache_place = get_conf(configfile, "cache_place") my_ip = get_my_ip() print("my_ip", my_ip) if cache_place == 'local': mc = memcache.Client(['127.0.0.1:11211'], server_max_value_length=CACHE_SIZE) elif cache_place == 'servers': servers = get_conf(configfile, "cache_servers").split(",") if iprint: print("cache servers", servers) if my_ip.startswith("172.22"): ips = [tarekc2ip[host] + ":11211" for host in servers] else: ips = [host + ":11211" for host in servers] mc = memcache.Client(ips, server_max_value_length=CACHE_SIZE)
import socket, SocketServer from datetime import datetime, timedelta from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer from sumolib import checkBinary import traci sys.path.append(mypydir + Mytools_Rel_Dir) # install greenroute from namehostip import get_my_ip from logger import SimpleAppendLogger from a4loadServer import load_osm_nodes, load_sumo_juntions, load_sumo_edges, load_sumo_connections, load_sumo_out_edges l = SimpleAppendLogger("./logs/" + __file__.split(os.sep)[-1].split(".", 1)[0] + ".log", 500) # KB DateFormat = "%a %b %d %H:%M %Y" l.lg_list(["server_run", datetime.now().strftime(DateFormat), get_my_ip()]) iprint = 1 class DataManager(object): def __init__(self, *args): self.osm_nodes = None # osm idstr to {lat,lng} self.sumo_junctions = None # idstr to {'id','type','x','y','incLanes'} self.sumo_edges = None # idstr to {'id','from','to','priority','type'} self.sumo_connections = None # tup(e0,e1) to [{'from','to','via','dir','tl','linkIndex','fromLane','toLane'}, ] self.sumo_out_edges = None # eid to {out-eid: ['via','dir',fromLane','toLane']} self.OSM_File = mypydir + mapFolder + os.sep + config.osmfile + ".osm" self.AddTlsNetfile = mypydir + mapFolder + os.sep + "%s.tls.net.xml" % config.osmfile def load_data(self, ):