def __init__(self, fname, tag="", maxsize=10000): # max log size KB
self.fname = HomeDir + os.sep + "errorlogger" + os.sep + fname
self.tag = fname if tag == "" else tag
if _iprint >= 2:
print("ErrorLogger created at: " + self.fname + ", TAG " +
self.tag)
self.maxsize = maxsize
self.fd_list = []
tmp = glob.glob(self.fname)
for fn in tmp:
try:
sz = float(os.stat(fn).st_size) / 1024
if sz >= maxsize:
os.remove(fn)
except:
pass
tmpdir = os.path.dirname(self.fname)
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
fd = open(self.fname, "a")
self.fd_list.append(fd)
self.my_ip = get_my_ip()
self.create_time = (
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
i = 0
self.fd_list[i].close()
def __init__(self, tag=""):
self.lg_index = 0
self.my_ip = get_my_ip()
if not os.path.exists(folder):
os.makedirs(folder)
try:
self.my_tname = tag + ip2tarekc[self.my_ip]
except:
self.my_tname = tag + self.my_ip.split(".", 2)[-1]
if iprint >= 2: print(self.my_tname)
self.fd_list = []
self.fnames = []
self.freshness = 0 # latest log always named 0: log-tarekc-`0`-date-txt
tmp = glob.glob(folder + "log*") # log/log*
for fn in tmp:
try:
st = fn.replace(fnamePrefix + self.my_tname + "-",
"").split("-", 1)
ind = int(st[0]) + 1
newfn = fnamePrefix + self.my_tname + "-%d-" % ind + st[-1]
movefile(fn, newfn)
if iprint >= 2: print(fn, newfn)
except:
pass
fmain = fnamePrefix + self.my_tname + "-0-" + datetime.datetime.now(
).strftime("%Y-%m-%d-%H-%M-%S") + ".txt"
if iprint >= 1: print("create", fmain)
fd = open(fmain, "w")
self.fd_list.append(fd)
self.fnames.append(fmain)
self.lg(self.my_tname)
self.lg(self.my_ip)
self.lg(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
self.lg(time.time())
self.lg("\n")
def __init__(self, fname, maxsize=1000, overwrite=False):
self.fname = fname
self.maxsize = maxsize
self.fd_list = []
tmp = glob.glob(self.fname)
for fn in tmp:
try:
sz = float(os.stat(fn).st_size) / 1024
if sz >= maxsize:
os.remove(fn)
except:
pass
tmpdir = os.path.dirname(self.fname)
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
if not overwrite:
fd = open(self.fname, "a")
else:
fd = open(self.fname, "w")
self.fd_list.append(fd)
self.my_ip = get_my_ip()
self.lg("\n")
self.lg(self.my_ip)
self.lg(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
self.lg(time.time())
i = 0
self.fd_list[i].close()
self.history = {}
def __init__(self, **kwargs):
self.my_ip = get_my_ip()
self.my_home_dir = os.path.expanduser("~") #'/root'
self.whoami = getpass.getuser() #'root'
self.my_platform = get_platform() # 'mac','ubuntu','centos'
assert not " " in self.my_ip, "[ CM ] my_ip cannot multi interface!"
self.my_meta_dir = self.my_home_dir + CACHE_META_DIR
# run which func if redis return None? params follow the format: func_if_get_none(self.meta, key, *args , **kwargs) and return val (->key)
self.func_if_get_none = None
# shorten the prefix, like TinyURL.
self.overwrite_prefix = kwargs.get("overwrite_prefix", False)
# for Mem use. Use recent host ranking?
self.rt_servers = kwargs.get("rt_servers", False)
# overwrite redis cluster config? need to reload mem !
self.overwrite_redis_servers = kwargs.get("overwrite_redis_servers",
False)
if self.overwrite_redis_servers:
print('[ CM ] overwrite_redis_servers !!! Reload !')
def __init__(self):
self.lg_index = 0
self.my_ip = get_my_ip()
try:
self.my_tname = ip2tarekc[self.my_ip]
except:
self.my_tname = self.my_ip
self.fd_list = []
self.fnames = []
fmain = fnamePrefix + self.my_tname + ".txt"
fd = open(fmain, "w")
self.fd_list.append(fd)
self.fnames.append(fmain)
self.lg(self.my_tname)
self.lg(self.my_ip)
self.lg(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
self.lg(time.time())
self.lg("\n")
def __init__(self ):
self.lg_index=0
self.my_ip = get_my_ip()
try:
self.my_tname = ip2tarekc[self.my_ip]
except:
self.my_tname = self.my_ip
self.fd_list=[]
self.fnames =[]
fmain = fnamePrefix+self.my_tname+".txt"
fd = open(fmain,"w")
self.fd_list.append(fd)
self.fnames.append(fmain)
self.lg(self.my_tname)
self.lg(self.my_ip)
self.lg(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
self.lg(time.time())
self.lg("\n")
#!/usr/bin/env python
import os, sys
import subprocess
import random,time
import inspect
mypydir =os.path.abspath(os.path.dirname(inspect.getfile(inspect.currentframe())))
sys.path.append(mypydir+"/mytools")
from readconf import get_conf,get_conf_int
import collections,math
from random import randint
from namehostip import get_my_ip
MYIP = get_my_ip()
print("my IP= "+MYIP)
try:
SUMOPATH=str(os.environ.get("SUMO_HOME"))
if SUMOPATH.endswith('/'):
sys.path.append(os.path.join(SUMOPATH, "tools")) # */sumo/
else:
sys.path.append(os.path.join(SUMOPATH+'/', "tools")) # has to end with '/'
from sumolib import checkBinary
except ImportError:
sys.exit("$SUMO_HOME/tools not defined...")
random.seed(1)
iprint = 1
cfg = "conf.txt"
nf=get_conf(cfg,"node-files")
numDataSets = len(useDataSets)
ind2dataName = {
0: 'bci3d3a',
1: 'bci3d4a',
2: 'bci3d4c',
3: 'bci3d5',
4: 'bci4d2a',
}
dataName2ind = {}
for k, v in ind2dataName.items():
dataName2ind[v] = k
configfile = "conf.txt"
confs = "conf/conf-%s.txt"
My_IP = get_my_ip()
lg = Logger()
lg.lg_list([__file__])
from_conf = get_conf_int(configfile, "from_conf")
from_tasklist = get_conf_int(configfile, "from_tasklist")
use_cache = get_conf_int(configfile, "use_cache")
if use_cache:
try:
import memcache # https://www.sinacloud.com/doc/_static/memcache.html
except:
use_cache = 0
if use_cache:
CACHE_SIZE = 100 * 1024 * 1024 * 1024 # 10G
CACHE_TIME = 0
from geo import get_bearing_latlng2, min_angle_diff, convert_line_to_dic, get_dist_meters_latlng, latlng_to_city_state_country, get_osm_file_quote_given_file
from CacheManager import CacheManager
from osmutil import osmPipeline
from mem import AccessRestrictionContext, Synchronizer, Mem
from myosrm import crawl_nid_to_latlng
configfile = "conf.txt"
HomeDir = os.path.expanduser("~")
DirData = get_conf(configfile, "DirData") #~/greendrive/proc
DirOSM = get_conf(configfile, "DirOSM") #~/greendrive/osmdata
DirData = replace_user_home(DirData)
DirOSM = replace_user_home(DirOSM)
gpsfolder = "gps"
obdfolder = "obd"
combinefolder = "combine"
MyIp = get_my_ip()
My_Platform = get_platform() # "centos" means cluster
On_Cluster = False
if My_Platform == 'centos': On_Cluster = True
iprint = 2
err = ErrorLogger("allerror.txt", py_fname(__file__, False))
lg = SimpleAppendLogger("./logs/" + py_fname(__file__, False))
mm = CacheManager() # using redis on cluster
EXT = get_conf(configfile, "EXT") # .gz
CUT = get_conf(configfile, "CUT") # ~|
EQU = get_conf(configfile, "EQU", delimiter=":")
KeyUserEmail = get_conf(configfile, "KeyUserEmail")
KeyUserName = get_conf(configfile, "KeyUserName")
import inspect
mypydir = os.path.abspath(
os.path.dirname(inspect.getfile(inspect.currentframe())))
sys.path.append(mypydir)
from os.path import expanduser
from hostip import ip2tarekc, get_username
from readconf import get_conf
from namehostip import get_my_ip
iprint = 1
h2score = {}
h2mem = {}
h2cpu = {}
MYIP = get_my_ip()
def invoke_mem(cmd, host):
global h2mem
h2mem[host] = 0
try:
output = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE).communicate()[0]
h2mem[host] = int(output.split(":")[1].strip().split(' ')[0])
if iprint: print(host + " " + str(h2mem[host]))
except:
if iprint: print(host + " die")
pass
WORKDIR = os.path.expanduser("~") + "/eeg"
configfile = "conf.txt"
import memcache # https://www.sinacloud.com/doc/_static/memcache.html
CACHE_TIME = 0 # or unix 1497190272, if larger than 30 days.
CACHE_SIZE = 100 * 1024 * 1024 * 1024 # 100G, no use, use etc/config
use_cache = get_conf_int(configfile, "use_cache")
if not use_cache:
print(
__file__.split("/")[-1], "did not plan to use cache, exit, edit conf")
sys.exit(0)
cache_place = get_conf(configfile, "cache_place")
my_ip = get_my_ip()
print("my_ip", my_ip)
if cache_place == 'local':
mc = memcache.Client(['127.0.0.1:11211'],
server_max_value_length=CACHE_SIZE)
elif cache_place == 'servers':
servers = get_conf(configfile, "cache_servers").split(",")
if iprint: print("cache servers", servers)
if my_ip.startswith("172.22"):
ips = [tarekc2ip[host] + ":11211" for host in servers]
else:
ips = [host + ":11211" for host in servers]
mc = memcache.Client(ips, server_max_value_length=CACHE_SIZE)
import socket, SocketServer
from datetime import datetime, timedelta
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
from sumolib import checkBinary
import traci
sys.path.append(mypydir + Mytools_Rel_Dir) # install greenroute
from namehostip import get_my_ip
from logger import SimpleAppendLogger
from a4loadServer import load_osm_nodes, load_sumo_juntions, load_sumo_edges, load_sumo_connections, load_sumo_out_edges
l = SimpleAppendLogger("./logs/" +
__file__.split(os.sep)[-1].split(".", 1)[0] + ".log",
500) # KB
DateFormat = "%a %b %d %H:%M %Y"
l.lg_list(["server_run", datetime.now().strftime(DateFormat), get_my_ip()])
iprint = 1
class DataManager(object):
def __init__(self, *args):
self.osm_nodes = None # osm idstr to {lat,lng}
self.sumo_junctions = None # idstr to {'id','type','x','y','incLanes'}
self.sumo_edges = None # idstr to {'id','from','to','priority','type'}
self.sumo_connections = None # tup(e0,e1) to [{'from','to','via','dir','tl','linkIndex','fromLane','toLane'}, ]
self.sumo_out_edges = None # eid to {out-eid: ['via','dir',fromLane','toLane']}
self.OSM_File = mypydir + mapFolder + os.sep + config.osmfile + ".osm"
self.AddTlsNetfile = mypydir + mapFolder + os.sep + "%s.tls.net.xml" % config.osmfile
def load_data(self, ):
