def run (rex, main, others):
util.reading(main.name,logger)
mainH,mainP = tempfile.mkstemp(dir=os.path.dirname(main.name))
othersIO = list()
for o in others:
util.reading(o.name,logger)
h,p = tempfile.mkstemp(dir=os.path.dirname(o.name))
othersIO.append((o,h,p))
# read the files in parallel
dropped = 0
lines = 0
for line in main:
lines += 1
prefix = line.split('\t')[0]
keep = rex.search(line) is None
if keep:
os.write(mainH,line)
else:
dropped += 1
for i,o,_p in othersIO:
line1 = i.readline()
prefix1 = line1.split('\t')[0].rstrip()
if prefix1 != prefix:
raise Exception("prefix mismatch",prefix,prefix,i.name)
if keep:
os.write(o,line1)
for i in others:
line1 = i.readline()
if line1 != '':
raise Exception('uneven files',line1,i.name)
logger.info("Dropped {:,d} lines out of {:,d}".format(dropped,lines))
# close streams and rename
finish(main,mainH,mainP)
for i,o,p in othersIO:
finish(i,o,p)
def random_stats(actual, repeat=1000, delimiter='\t', abeg=1):
"Score a file against shuffled self."
util.reading(actual, MuLabCat.logger)
nca = runstat.NumStat("cat/ex", integer=True)
empty = frozenset([''])
data = list()
with open(actual) as af:
for sa in csv.reader(af, delimiter=delimiter):
sa = frozenset(sa[abeg:]) - empty
data.append(sa)
nca.add(len(sa))
MuLabCat.logger.info(
"random_stats({r:,d}): Read {l:,d} lines: {o:s}".format(
r=repeat, l=len(data), o=nca.__str__("{0:.2f}".format)))
prre = runstat.NumStat("prec/recall")
prof = runstat.NumStat("proficiency")
for _ in range(repeat):
indexes = range(len(data))
random.shuffle(indexes)
mlc = MuLabCat('shuffle+' + actual, reassign=False)
for i in range(len(data)):
mlc.add(data[i], data[indexes[i]])
# MuLabCat.logger.info("%s",mlc)
f1, p, r = mlc.f1score()
assert f1 == p and f1 == r
prre.add(p)
prof.add(mlc.proficiency_raw())
MuLabCat.logger.info("%d runs:\n %s\n %s", repeat,
prof.__str__("{0:.2%}".format),
prre.__str__("{0:.2%}".format))
def __init__(self, model=None, vw=None, parser=None, logger=None):
if vw is None:
self.vw = parser.vw_exe = file2path(parser.vw_exe,
"VowpalWabbit.exe")
else:
self.vw = file2path(vw, "VowpalWabbit.exe")
if model is None:
self.model = parser.vw_model = file2path(parser.vw_model,
"VowpalWabbit.model")
else:
self.model = file2path(model, "VowpalWabbit.model")
self.logger = VowpalWabbitPipe.logger if logger is None else logger
cmd = [
self.vw, "--initial_regressor", self.model, "--raw_predictions",
"/dev/stdout", "--testonly"
]
util.reading(self.model, self.logger)
self.logger.info("Running %s", cmd)
self.s = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=VowpalWabbitPipe.devnull)
self.logger.info("Started PID=%d", self.s.pid)
self.model = model
self.called = 0
self.pending = 0
def random_stats (actual, repeat=1000, delimiter='\t', abeg=1):
"Score a file against shuffled self."
util.reading(actual,MuLabCat.logger)
nca = runstat.NumStat("cat/ex",integer=True)
empty = frozenset([''])
data = list()
with open(actual) as af:
for sa in csv.reader(af,delimiter=delimiter):
sa = frozenset(sa[abeg:])-empty
data.append(sa)
nca.add(len(sa))
MuLabCat.logger.info("random_stats({r:,d}): Read {l:,d} lines: {o:s}".format(r=repeat,l=len(data),o=nca.__str__("{0:.2f}".format)))
prre = runstat.NumStat("prec/recall")
prof = runstat.NumStat("proficiency")
for _ in range(repeat):
indexes = range(len(data))
random.shuffle(indexes)
mlc = MuLabCat('shuffle+'+actual,reassign=False)
for i in range(len(data)):
mlc.add(data[i],data[indexes[i]])
# MuLabCat.logger.info("%s",mlc)
f1,p,r = mlc.f1score()
assert f1 == p and f1 == r
prre.add(p)
prof.add(mlc.proficiency_raw())
MuLabCat.logger.info("%d runs:\n %s\n %s",repeat,
prof.__str__("{0:.2%}".format),
prre.__str__("{0:.2%}".format))
def run(rex, main, others):
util.reading(main.name, logger)
mainH, mainP = tempfile.mkstemp(dir=os.path.dirname(main.name))
othersIO = list()
for o in others:
util.reading(o.name, logger)
h, p = tempfile.mkstemp(dir=os.path.dirname(o.name))
othersIO.append((o, h, p))
# read the files in parallel
dropped = 0
lines = 0
for line in main:
lines += 1
prefix = line.split('\t')[0]
keep = rex.search(line) is None
if keep:
os.write(mainH, line)
else:
dropped += 1
for i, o, _p in othersIO:
line1 = i.readline()
prefix1 = line1.split('\t')[0].rstrip()
if prefix1 != prefix:
raise Exception("prefix mismatch", prefix, prefix, i.name)
if keep:
os.write(o, line1)
for i in others:
line1 = i.readline()
if line1 != '':
raise Exception('uneven files', line1, i.name)
logger.info("Dropped {:,d} lines out of {:,d}".format(dropped, lines))
# close streams and rename
finish(main, mainH, mainP)
for i, o, p in othersIO:
finish(i, o, p)
def score_vw_oaa (predictions, base=None, NumRP=0):
"""Score $(vw -t) output from a $(vw --oaa) model.
IDs should be the true values, unless base is supplied."""
cmx = ConfusionMX(os.path.basename(predictions),NumRP=NumRP)
if base is not None:
base = ord(base[0])-1
util.reading(predictions,ConfusionMX.logger)
with open(predictions) as ins:
for line in ins:
vals = line.split()
cmx.add(int(vals[1]) if base is None else ord(vals[1][0])-base,
int(float(vals[0])))
return cmx
def score_vw_oaa(predictions, base=None, NumRP=0):
"""Score $(vw -t) output from a $(vw --oaa) model.
IDs should be the true values, unless base is supplied."""
cmx = ConfusionMX(os.path.basename(predictions), NumRP=NumRP)
if base is not None:
base = ord(base[0]) - 1
util.reading(predictions, ConfusionMX.logger)
with open(predictions) as ins:
for line in ins:
vals = line.split()
cmx.add(
int(vals[1]) if base is None else ord(vals[1][0]) - base,
int(float(vals[0])))
return cmx
def score (actual, predicted, delimiter='\t', abeg=1, pbeg=1, NumRP=0):
"""Score one file against another.
File format: <query string>[<delimiter><category>]+
The queries in both files must be identical.
abeg and pbeg are the columns where actual and pedicted categories start.
"""
mlc = MuLabCat(util.title_from_2paths(actual,predicted),NumRP=NumRP)
util.reading(actual,MuLabCat.logger)
util.reading(predicted,MuLabCat.logger)
empty = frozenset([''])
with open(actual) as af, open(predicted) as pf:
for sa,sp in itertools.izip_longest(csv.reader(af,delimiter=delimiter),
csv.reader(pf,delimiter=delimiter)):
if sa is None or sp is None:
raise ValueError("uneven files",af,pf,sa,sp)
if sa[0] != sp[0]:
raise ValueError("query string mismatch",sa,sp)
mlc.add(frozenset(sa[abeg:])-empty,frozenset(sp[pbeg:])-empty)
return mlc
def __init__ (self, model = None, vw = None, parser = None, logger = None):
if vw is None:
self.vw = parser.vw_exe = file2path(parser.vw_exe,"VowpalWabbit.exe")
else:
self.vw = file2path(vw,"VowpalWabbit.exe")
if model is None:
self.model = parser.vw_model = file2path(parser.vw_model,"VowpalWabbit.model")
else:
self.model = file2path(model,"VowpalWabbit.model")
self.logger = VowpalWabbitPipe.logger if logger is None else logger
cmd = [self.vw,"--initial_regressor",self.model,"--raw_predictions",
"/dev/stdout","--testonly"]
util.reading(self.model,self.logger)
self.logger.info("Running %s",cmd)
self.s = subprocess.Popen(cmd,stdout=subprocess.PIPE,stdin=subprocess.PIPE,
stderr=VowpalWabbitPipe.devnull)
self.logger.info("Started PID=%d",self.s.pid)
self.model = model
self.called = 0
self.pending = 0
def score(actual, predicted, delimiter='\t', abeg=1, pbeg=1, NumRP=0):
"""Score one file against another.
File format: <query string>[<delimiter><category>]+
The queries in both files must be identical.
abeg and pbeg are the columns where actual and pedicted categories start.
"""
mlc = MuLabCat(util.title_from_2paths(actual, predicted), NumRP=NumRP)
util.reading(actual, MuLabCat.logger)
util.reading(predicted, MuLabCat.logger)
empty = frozenset([''])
with open(actual) as af, open(predicted) as pf:
for sa, sp in itertools.izip_longest(
csv.reader(af, delimiter=delimiter),
csv.reader(pf, delimiter=delimiter)):
if sa is None or sp is None:
raise ValueError("uneven files", af, pf, sa, sp)
if sa[0] != sp[0]:
raise ValueError("query string mismatch", sa, sp)
mlc.add(
frozenset(sa[abeg:]) - empty,
frozenset(sp[pbeg:]) - empty)
return mlc
help='the Number of RandomPairs for testing numeric stability')
args = ap.parse_args()
if args.ConfusionMX:
if args.predictions is not None:
print ConfusionMX.score_vw_oaa(args.predictions,
base=args.base,
NumRP=args.NumRP)
else:
# Parse output from $(make -C vw/demo/mnist raw png)
vwdemo = os.path.expandvars("$HOME/src/sds-vw/demo")
if os.path.isdir(vwdemo):
for (dirpath, _dirnames, filenames) in os.walk(vwdemo):
for f in filenames:
if f.endswith('.out'):
f = os.path.join(dirpath, f)
util.reading(f, ConfusionMX.logger)
with open(f) as inst:
while True:
try:
cm = ConfusionMX.read_VW_demo_mnist(
inst)
except Exception as ex:
print ex
break
if cm is None:
break
print cm
if f.endswith('.predictions'):
try:
print ConfusionMX.score_vw_oaa(
os.path.join(dirpath, f),
ap.add_argument('-NumRP',type=int,default=0,
help='the Number of RandomPairs for testing numeric stability')
args = ap.parse_args()
if args.ConfusionMX:
if args.predictions is not None:
print ConfusionMX.score_vw_oaa(args.predictions, base=args.base,
NumRP=args.NumRP)
else:
# Parse output from $(make -C vw/demo/mnist raw png)
vwdemo = os.path.expandvars("$HOME/src/sds-vw/demo")
if os.path.isdir(vwdemo):
for (dirpath, _dirnames, filenames) in os.walk(vwdemo):
for f in filenames:
if f.endswith('.out'):
f = os.path.join(dirpath,f)
util.reading(f,ConfusionMX.logger)
with open(f) as inst:
while True:
try:
cm = ConfusionMX.read_VW_demo_mnist(inst)
except Exception as ex:
print ex
break
if cm is None:
break
print cm
if f.endswith('.predictions'):
try:
print ConfusionMX.score_vw_oaa(
os.path.join(dirpath,f),base=args.base,
NumRP=args.NumRP)
import numpy as np
video = "mp4_h264_aac.mp4"
cmd = "omxplayer -o hdmi " + video
cmdline = shlex.split(cmd)
ID = Setting.ID;
url = Setting.url;
Rtime = Setting.time
dist = Setting.dist
pre_exist = False #現在のavailable
while(True):
if(reading(0)<dist):
print "true"
#send room information to server
ch = Checker(ID, url, True, pre_exist)
pre_exist = ch.start()
sp = subprocess.Popen(cmdline,preexec_fn=os.setsid) #subprocess called
while(sp.poll() == None):
time.sleep(Rtime)
if(reading(0)>dist):
print "break"
os.killpg(sp.pid, signal.SIGTERM) #subprocess is stoped, send SIGTERM signal
break
else:
print "false"
#send room information to server
ch = Checker(ID, url, False, pre_exist)