def timePrint(message: str):
timeStamp = datetime.now().strftime('%H:%M:%S')
output = f'[{timeStamp}] : {message}'
print(output)
with open('messages.txt', 'a') as log:
log.write(output + '\n')
def security_start(conf):
fixed_path = conf.default.prog_status_path
if os.path.isfile(fixed_path):
log.write("info", "Prog already exists, stop spawning.")
sys.exit(2)
dirname = os.path.dirname(os.path.abspath(fixed_path))
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(fixed_path, "w") as f:
f.write(str(os.getpid()))
def warn_with_traceback(message,
category,
filename,
lineno,
file=None,
line=None):
log = file if hasattr(file, 'write') else sys.stderr
traceback.print_stack(file=log)
log.write(warnings.formatwarning(message, category, filename, lineno,
line))
def summary(self,
method='standard',
direction='central',
freq=None,
log=sys.stdout):
if freq is not None:
energies = freq * units.invcm
else:
energies = self.get_energies(method=method, direction=direction)
summary_lines = VibrationsData._tabulate_from_energies(energies)
log_text = '\n'.join(summary_lines) + '\n'
if isinstance(log, str):
with paropen(log, 'a') as log_file:
log_file.write(log_text)
else:
log.write(log_text)
def trainCRF(data, c, maxiter, log):
w = np.ndarray.flatten(np.zeros((26, 128)))
t = np.ndarray.flatten(np.zeros((26, 26)))
theta = np.concatenate([w, t])
t0 = time.time()
theta, fmin, _ = fmin_l_bfgs_b(likelihood,
theta,
fprime=likelihood_prime,
args=(data, c),
disp=1,
maxiter=maxiter)
t1 = time.time()
log.write("Training time: " + str(t1 - t0) + "\n")
p = 128
w = np.reshape(theta[:26 * p], (26, p))
t = np.reshape(theta[26 * p:], (26, 26))
return (w, t)
def savehist(logmsg=""):
import shutil, os, glob
print "Saving current setup to hist"
if not os.path.exists('./history/'):
os.mkdir('./history')
cur_entry=1
log=file('./history/history.txt', 'w')
else:
if os.path.exists('./history/history.txt'): log=file('./history/history.txt', 'a')
else: log=file('./history/history.txt', 'w')
filelist=glob.glob('./history/*.hist.????')
cur_entry=max(int(item[-4:]) for item in filelist)+1
print "Creating new history item with number %04d"%cur_entry
suffix=".hist.%04d"%cur_entry
bup_files=['comp.ind', 'dica.dat', 'spct.dat']
for ifile in bup_files:
shutil.copy(ifile, './history/'+ifile+suffix)
log.write("%04d log:"%(cur_entry)+logmsg+"\n")
log.close()
def regret_vs_T(N, simulations=10, epsilon=None, Tstep=None, TperK=10):
print "STARTED SIMULATION"
a = 4.0
q = part_balanced_q(N, 2)
if epsilon:
model = Parallel(q, epsilon)
Tmin = 10
vary_epsilon = False
else:
model = Parallel(q, .5)
Tmin = int(ceil(4 * model.K / a))
vary_epsilon = True
Tstep = model.K if not Tstep else Tstep
eta, mq = model.analytic_eta()
T_vals = range(Tmin, TperK * model.K, Tstep)
causal = GeneralCausal()
causal_parallel = ParallelCausal()
baseline = SuccessiveRejects()
regret = np.zeros((len(T_vals), 4, simulations))
with open(logfile(), "w") as log:
for s in xrange(simulations):
if s % 100 == 0:
log.write(str(s) + "\n")
log.flush()
for T_indx, T in enumerate(T_vals):
if vary_epsilon: #variable epsilon
epsilon = sqrt(model.K / (a * T))
model.set_epsilon(epsilon)
regret[T_indx, 0, s] = causal.run(T, model, eta, mq)
regret[T_indx, 1, s] = causal_parallel.run(T, model)
regret[T_indx, 2, s] = baseline.run(T, model)
regret[T_indx, 3, s] = epsilon
return regret
def datalog(nom, lastevent, mission_start):
log = open('/sd/log{}.csv'.format(nom), 'a')
senBMP()
senMPU()
senGPS(2)
orientacio.calc()
t = pyb.elapsed_millis(mission_start)
line = '{},{},{},{},{},{},{},{},{},{},{},{}\n'.format(
t,
lastevent,
str(my_gps.latitude).replace(",", "").replace("'", ""),
str(my_gps.longitude).replace(",", "").replace("'", ""),
p,
altitud,
90 + orientacio.calc()[1],
VelVert,
my_gps.speed[2] / 3.6,
acX,
acY,
acZ,
)
print(line)
log.write(line)
log.close()
def testCRF(data, wmatr, transmatrx, log):
predictions = []
correct_letter, correct_word = 0, 0
start_time = time.time()
for x, y in data:
infer = max_sum(x, wmatr, transmatrx).tolist()
correct_letter += sum(1 if infer[i] == y[i] else 0
for i in range(0, len(y)))
if infer == y.tolist():
correct_word += 1
predictions += infer
character_accuracy = 100 * correct_letter / len(predictions)
word_accuracy = 100 * correct_word / len(data)
log.write("Time taken={}\n".format(time.time() - start_time))
log.write("letter-wise accuracy={}%\n".format(character_accuracy))
log.write("word-wise accuracy={}%\n".format(word_accuracy))
return predictions
def fitOneEfficiency(name, hpass, hfail, sigModel, bkgModel, refpass, reffail, options):
w = ROOT.RooWorkspace("w")
w.factory("mass[%g,%g]" % (hpass.GetXaxis().GetXmin(), hpass.GetXaxis().GetXmax()))
w.var("mass").setBins(hpass.GetNbinsX())
w.factory("passing[yes=1,no=0]")
# make combined dataset, relying on combine
dpass = ROOT.RooDataHist("dpass","dpass", ROOT.RooArgList(w.var("mass")), hpass)
dfail = ROOT.RooDataHist("dfail","dfail", ROOT.RooArgList(w.var("mass")), hfail)
dfactory = ROOT.CombDataSetFactory(ROOT.RooArgSet(w.var("mass")), w.cat("passing"))
dfactory.addSetBin("yes", dpass)
dfactory.addSetBin("no", dfail)
data = dfactory.done("data","data")
nsig = hpass.Integral(); nbkg = hfail.Integral(); nall = nsig+nbkg
# make PDF
signals = makeSignalModel(w, sigModel, refpass, reffail, options)
if not signals: return None
backgrounds = makeBackgroundModel(w, bkgModel, options)
if not backgrounds: return None
w.factory('expr::sigPass("@0* @1", Nsig[%g,0,%g], efficiency[0.95,0,1])' % (nsig,nall))
w.factory('expr::sigFail("@0*(1-@1)", Nsig , efficiency )')
w.factory('expr::bkgPass("@0* @1", Nbkg[%g,0,%g], effbkg[0.5,0,1])' % (nbkg,nall))
w.factory('expr::bkgFail("@0*(1-@1)", Nbkg , effbkg )')
w.factory('SUM::pdfPass(sigPass*{sigPdfPass}, bkgPass*{bkgPdfPass})'.format(sigPdfPass = signals[0], bkgPdfPass = backgrounds[0]))
w.factory('SUM::pdfFail(sigFail*{sigPdfFail}, bkgFail*{bkgPdfFail})'.format(sigPdfFail = signals[1], bkgPdfFail = backgrounds[1]))
w.factory('SIMUL::pdf(passing, yes=pdfPass, no=pdfFail)')
#pdf = ROOT.RooSimultaneousOpt(w.pdf("pdf0"), "pdf")
#getattr(w,'import')(pdf, ROOT.RooFit.RecycleConflictNodes(True), ROOT.RooFit.Silence())
getattr(w,'import')(dpass, ROOT.RooCmdArg())
getattr(w,'import')(dfail, ROOT.RooCmdArg())
# make FIT
if hfail.Integral() == 0:
w.var("efficiency").setVal(1)
w.var("effbkg").setVal(1)
w.var("efficiency").setConstant(True)
w.var("effbkg").setConstant(True)
#result = w.pdf("pdf").fitTo(data, ROOT.RooFit.Save(True), ROOT.RooFit.Minos(ROOT.RooArgSet(w.var("efficiency"))))
nll, minim, retval = minimize(w.pdf("pdf"), data, options, strategy=0)
if options.doScan: minim.minimize("Minuit2","scan");
#minim.minimize("Minuit2","migrad")
minim.hesse()
nll, minim, retval = minimize(w.pdf("pdf"), data, options, strategy=None)
if hfail.Integral() > 0 and w.var("efficiency").getVal() > 0.9:
w.var("efficiency").setMin(0.85)
retval = minim.minimize("Minuit2","migrad");
go = True; niter = 0; minoslog = []; efferr = None
while go:
niter += 1
if niter == 5:
minim.minimize("Minuit2","scan");
if niter == 7:
minim.minimize("Minuit","minimize");
if niter >= 10:
minoslog += [ "STOP AFTER %d ITERATIONS" % niter ]
go = False
minim.hesse()
nll, minim, retval = minimize(w.pdf("pdf"), data, options, strategy=None)
result = minim.save()
effval = w.var("efficiency").getVal()
if hfail.Integral() == 0:
cpcalc = ROOT.TEfficiency.ClopperPearson
nsig = int(floor(w.var("Nsig").getVal() - w.var("Nsig").getError()))
emin = cpcalc(nsig,nsig,0.6827,False)
efferr = [ emin-1, 0 ]
minoslog += [ "No passing probes: using clopper pearson for the fitted number of signal events minus -1 sigma from the fit."]
minoslog += [ "Nsig fit: %.1f +- %.1f " % (w.var("Nsig").getVal(), w.var("Nsig").getError()) ]
minoslog += [ "CP lower bound for %d events, all passing: %.4f" % (nsig,emin) ]
else:
try:
efferr, minoslog2 = manualMinos(w.pdf("pdf"), data, w.var("efficiency"), options)
go = False; minoslog += minoslog2
except MinosException:
print "Negative value in Minos, restarting"
minoslog += [ "Negative value in Minos, restarting everything", '------' ]
w.var("efficiency").setConstant(False)
#print "MANUAL MINOS: ",efferr,"\n\t","\n\t".join(minoslog)
# plot
c1 = ROOT.TCanvas("c1","c1"); c1.SetCanvasSize(900,500);
c1.Divide(2,1)
c1.cd(1)
fpass = w.var("mass").frame()
dpass.plotOn(fpass)
w.pdf("pdfPass").plotOn(fpass, ROOT.RooFit.Components(backgrounds[0]),ROOT.RooFit.LineColor(ROOT.kGreen+3))
w.pdf("pdfPass").plotOn(fpass, ROOT.RooFit.LineColor(ROOT.kGreen+1))
fpass.Draw()
c1.cd(2)
ffail = w.var("mass").frame()
dfail.plotOn(ffail)
w.pdf("pdfFail").plotOn(ffail, ROOT.RooFit.Components(backgrounds[1]),ROOT.RooFit.LineColor(ROOT.kRed+3))
w.pdf("pdfFail").plotOn(ffail, ROOT.RooFit.LineColor(ROOT.kRed+0))
ffail.Draw()
for ext in "pdf","png":
if ext == "pdf" and not options.fullOut: continue
c1.Print("%s/%s.%s" % (options.printDirBins, name, ext))
log = open("%s/%s.txt" % (options.printDirBins, name), "w")
fpf = result.floatParsFinal()
for i in range(fpf.getSize()):
par = fpf.at(i)
log.write("%-20s : " % par.GetName())
if par.hasAsymError():
log.write("%8.4f %+8.4f / %+8.4f" % (par.getVal(), par.getAsymErrorLo(), par.getAsymErrorHi()))
plo, phi = par.getVal() - par.getAsymErrorLo(), par.getVal() + par.getAsymErrorHi()
else:
if par.GetName() in ("Nsig", "Nbkg"):
log.write("%8.1f %+8.1f " % (par.getVal(), par.getError()))
else:
log.write("%8.4f %+8.4f " % (par.getVal(), par.getError()))
plo, phi = par.getVal() - par.getError(), par.getVal() + par.getError()
pmax, pmin = par.getMax(), par.getMin()
log.write(" [ %8.4f , %8.4f ]" % (pmin, pmax))
if phi > 0.95*pmax + 0.05*pmin: log.write(" <-- close to max")
if plo < 0.05*pmax + 0.95*pmin: log.write(" <-- close to min")
log.write("\n")
log.write("MANUAL MINOS: %s\n\t%s\n" % (efferr,"\n\t".join(minoslog)) )
# goodness of fit
chi2pass = chi2(hpass, w.var("mass"), w.pdf("pdfPass"), w.function("sigPass").getVal() + w.function("bkgPass").getVal())
chi2fail = chi2(hfail, w.var("mass"), w.pdf("pdfFail"), w.function("sigFail").getVal() + w.function("bkgFail").getVal())
log.write("\nChi2: pass %.2f, fail %.2f, total %.2f, ndof: %d - %d = %d \n" % (
chi2pass, chi2fail, chi2pass + chi2fail,
hpass.GetNbinsX()*2, fpf.getSize(), hpass.GetNbinsX()*2 - fpf.getSize()+1))
log.write("\n"+options.textBlob+"\n")
log.close()
if options.fullOut:
tfout = ROOT.TFile.Open("%s/%s.%s" % (options.printDirBins, name, "root"), "RECREATE")
w.SetName("w_"+name)
tfout.WriteTObject(w, "w_"+name)
tfout.Close()
# report result
if efferr:
return effval, efferr[0], efferr[1]
else:
return None
def mesurar(nom):
log = open('/sd/log{}.csv'.format(nom), 'w')
log.write(
'temps (ms), etapa, latitud, longitud, pressio (Pa), altitud (m), AoA(º), Velocitat(m/s), Velocitat_GPS(m/s), acX(m/s²), acY(m/s²), acZ(m/s²)\n'
)
log.close()
frequent = set([x for x in frequent])
coca = open("_COCA2_mapping.txt", "r")
mapping = {}
log = open("conversionlog.txt", "w+")
for line in coca:
bgs = json.loads(line)
bgs = [[x.strip(), y.strip()]
for x, y in bgs] #if not x.startswith("##")]
for string, lemma in bgs:
if string in frequent:
if string in mapping:
if mapping[string] == lemma:
pass
else:
log.write('%s : %s versus %s\n' %
(string, lemma, mapping[string]))
else:
mapping[string] = lemma
else:
print(string)
del bgs
coca.close()
import gc
gc.collect()
with open("mapping.json", "w+") as f:
json.dump(mapping, f)
os.remove(deletable_file)
if file_count > 0:
file_total_size = sizeof_fmt(total_size)
space_freed = total_size + space_freed
time_stamp = time.asctime(time.localtime(time.time()))
log_out.append(
"%s - Deleted %s files older than %s days totalling %s - %s \n" %
(time_stamp, file_count, days, file_total_size, working_dir))
else:
time_stamp = time.asctime(time.localtime(time.time()))
log_out.append("%s - No files older than %s days found - %s \n" %
(time_stamp, days, working_dir))
summary_line = "%s - %s of disk space freed \n" % (time_stamp,
sizeof_fmt(space_freed))
if test_mode:
for line in log_out:
print line
print summary_line
print "Running in test mode - no files were removed and no log entry was made"
done()
else:
log = open(log_file, 'a')
for line in log_out:
log.write(line)
log.write(summary_line)
log.close()
sys.exit(0)
def print_log(print_string, log):
print("{}".format(print_string))
log.write('{}\n'.format(print_string))
log.flush()
for deletable_file in files:
if os.path.isfile(deletable_file) and not test_mode:
os.remove(deletable_file)
if file_count > 0:
file_total_size = sizeof_fmt(total_size)
space_freed = total_size + space_freed
time_stamp = time.asctime( time.localtime(time.time()))
log_out.append("%s - Deleted %s files older than %s days totalling %s - %s \n" % (time_stamp, file_count, days, file_total_size, working_dir))
else:
time_stamp = time.asctime( time.localtime(time.time()))
log_out.append("%s - No files older than %s days found - %s \n" % (time_stamp, days, working_dir))
summary_line = "%s - %s of disk space freed \n" % (time_stamp, sizeof_fmt(space_freed))
if test_mode:
for line in log_out:
print line
print summary_line
print "Running in test mode - no files were removed and no log entry was made"
done()
else:
log = open(log_file, 'a')
for line in log_out:
log.write(line)
log.write(summary_line)
log.close()
sys.exit(0)
def fitOneEfficiency(name, hpass, hfail, sigModel, bkgModel, refpass, reffail,
options):
w = ROOT.RooWorkspace("w")
w.factory("mass[%g,%g]" %
(hpass.GetXaxis().GetXmin(), hpass.GetXaxis().GetXmax()))
w.var("mass").setBins(hpass.GetNbinsX())
w.factory("passing[yes=1,no=0]")
# make combined dataset, relying on combine
dpass = ROOT.RooDataHist("dpass", "dpass", ROOT.RooArgList(w.var("mass")),
hpass)
dfail = ROOT.RooDataHist("dfail", "dfail", ROOT.RooArgList(w.var("mass")),
hfail)
dfactory = ROOT.CombDataSetFactory(ROOT.RooArgSet(w.var("mass")),
w.cat("passing"))
dfactory.addSetBin("yes", dpass)
dfactory.addSetBin("no", dfail)
data = dfactory.done("data", "data")
nsig = hpass.Integral()
nbkg = hfail.Integral()
nall = nsig + nbkg
# make PDF
signals = makeSignalModel(w, sigModel, refpass, reffail, options)
if not signals: return None
backgrounds = makeBackgroundModel(w, bkgModel, options)
if not backgrounds: return None
w.factory(
'expr::sigPass("@0* @1", Nsig[%g,0,%g], efficiency[0.9,0,1])' %
(nsig, nall))
w.factory('expr::sigFail("@0*(1-@1)", Nsig , efficiency )')
w.factory('expr::bkgPass("@0* @1", Nbkg[%g,0,%g], effbkg[0.5,0,1])' %
(nbkg, nall))
w.factory('expr::bkgFail("@0*(1-@1)", Nbkg , effbkg )')
w.factory(
'SUM::pdfPass(sigPass*{sigPdfPass}, bkgPass*{bkgPdfPass})'.format(
sigPdfPass=signals[0], bkgPdfPass=backgrounds[0]))
w.factory(
'SUM::pdfFail(sigFail*{sigPdfFail}, bkgFail*{bkgPdfFail})'.format(
sigPdfFail=signals[1], bkgPdfFail=backgrounds[1]))
w.factory('SIMUL::pdf(passing, yes=pdfPass, no=pdfFail)')
getattr(w, 'import')(dpass, ROOT.RooCmdArg())
getattr(w, 'import')(dfail, ROOT.RooCmdArg())
# make FIT
#result = w.pdf("pdf").fitTo(data, ROOT.RooFit.Save(True), ROOT.RooFit.Minos(ROOT.RooArgSet(w.var("efficiency"))))
nll, minim, retval = minimize(w.pdf("pdf"), data, options, strategy=0)
nll, minim, retval = minimize(w.pdf("pdf"), data, options, strategy=None)
if w.var("efficiency").getVal() > 0.9:
w.var("efficiency").setMin(0.85)
retval = minim.minimize("Minuit2", "migrad")
result = minim.save()
effval = w.var("efficiency").getVal()
efferr, minoslog = manualMinos(w.pdf("pdf"), data, w.var("efficiency"),
options)
#poi = ROOT.RooArgSet(w.var("efficiency"))
#ret = minim.minos(poi)
print "MANUAL MINOS: ", efferr, "\n\t", "\n\t".join(minoslog)
# plot
c1 = ROOT.TCanvas("c1", "c1")
c1.SetCanvasSize(900, 500)
c1.Divide(2, 1)
c1.cd(1)
fpass = w.var("mass").frame()
dpass.plotOn(fpass)
w.pdf("pdfPass").plotOn(fpass, ROOT.RooFit.Components(backgrounds[0]),
ROOT.RooFit.LineColor(ROOT.kGreen + 3))
w.pdf("pdfPass").plotOn(fpass, ROOT.RooFit.LineColor(ROOT.kGreen + 1))
fpass.Draw()
c1.cd(2)
ffail = w.var("mass").frame()
dfail.plotOn(ffail)
w.pdf("pdfFail").plotOn(ffail, ROOT.RooFit.Components(backgrounds[1]),
ROOT.RooFit.LineColor(ROOT.kRed + 3))
w.pdf("pdfFail").plotOn(ffail, ROOT.RooFit.LineColor(ROOT.kRed + 0))
ffail.Draw()
for ext in "pdf", "png":
c1.Print("%s/%s.%s" % (options.printDirBins, name, ext))
log = open("%s/%s.txt" % (options.printDirBins, name), "w")
fpf = result.floatParsFinal()
for i in range(fpf.getSize()):
par = fpf.at(i)
log.write("%-20s : " % par.GetName())
if par.hasAsymError():
log.write(
"%8.4f %+8.4f / %+8.4f" %
(par.getVal(), par.getAsymErrorLo(), par.getAsymErrorHi()))
plo, phi = par.getVal() - par.getAsymErrorLo(), par.getVal(
) + par.getAsymErrorHi()
else:
if par.GetName() in ("Nsig", "Nbkg"):
log.write("%8.1f %+8.1f " %
(par.getVal(), par.getError()))
else:
log.write("%8.4f %+8.4f " %
(par.getVal(), par.getError()))
plo, phi = par.getVal() - par.getError(), par.getVal(
) + par.getError()
pmax, pmin = par.getMax(), par.getMin()
log.write(" [ %8.4f , %8.4f ]" % (pmin, pmax))
if phi > 0.95 * pmax + 0.05 * pmin: log.write(" <-- close to max")
if plo < 0.05 * pmax + 0.95 * pmin: log.write(" <-- close to min")
log.write("\n")
log.write("MANUAL MINOS: %s\n\t%s\n" % (efferr, "\n\t".join(minoslog)))
# goodness of fit
chi2pass = chi2(
hpass, w.var("mass"), w.pdf("pdfPass"),
w.function("sigPass").getVal() + w.function("bkgPass").getVal())
chi2fail = chi2(
hfail, w.var("mass"), w.pdf("pdfFail"),
w.function("sigFail").getVal() + w.function("bkgFail").getVal())
log.write(
"\nChi2: pass %.2f, fail %.2f, total %.2f, ndof: %d - %d = %d \n" %
(chi2pass, chi2fail, chi2pass + chi2fail, hpass.GetNbinsX() * 2,
fpf.getSize(), hpass.GetNbinsX() * 2 - fpf.getSize() + 1))
log.close()
tfout = ROOT.TFile.Open("%s/%s.%s" % (options.printDirBins, name, "root"),
"RECREATE")
w.SetName("w_" + name)
tfout.WriteTObject(w, "w_" + name)
tfout.Close()
# report result
if efferr:
return effval, efferr[0], efferr[1]
else:
return None