Ejemplo n.º 1
0
def timePrint(message: str):
    
    timeStamp = datetime.now().strftime('%H:%M:%S')
    output    = f'[{timeStamp}] : {message}'
    
    print(output)
    
    with open('messages.txt', 'a') as log:
        log.write(output + '\n')
Ejemplo n.º 2
0
def security_start(conf):
    fixed_path = conf.default.prog_status_path
    if os.path.isfile(fixed_path):
        log.write("info", "Prog already exists, stop spawning.")
        sys.exit(2)

    dirname = os.path.dirname(os.path.abspath(fixed_path))
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    with open(fixed_path, "w") as f:
        f.write(str(os.getpid()))
Ejemplo n.º 3
0
def warn_with_traceback(message,
                        category,
                        filename,
                        lineno,
                        file=None,
                        line=None):

    log = file if hasattr(file, 'write') else sys.stderr
    traceback.print_stack(file=log)
    log.write(warnings.formatwarning(message, category, filename, lineno,
                                     line))
Ejemplo n.º 4
0
    def summary(self,
                method='standard',
                direction='central',
                freq=None,
                log=sys.stdout):
        if freq is not None:
            energies = freq * units.invcm
        else:
            energies = self.get_energies(method=method, direction=direction)

        summary_lines = VibrationsData._tabulate_from_energies(energies)
        log_text = '\n'.join(summary_lines) + '\n'

        if isinstance(log, str):
            with paropen(log, 'a') as log_file:
                log_file.write(log_text)
        else:
            log.write(log_text)
def trainCRF(data, c, maxiter, log):
    w = np.ndarray.flatten(np.zeros((26, 128)))
    t = np.ndarray.flatten(np.zeros((26, 26)))
    theta = np.concatenate([w, t])
    t0 = time.time()
    theta, fmin, _ = fmin_l_bfgs_b(likelihood,
                                   theta,
                                   fprime=likelihood_prime,
                                   args=(data, c),
                                   disp=1,
                                   maxiter=maxiter)
    t1 = time.time()
    log.write("Training time: " + str(t1 - t0) + "\n")
    p = 128
    w = np.reshape(theta[:26 * p], (26, p))
    t = np.reshape(theta[26 * p:], (26, 26))

    return (w, t)
Ejemplo n.º 6
0
def savehist(logmsg=""):
    import shutil, os, glob
    print "Saving current setup to hist"
    if not os.path.exists('./history/'):
        os.mkdir('./history')
        cur_entry=1
        log=file('./history/history.txt', 'w')
    else:
        if os.path.exists('./history/history.txt'): log=file('./history/history.txt', 'a')
        else: log=file('./history/history.txt', 'w')
        filelist=glob.glob('./history/*.hist.????')
        cur_entry=max(int(item[-4:]) for item in filelist)+1
    print "Creating new history item with number %04d"%cur_entry
    suffix=".hist.%04d"%cur_entry
    bup_files=['comp.ind', 'dica.dat', 'spct.dat']
    for ifile in bup_files:
        shutil.copy(ifile, './history/'+ifile+suffix)
    log.write("%04d log:"%(cur_entry)+logmsg+"\n")
    log.close()
Ejemplo n.º 7
0
def regret_vs_T(N, simulations=10, epsilon=None, Tstep=None, TperK=10):
    print "STARTED SIMULATION"
    a = 4.0
    q = part_balanced_q(N, 2)
    if epsilon:
        model = Parallel(q, epsilon)
        Tmin = 10
        vary_epsilon = False
    else:
        model = Parallel(q, .5)
        Tmin = int(ceil(4 * model.K / a))
        vary_epsilon = True

    Tstep = model.K if not Tstep else Tstep
    eta, mq = model.analytic_eta()

    T_vals = range(Tmin, TperK * model.K, Tstep)

    causal = GeneralCausal()
    causal_parallel = ParallelCausal()
    baseline = SuccessiveRejects()

    regret = np.zeros((len(T_vals), 4, simulations))

    with open(logfile(), "w") as log:
        for s in xrange(simulations):
            if s % 100 == 0:
                log.write(str(s) + "\n")
                log.flush()
            for T_indx, T in enumerate(T_vals):
                if vary_epsilon:  #variable epsilon
                    epsilon = sqrt(model.K / (a * T))
                    model.set_epsilon(epsilon)
                regret[T_indx, 0, s] = causal.run(T, model, eta, mq)
                regret[T_indx, 1, s] = causal_parallel.run(T, model)
                regret[T_indx, 2, s] = baseline.run(T, model)
                regret[T_indx, 3, s] = epsilon

    return regret
Ejemplo n.º 8
0
def datalog(nom, lastevent, mission_start):
    log = open('/sd/log{}.csv'.format(nom), 'a')
    senBMP()
    senMPU()
    senGPS(2)
    orientacio.calc()
    t = pyb.elapsed_millis(mission_start)
    line = '{},{},{},{},{},{},{},{},{},{},{},{}\n'.format(
        t,
        lastevent,
        str(my_gps.latitude).replace(",", "").replace("'", ""),
        str(my_gps.longitude).replace(",", "").replace("'", ""),
        p,
        altitud,
        90 + orientacio.calc()[1],
        VelVert,
        my_gps.speed[2] / 3.6,
        acX,
        acY,
        acZ,
    )
    print(line)
    log.write(line)
    log.close()
def testCRF(data, wmatr, transmatrx, log):
    predictions = []
    correct_letter, correct_word = 0, 0
    start_time = time.time()
    for x, y in data:
        infer = max_sum(x, wmatr, transmatrx).tolist()
        correct_letter += sum(1 if infer[i] == y[i] else 0
                              for i in range(0, len(y)))
        if infer == y.tolist():
            correct_word += 1
        predictions += infer
    character_accuracy = 100 * correct_letter / len(predictions)
    word_accuracy = 100 * correct_word / len(data)
    log.write("Time taken={}\n".format(time.time() - start_time))
    log.write("letter-wise accuracy={}%\n".format(character_accuracy))
    log.write("word-wise accuracy={}%\n".format(word_accuracy))
    return predictions
Ejemplo n.º 10
0
def fitOneEfficiency(name, hpass, hfail, sigModel, bkgModel, refpass, reffail, options):
    w = ROOT.RooWorkspace("w")
    w.factory("mass[%g,%g]" % (hpass.GetXaxis().GetXmin(), hpass.GetXaxis().GetXmax()))
    w.var("mass").setBins(hpass.GetNbinsX())
    w.factory("passing[yes=1,no=0]")
    # make combined dataset, relying on combine
    dpass = ROOT.RooDataHist("dpass","dpass", ROOT.RooArgList(w.var("mass")), hpass)
    dfail = ROOT.RooDataHist("dfail","dfail", ROOT.RooArgList(w.var("mass")), hfail)
    dfactory = ROOT.CombDataSetFactory(ROOT.RooArgSet(w.var("mass")), w.cat("passing"))
    dfactory.addSetBin("yes", dpass)
    dfactory.addSetBin("no",  dfail)
    data = dfactory.done("data","data")
    nsig = hpass.Integral(); nbkg = hfail.Integral(); nall = nsig+nbkg
    # make PDF
    signals = makeSignalModel(w, sigModel, refpass, reffail, options)
    if not signals: return None
    backgrounds = makeBackgroundModel(w, bkgModel, options)
    if not backgrounds: return None
    w.factory('expr::sigPass("@0*  @1",   Nsig[%g,0,%g], efficiency[0.95,0,1])' % (nsig,nall))
    w.factory('expr::sigFail("@0*(1-@1)", Nsig         , efficiency         )')
    w.factory('expr::bkgPass("@0*  @1",   Nbkg[%g,0,%g], effbkg[0.5,0,1])' % (nbkg,nall))
    w.factory('expr::bkgFail("@0*(1-@1)", Nbkg         , effbkg         )')
    w.factory('SUM::pdfPass(sigPass*{sigPdfPass}, bkgPass*{bkgPdfPass})'.format(sigPdfPass = signals[0], bkgPdfPass = backgrounds[0]))
    w.factory('SUM::pdfFail(sigFail*{sigPdfFail}, bkgFail*{bkgPdfFail})'.format(sigPdfFail = signals[1], bkgPdfFail = backgrounds[1]))
    w.factory('SIMUL::pdf(passing, yes=pdfPass, no=pdfFail)')   
    #pdf = ROOT.RooSimultaneousOpt(w.pdf("pdf0"), "pdf")
    #getattr(w,'import')(pdf, ROOT.RooFit.RecycleConflictNodes(True), ROOT.RooFit.Silence())

    getattr(w,'import')(dpass, ROOT.RooCmdArg())
    getattr(w,'import')(dfail, ROOT.RooCmdArg())
    # make FIT

    if hfail.Integral() == 0:
        w.var("efficiency").setVal(1)
        w.var("effbkg").setVal(1)
        w.var("efficiency").setConstant(True)
        w.var("effbkg").setConstant(True)
    #result = w.pdf("pdf").fitTo(data, ROOT.RooFit.Save(True), ROOT.RooFit.Minos(ROOT.RooArgSet(w.var("efficiency"))))
    nll, minim, retval = minimize(w.pdf("pdf"), data, options, strategy=0)
    if options.doScan: minim.minimize("Minuit2","scan");
    #minim.minimize("Minuit2","migrad")
    minim.hesse()
    nll, minim, retval = minimize(w.pdf("pdf"), data, options, strategy=None)
    if hfail.Integral() > 0 and w.var("efficiency").getVal() > 0.9:
        w.var("efficiency").setMin(0.85)
        retval = minim.minimize("Minuit2","migrad");
    go = True; niter = 0; minoslog = []; efferr = None
    while go:
        niter += 1
        if niter == 5: 
            minim.minimize("Minuit2","scan");
        if niter == 7: 
            minim.minimize("Minuit","minimize");
        if niter >= 10:
            minoslog += [ "STOP AFTER %d ITERATIONS" % niter ]
            go = False
        minim.hesse()
        nll, minim, retval = minimize(w.pdf("pdf"), data, options, strategy=None)
        result = minim.save()
        effval = w.var("efficiency").getVal()
        if hfail.Integral() == 0:
            cpcalc = ROOT.TEfficiency.ClopperPearson
            nsig = int(floor(w.var("Nsig").getVal() - w.var("Nsig").getError()))
            emin = cpcalc(nsig,nsig,0.6827,False)
            efferr = [ emin-1, 0 ]
            minoslog += [ "No passing probes: using clopper pearson for the fitted number of signal events minus -1 sigma from the fit."]
            minoslog += [ "Nsig fit: %.1f +- %.1f " % (w.var("Nsig").getVal(), w.var("Nsig").getError()) ]
            minoslog += [ "CP lower bound for %d events, all passing: %.4f" % (nsig,emin) ]
        else:
            try:
                efferr, minoslog2 = manualMinos(w.pdf("pdf"), data, w.var("efficiency"), options)
                go = False; minoslog += minoslog2
            except MinosException:
                print         "Negative value in Minos, restarting"
                minoslog += [ "Negative value in Minos, restarting everything", '------' ]
                w.var("efficiency").setConstant(False)
                
    #print "MANUAL MINOS: ",efferr,"\n\t","\n\t".join(minoslog)
    # plot
    c1 = ROOT.TCanvas("c1","c1"); c1.SetCanvasSize(900,500);
    c1.Divide(2,1)
    c1.cd(1)
    fpass = w.var("mass").frame()
    dpass.plotOn(fpass)
    w.pdf("pdfPass").plotOn(fpass, ROOT.RooFit.Components(backgrounds[0]),ROOT.RooFit.LineColor(ROOT.kGreen+3))
    w.pdf("pdfPass").plotOn(fpass, ROOT.RooFit.LineColor(ROOT.kGreen+1))
    fpass.Draw()
    c1.cd(2)
    ffail = w.var("mass").frame()
    dfail.plotOn(ffail)
    w.pdf("pdfFail").plotOn(ffail, ROOT.RooFit.Components(backgrounds[1]),ROOT.RooFit.LineColor(ROOT.kRed+3))
    w.pdf("pdfFail").plotOn(ffail, ROOT.RooFit.LineColor(ROOT.kRed+0))
    ffail.Draw()
    for ext in "pdf","png":
        if ext == "pdf" and not options.fullOut: continue
        c1.Print("%s/%s.%s" % (options.printDirBins, name, ext))
    log = open("%s/%s.txt" % (options.printDirBins, name), "w")
    fpf = result.floatParsFinal()
    for i in range(fpf.getSize()):
        par = fpf.at(i)
        log.write("%-20s : " % par.GetName())
        if par.hasAsymError():
            log.write("%8.4f  %+8.4f / %+8.4f" % (par.getVal(), par.getAsymErrorLo(), par.getAsymErrorHi()))
            plo, phi = par.getVal() - par.getAsymErrorLo(), par.getVal() + par.getAsymErrorHi()
        else:
            if par.GetName() in ("Nsig", "Nbkg"):
                log.write("%8.1f  %+8.1f           " % (par.getVal(), par.getError()))
            else:
                log.write("%8.4f  %+8.4f           " % (par.getVal(), par.getError()))
            plo, phi = par.getVal() - par.getError(), par.getVal() + par.getError()
        pmax, pmin = par.getMax(), par.getMin()
        log.write("   [ %8.4f , %8.4f ]" % (pmin, pmax))
        if phi > 0.95*pmax + 0.05*pmin: log.write("   <-- close to max")
        if plo < 0.05*pmax + 0.95*pmin: log.write("   <-- close to min")
        log.write("\n")

    log.write("MANUAL MINOS: %s\n\t%s\n"  % (efferr,"\n\t".join(minoslog)) )

    # goodness of fit
    chi2pass = chi2(hpass, w.var("mass"), w.pdf("pdfPass"), w.function("sigPass").getVal() + w.function("bkgPass").getVal())
    chi2fail = chi2(hfail, w.var("mass"), w.pdf("pdfFail"), w.function("sigFail").getVal() + w.function("bkgFail").getVal())
    log.write("\nChi2: pass %.2f, fail %.2f, total %.2f, ndof: %d - %d = %d \n" % (
        chi2pass, chi2fail, chi2pass + chi2fail, 
        hpass.GetNbinsX()*2, fpf.getSize(),  hpass.GetNbinsX()*2 - fpf.getSize()+1))

    log.write("\n"+options.textBlob+"\n")

    log.close()

    if options.fullOut:
        tfout = ROOT.TFile.Open("%s/%s.%s" % (options.printDirBins, name, "root"), "RECREATE")
        w.SetName("w_"+name)
        tfout.WriteTObject(w, "w_"+name)
        tfout.Close()

    # report result
    if efferr: 
        return effval, efferr[0], efferr[1]
    else:
        return None
Ejemplo n.º 11
0
def mesurar(nom):
    log = open('/sd/log{}.csv'.format(nom), 'w')
    log.write(
        'temps (ms), etapa, latitud, longitud, pressio (Pa), altitud (m), AoA(º), Velocitat(m/s), Velocitat_GPS(m/s), acX(m/s²), acY(m/s²), acZ(m/s²)\n'
    )
    log.close()
Ejemplo n.º 12
0
        frequent = set([x for x in frequent])

    coca = open("_COCA2_mapping.txt", "r")
    mapping = {}
    log = open("conversionlog.txt", "w+")

    for line in coca:
        bgs = json.loads(line)
        bgs = [[x.strip(), y.strip()]
               for x, y in bgs]  #if not x.startswith("##")]
        for string, lemma in bgs:
            if string in frequent:
                if string in mapping:
                    if mapping[string] == lemma:
                        pass
                    else:
                        log.write('%s : %s versus %s\n' %
                                  (string, lemma, mapping[string]))
                else:
                    mapping[string] = lemma
            else:
                print(string)
        del bgs
    coca.close()

    import gc
    gc.collect()

    with open("mapping.json", "w+") as f:
        json.dump(mapping, f)
Ejemplo n.º 13
0
                os.remove(deletable_file)

    if file_count > 0:
        file_total_size = sizeof_fmt(total_size)
        space_freed = total_size + space_freed
        time_stamp = time.asctime(time.localtime(time.time()))
        log_out.append(
            "%s - Deleted %s files older than %s days totalling %s - %s \n" %
            (time_stamp, file_count, days, file_total_size, working_dir))
    else:
        time_stamp = time.asctime(time.localtime(time.time()))
        log_out.append("%s - No files older than %s days found - %s \n" %
                       (time_stamp, days, working_dir))

summary_line = "%s - %s of disk space freed \n" % (time_stamp,
                                                   sizeof_fmt(space_freed))

if test_mode:
    for line in log_out:
        print line
    print summary_line
    print "Running in test mode - no files were removed and no log entry was made"
    done()
else:
    log = open(log_file, 'a')
    for line in log_out:
        log.write(line)
    log.write(summary_line)
    log.close()
    sys.exit(0)
Ejemplo n.º 14
0
def print_log(print_string, log):
  print("{}".format(print_string))
  log.write('{}\n'.format(print_string))
  log.flush()
Ejemplo n.º 15
0
        for deletable_file in files:
            if os.path.isfile(deletable_file) and not test_mode:
                os.remove(deletable_file)
    
    if file_count > 0:
        file_total_size = sizeof_fmt(total_size)
        space_freed = total_size + space_freed
        time_stamp = time.asctime( time.localtime(time.time()))
        log_out.append("%s - Deleted %s files older than %s days totalling %s - %s \n" % (time_stamp, file_count, days, file_total_size, working_dir))
    else:
        time_stamp = time.asctime( time.localtime(time.time()))
        log_out.append("%s - No files older than %s days found - %s \n" % (time_stamp, days, working_dir))

	
summary_line = "%s - %s of disk space freed \n" % (time_stamp, sizeof_fmt(space_freed))
        
if test_mode:
    for line in log_out:
        print line
    print summary_line
    print "Running in test mode - no files were removed and no log entry was made"
    done()
else:
    log = open(log_file, 'a')
    for line in log_out:
        log.write(line)
    log.write(summary_line)
    log.close()
    sys.exit(0)

Ejemplo n.º 16
0
def fitOneEfficiency(name, hpass, hfail, sigModel, bkgModel, refpass, reffail,
                     options):
    w = ROOT.RooWorkspace("w")
    w.factory("mass[%g,%g]" %
              (hpass.GetXaxis().GetXmin(), hpass.GetXaxis().GetXmax()))
    w.var("mass").setBins(hpass.GetNbinsX())
    w.factory("passing[yes=1,no=0]")
    # make combined dataset, relying on combine
    dpass = ROOT.RooDataHist("dpass", "dpass", ROOT.RooArgList(w.var("mass")),
                             hpass)
    dfail = ROOT.RooDataHist("dfail", "dfail", ROOT.RooArgList(w.var("mass")),
                             hfail)
    dfactory = ROOT.CombDataSetFactory(ROOT.RooArgSet(w.var("mass")),
                                       w.cat("passing"))
    dfactory.addSetBin("yes", dpass)
    dfactory.addSetBin("no", dfail)
    data = dfactory.done("data", "data")
    nsig = hpass.Integral()
    nbkg = hfail.Integral()
    nall = nsig + nbkg
    # make PDF
    signals = makeSignalModel(w, sigModel, refpass, reffail, options)
    if not signals: return None
    backgrounds = makeBackgroundModel(w, bkgModel, options)
    if not backgrounds: return None
    w.factory(
        'expr::sigPass("@0*  @1",   Nsig[%g,0,%g], efficiency[0.9,0,1])' %
        (nsig, nall))
    w.factory('expr::sigFail("@0*(1-@1)", Nsig         , efficiency         )')
    w.factory('expr::bkgPass("@0*  @1",   Nbkg[%g,0,%g], effbkg[0.5,0,1])' %
              (nbkg, nall))
    w.factory('expr::bkgFail("@0*(1-@1)", Nbkg         , effbkg         )')
    w.factory(
        'SUM::pdfPass(sigPass*{sigPdfPass}, bkgPass*{bkgPdfPass})'.format(
            sigPdfPass=signals[0], bkgPdfPass=backgrounds[0]))
    w.factory(
        'SUM::pdfFail(sigFail*{sigPdfFail}, bkgFail*{bkgPdfFail})'.format(
            sigPdfFail=signals[1], bkgPdfFail=backgrounds[1]))
    w.factory('SIMUL::pdf(passing, yes=pdfPass, no=pdfFail)')

    getattr(w, 'import')(dpass, ROOT.RooCmdArg())
    getattr(w, 'import')(dfail, ROOT.RooCmdArg())
    # make FIT

    #result = w.pdf("pdf").fitTo(data, ROOT.RooFit.Save(True), ROOT.RooFit.Minos(ROOT.RooArgSet(w.var("efficiency"))))
    nll, minim, retval = minimize(w.pdf("pdf"), data, options, strategy=0)
    nll, minim, retval = minimize(w.pdf("pdf"), data, options, strategy=None)
    if w.var("efficiency").getVal() > 0.9:
        w.var("efficiency").setMin(0.85)
        retval = minim.minimize("Minuit2", "migrad")
    result = minim.save()
    effval = w.var("efficiency").getVal()
    efferr, minoslog = manualMinos(w.pdf("pdf"), data, w.var("efficiency"),
                                   options)
    #poi = ROOT.RooArgSet(w.var("efficiency"))
    #ret = minim.minos(poi)
    print "MANUAL MINOS: ", efferr, "\n\t", "\n\t".join(minoslog)

    # plot
    c1 = ROOT.TCanvas("c1", "c1")
    c1.SetCanvasSize(900, 500)
    c1.Divide(2, 1)
    c1.cd(1)
    fpass = w.var("mass").frame()
    dpass.plotOn(fpass)
    w.pdf("pdfPass").plotOn(fpass, ROOT.RooFit.Components(backgrounds[0]),
                            ROOT.RooFit.LineColor(ROOT.kGreen + 3))
    w.pdf("pdfPass").plotOn(fpass, ROOT.RooFit.LineColor(ROOT.kGreen + 1))
    fpass.Draw()
    c1.cd(2)
    ffail = w.var("mass").frame()
    dfail.plotOn(ffail)
    w.pdf("pdfFail").plotOn(ffail, ROOT.RooFit.Components(backgrounds[1]),
                            ROOT.RooFit.LineColor(ROOT.kRed + 3))
    w.pdf("pdfFail").plotOn(ffail, ROOT.RooFit.LineColor(ROOT.kRed + 0))
    ffail.Draw()
    for ext in "pdf", "png":
        c1.Print("%s/%s.%s" % (options.printDirBins, name, ext))
    log = open("%s/%s.txt" % (options.printDirBins, name), "w")
    fpf = result.floatParsFinal()
    for i in range(fpf.getSize()):
        par = fpf.at(i)
        log.write("%-20s : " % par.GetName())
        if par.hasAsymError():
            log.write(
                "%8.4f  %+8.4f / %+8.4f" %
                (par.getVal(), par.getAsymErrorLo(), par.getAsymErrorHi()))
            plo, phi = par.getVal() - par.getAsymErrorLo(), par.getVal(
            ) + par.getAsymErrorHi()
        else:
            if par.GetName() in ("Nsig", "Nbkg"):
                log.write("%8.1f  %+8.1f           " %
                          (par.getVal(), par.getError()))
            else:
                log.write("%8.4f  %+8.4f           " %
                          (par.getVal(), par.getError()))
            plo, phi = par.getVal() - par.getError(), par.getVal(
            ) + par.getError()
        pmax, pmin = par.getMax(), par.getMin()
        log.write("   [ %8.4f , %8.4f ]" % (pmin, pmax))
        if phi > 0.95 * pmax + 0.05 * pmin: log.write("   <-- close to max")
        if plo < 0.05 * pmax + 0.95 * pmin: log.write("   <-- close to min")
        log.write("\n")

    log.write("MANUAL MINOS: %s\n\t%s\n" % (efferr, "\n\t".join(minoslog)))

    # goodness of fit
    chi2pass = chi2(
        hpass, w.var("mass"), w.pdf("pdfPass"),
        w.function("sigPass").getVal() + w.function("bkgPass").getVal())
    chi2fail = chi2(
        hfail, w.var("mass"), w.pdf("pdfFail"),
        w.function("sigFail").getVal() + w.function("bkgFail").getVal())
    log.write(
        "\nChi2: pass %.2f, fail %.2f, total %.2f, ndof: %d - %d = %d \n" %
        (chi2pass, chi2fail, chi2pass + chi2fail, hpass.GetNbinsX() * 2,
         fpf.getSize(), hpass.GetNbinsX() * 2 - fpf.getSize() + 1))
    log.close()

    tfout = ROOT.TFile.Open("%s/%s.%s" % (options.printDirBins, name, "root"),
                            "RECREATE")
    w.SetName("w_" + name)
    tfout.WriteTObject(w, "w_" + name)
    tfout.Close()

    # report result
    if efferr:
        return effval, efferr[0], efferr[1]
    else:
        return None