Example #1
0
def backup(SD_NAME=None, session=None, erase=False):
    if not os.path.exists('raw'):
        os.makedirs('raw')
    SD_NAME = 'test'
    SD_ROOT = '/media/' + os.getusername() + '/' + SD_NAME
    DATE = time.strftime('%Y-%m-%d_%H-%M-%S')
    if not session:
        BACKUP_DIR = 'video_' + DATE
    else:
        BACKUP_DIR = session
    os.chdir('raw')
    if not os.path.exists(BACKUP_DIR):
        os.makedirs(BACKUP_DIR)
    os.chdir(BACKUP_DIR)
    command = [
        'rsync', '-a', '-r', '--stats', '--progress',
        SD_ROOT + '/PRIVATE/AVCHD/BDMV/STREAM', '.'
    ]
    if subprocess.check_call(command) == 0:
        if erase:
            shutil.rmtree(SD_ROOT)
        print("Done")
    else:
        print("Error while running rsync")
    for filename in glob.glob('00*.MTS'):
        os.rename(filename, DATE + filename)
    os.chdir('../..')
Example #2
0
def postInstall(fromVersion, fromRelease, toVersion, toRelease):
    for _file, perms in permissions.items():
        # The list above is general, some paths may not exist depending on the configuration
        if os.path.exists(_file):
            os.system("/bin/chown -R %s %s" % (perms[1], _file))
            os.system("/bin/chmod %s %s" % (perms[0], _file))
    os.system("groupadd libvirt")
    os.system("usermod -G libvirt %s" % os.getusername())
Example #3
0
def postInstall(fromVersion, fromRelease, toVersion, toRelease):
    try:
        os.system("/usr/sbin/groupadd %s" % OUR_NAME)
        os.system("usermod -a -G docker %s" % os.getusername())
        os.system("/bin/touch %s" % logfile)
        os.system("/bin/chown root:docker %s" % logfile)
        os.system("chmod 0644 %s" % logfile)
    except:
        pass
Example #4
0
 def prepare_transmit( self, job, resume ):
    # NOW get the data
    self.file_to_send = job.get_attr( iftfile.JOB_ATTR_SRC_NAME )
    self.remote_host = job.get_attr( iftfile.JOB_ATTR_DEST_HOST )
    self.remote_path = job.get_attr( iftfile.JOB_ATTR_DEST_NAME )
    self.remote_user = job.get_attr( IFTSCP_REMOTE_LOGIN )
    if self.remote_user == None:
       self.remote_user = os.getusername()
    return 0
Example #5
0
def postInstall(fromVersion, fromRelease, toVersion, toRelease):
    try:
        os.system("/usr/sbin/groupadd %s" % OUR_NAME)
        os.system("usermod -a -G docker %s" % os.getusername())
        os.system("/bin/touch %s" % logfile)
        os.system("/bin/chown root:docker %s" % logfile)
        os.system("chmod 0644 %s" % logfile)
    except:
        pass
Example #6
0
def postInstall(fromVersion, fromRelease, toVersion, toRelease):
    try:
        os.system("usr/sbin/groupadd -g %d %s" % (OUR_ID, OUR_NAME))
        os.system("usermod -a -G docker %s" % os.getusername())
        os.system("/bin/touch %s" % logfile)
        os.system("/bin/chown root:docker %s" % logfile)
        os.system("chmod 0644 %s" % logfile)
        os.system("rc-update add docker default")
    except:
        pass
Example #7
0
def copyConfigAndConnect(hosts):
    print(
        "Connecting to remote hosts and copying config files, tapes and snapshots"
    )
    #create a list of files to be copied across all the remote hosts
    file_list = []
    for h in hosts:
        for r in h['runs']:
            if (r.isFromSnapshot):
                file_list.append(r.snapshotFile)
            else:
                file_list.append(r.tapeFilename)
    file_list.append(main.__file__)
    for host in hosts:
        if (host['name'] != socket.gethostname()
            ):  #if I am not the computer named in host name
            try:
                username = host['username']
            except:
                username = os.getusername(
                )  #default username is current user user's name
            try:
                port = host['port']
            except:
                port = 22  #default ssh port
            rm = Remote.Connection(hostname=host['address'],
                                   username=username,
                                   port=port)
            rm.connect()
            #			print ("Sendind file:"+main.__file__)
            if ('remotebasepath' in host):
                remote_dir = host['remotebasepath']
            else:
                remote_dir = 'trisurf_simulations'
            rm.send_multiple_files_in_directory(file_list, remote_dir)
            #			rm.send_file(main.__file__,'remote_control.py')
            #			for run in host['runs']:
            #				try:
            #					rm.send_file(run.tapeFile,run.tapeFile)
            #				except:
            #					pass
            #				try:
            #					rm.send_file(run.snapshotFile,run.snapshotFile)
            #				except:
            #					pass
            host['_conn'] = rm
    # we are connected to all hosts...
    return hosts
Example #8
0
def get_users() -> list:
	if not requirement.are_presents(['win32net', 'win32netcon']):
		if requirement.is_present('os'):
			return [os.getusername()]
		else:
			return []
	filter = win32netcon.FILTER_NORMAL_ACCOUNT
	resume_handle = 0
	user_list = []
	while True:
		result = win32net.NetUserEnum(None, 0, filter, resume_handle)
		user_list += [user['name'] for user in result[0]]
		resume_handle = result[2]
		if not resume_handle:
			break
	user_list.sort()
	return user_list
Example #9
0
    def callback_notes(self):
        notes_path = os.path.join(self.sim_dir, "notes.txt")
        if os.path.isfile(notes_path) == False:
            out = open(notes_path, 'w')
            out.write("Notes on the simulation: " +
                      os.path.basenam(self.sim_dir) + "\n")
            out.write("Date: " +
                      datetime.datetime.today().strftime('%Y-%m-%d') + "\n")
            out.write("Generated by: " + os.getusername() + "\n")
            out.write("\n")
            out.close()
        else:
            out = open(notes_path, 'a')
            out.write("\n" + datetime.datetime.today().strftime('%Y-%m-%d') +
                      ":\n")
            out.close()

        desktop_open(notes_path)
Example #10
0
    def __openConnection(self):
        try:
            self.client_pre.load_system_host_keys()
            self.client_pre.set_missing_host_key_policy(
                paramiko.WarningPolicy())

            if not self.username:
                self.username = os.getusername()

            self.client_pre.connect(self.hostname,
                                    port=self.port,
                                    username=self.username,
                                    password=self.password,
                                    allow_agent=False,
                                    look_for_keys=False)
            self.client = self.client_pre.invoke_shell()
            output = self.client.recv(65535)
            print(output)
        finally:
            print(" Connection to {host} successfully set up. ".format(
                host=self.hostname).center(80, '*'))
Example #11
0
def backup(SD_NAME=None, session=None, erase=False):
	if not os.path.exists('raw'):
		os.makedirs('raw')
	SD_NAME = 'test' 
	SD_ROOT = '/media/' + os.getusername() + '/' + SD_NAME
	DATE = time.strftime('%Y-%m-%d_%H-%M-%S')
	if not session:
		BACKUP_DIR = 'video_' + DATE
	else:
		BACKUP_DIR = session
	os.chdir('raw')
	if not os.path.exists(BACKUP_DIR):
		os.makedirs(BACKUP_DIR)
	os.chdir(BACKUP_DIR)
	command = ['rsync', '-a', '-r', '--stats', '--progress', SD_ROOT+'/PRIVATE/AVCHD/BDMV/STREAM', '.']
	if subprocess.check_call(command) == 0:
		if erase:
			shutil.rmtree(SD_ROOT)
		print("Done")
	else:
		print("Error while running rsync")
	for filename in glob.glob('00*.MTS'):
		os.rename(filename,DATE+filename)
	os.chdir('../..')
Example #12
0
  def from_assigned_task(self, assigned_task, sandbox):
    """
    :param assigned_task:
    :param sandbox:
    :return: Instance of a HealthChecker.
    """
    mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
    health_check_config = mesos_task.health_check_config().get()
    health_checker = health_check_config.get('health_checker', {})
    timeout_secs = health_check_config.get('timeout_secs')
    if SHELL_HEALTH_CHECK in health_checker:
      shell_command = health_checker.get(SHELL_HEALTH_CHECK, {}).get('shell_command')

      # Filling in variables e.g. thermos.ports[http] that could have been passed in as part of
      # shell_command.
      interpolated_command = HealthCheckerProvider.interpolate_cmd(
        task=assigned_task,
        cmd=shell_command)

      # If we do not want the health check to execute as the user from the job's role
      # --nosetuid-health-checks should be passed as an argument to the executor.
      demote_to_job_role_user = None
      if not self._nosetuid_health_checks and not sandbox.is_filesystem_image:
        pw_entry = pwd.getpwnam(assigned_task.task.job.role)
        def demote_to_job_role_user():
          os.setgid(pw_entry.pw_gid)
          os.setuid(pw_entry.pw_uid)

      # If the task is executing in an isolated filesystem we'll want to wrap the health check
      # command within a mesos-containerizer invocation so that it's executed within that
      # filesystem.
      wrapper = None
      if sandbox.is_filesystem_image:
        health_check_user = (os.getusername() if self._nosetuid_health_checks
            else assigned_task.task.job.role)
        def wrapper(cmd):
          return wrap_with_mesos_containerizer(
              cmd,
              health_check_user,
              sandbox.container_root,
              self._mesos_containerizer_path)

      shell_signaler = ShellHealthCheck(
        cmd=interpolated_command,
        preexec_fn=demote_to_job_role_user,
        timeout_secs=timeout_secs,
        wrapper_fn=wrapper)
      a_health_checker = lambda: shell_signaler()
    else:
      portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
      if 'health' not in portmap:
        return None
      http_config = health_checker.get(HTTP_HEALTH_CHECK, {})
      http_endpoint = http_config.get('endpoint')
      http_expected_response = http_config.get('expected_response')
      http_expected_response_code = http_config.get('expected_response_code')

      http_signaler = HttpSignaler(
        portmap['health'],
        timeout_secs=timeout_secs)
      a_health_checker = lambda: http_signaler(
        endpoint=http_endpoint,
        expected_response=http_expected_response,
        expected_response_code=http_expected_response_code
      )

    health_checker = HealthChecker(
      a_health_checker,
      sandbox,
      interval_secs=health_check_config.get('interval_secs'),
      initial_interval_secs=health_check_config.get('initial_interval_secs'),
      max_consecutive_failures=health_check_config.get('max_consecutive_failures'))

    return health_checker
Example #13
0
File: ssh.py Project: r3c/creep
	def __init__ (self, host, port, user, directory, options):
		extra = shlex.split (options.get ('extra', ''))
		remote = str ((user or os.getusername ()) + '@' + (host or 'localhost'))

		self.directory = directory
		self.tunnel = ['ssh', '-T', '-p', str (port or 22)] + extra + [remote]
# Print out the command line arguments in sys.argv, one per line:
import sys
a = sys.argv[1]
b = sys.argv[2]
sum = a + b
# print("sum of a and b is {}".format(int(a)+ int(b)))

# Print out the OS platform you're using:
print(sys.platform)

# Print out the version of Python you're using:
print(sys.version)

import os
# See the docs for the OS module: https://docs.python.org/3.7/library/os.html

os.name
# Print the current process ID
print(os.getpid())

# Print the current working directory (cwd):
print(os.getcwd())  # get current working directory
print(os.listdir("."))  # files in the current direcotry

# Print out your machine's login name
os.getusername()
# The above didn't work so I used the followinf import instead
import getpass
getpass.getuser()
def optimizeMultiDim(options,args):


    signals = options.signals
    backgrounds = options.backgrounds

    variables = options.variables
    observable = options.observable
    selection = options.selection
    cutoff = options.cutoff

    selectioncuts = options.selectioncuts
    
    ndim = len(variables)
    ws = None

    cutoffs = numpy.array([cutoff]*ndim)
    
    obs,obsalias = defineVariables( [observable], options.label )
    obs = obs[0]
    mu = ROOT.RooRealVar("mu","mu",1.,0.,10.)
    
    varlist,aliases = defineVariables( variables, options.label )
    sellist,selaliases = defineVariables( selectioncuts, options.label )

    print "---------------------------------------------"
    print "Observables "
    obs.Print("")

    print 
    print "Variables"
    varlist.Print("V")

    print
    print "Selection cuts"
    sellist.Print("V")

    aliases.extend(obsalias+selaliases)
    print
    print "Aliases"
    pprint(aliases)
    print
    
    if options.infile == "":
        options.infile = args[0]
    fin = ROOT.TFile.Open(options.infile)

    if options.cont:
        if os.path.exists(os.path.abspath(options.cont)):
            options.cont = os.path.abspath(options.cont)
        else:
            options.cont = os.path.abspath(os.path.join(os.path.dirname(options.outdir),options.cont))
    if not os.path.exists(options.outdir):
        os.mkdir(options.outdir)
    os.chdir(options.outdir)

    if not os.path.exists("/tmp/%s" % os.getusername()):
        os.mkdir("/tmp/%s" % os.getusername())
    tmp = ROOT.TFile.Open("/tmp/%s/categoryOptimizationMultiDim.root" % os.getusername() ,"recreate")
    tmp.cd()

    ### ##########################################################################################################
    ### Minimizer and optimizer
    ###
    minimizer = ROOT.TMinuitMinimizer("Minimize")
    ## minimizer.SetPrintLevel(999)
    ## minimizer = ROOT.Minuit2.Minuit2Minimizer()
    optimizer = ROOT.CategoryOptimizer( minimizer, ndim )
    for isel in range(sellist.getSize()):
        sel = sellist[isel]
        if sel.GetName() in options.fix:
            optimizer.addFixedOrthoCut(sel.GetName(),sel.getVal())
        else:
            optimizer.addFloatingOrthoCut(sel.GetName(),sel.getVal(),(sel.getMax()-sel.getMin())/sel.getBins(),sel.getMin(),sel.getMax())
    optimizer.absoluteBoundaries()  ## Float absolut boundaries instead of telescopic ones
    for opt in options.settings:
        if isinstance(opt, types.StringTypes):
            getattr(optimizer,opt)()
        else:
            name, args = opt
            getattr(optimizer,name)(*args)

    
    ### ##########################################################################################################
    ### Model builders
    ###
    print "---------------------------------------------"
    print "Reading inputs"
    print
    sigTrees = [ mergeTrees(fin,selection,name,trees,aliases) for name,trees in signals.iteritems() ]
    bkgTrees = [ mergeTrees(fin,selection,name,trees,aliases) for name,trees in backgrounds.iteritems() ]

    for tree in sigTrees+bkgTrees:
        tree.Write()
        
    fin.Close()

    print "---------------------------------------------"
    print "Booking model buildes"
    print
    signals,sigconstr = modelBuilders( sigTrees, ROOT.AbsModel.sig, obs, varlist, sellist,
                                       getattr(options,"weights",{}), getattr(options,"shapes",{}), {}, {} )
    backgrounds,bkgconstr = modelBuilders( bkgTrees, ROOT.AbsModel.bkg, obs, varlist, sellist,
                                           getattr(options,"weights",{}), getattr(options,"shapes",{}),
                                           getattr(options,"minevents",{},), getattr(options,"constrained",{}),
                                           )
    
    if options.saveCompactTree:
        for model in signals+backgrounds:
            model.getTree().Write()
        
    normTF1s = []
    sumxTF1s = []
    sumx2TF1s = []
    for model in backgrounds:
        norm = model.getTF1N() 
        x    = model.getTF1X()
        x2   = model.getTF1X2() 
        norm.SetLineColor(ROOT.kRed)
        x.SetLineColor(ROOT.kRed)
        x2.SetLineColor(ROOT.kRed)
        normTF1s.append(norm)
        sumxTF1s.append(x)
        sumx2TF1s.append(x2)

    for model in signals:
        norm = model.getTF1N()
        x    = model.getTF1X()
        x2   = model.getTF1X2() 
        norm.SetLineColor(ROOT.kBlue)
        x.SetLineColor(ROOT.kBlue)
        x2.SetLineColor(ROOT.kBlue)
        normTF1s.append(norm)
        sumxTF1s.append(x)
        sumx2TF1s.append(x2)
        
    canv2 = ROOT.TCanvas("canv2","canv2")
    canv2.cd()
    normTF1s[0].Draw("")
    for tf1 in normTF1s[1:]:
        tf1.Draw("SAME")
    canv2.SaveAs("cat_opt_cdf.png")

    ### canv3 = ROOT.TCanvas("canv3","canv3")
    ### canv3.cd()
    ### hbkgMass.SetLineColor(ROOT.kRed)
    ### hbkgMass.Draw("hist")
    ### for hsigMass in hsigsMass:
    ###     hsigMass.SetLineColor(ROOT.kBlue)
    ###     hsigMass.Draw("hist SAME")
    ### canv3.SaveAs("cat_opt_mass.png")
    
    canv4 = ROOT.TCanvas("canv4","canv4")
    canv4.cd()
    sumxTF1s[0].Draw("hist")
    for tf1 in sumxTF1s[1:]:
        tf1.Draw("hist SAME")
    canv4.SaveAs("cat_opt_sum_mass.png")

    canv5 = ROOT.TCanvas("canv5","canv5")
    canv5.cd()
    sumx2TF1s[0].Draw("hist")
    for tf1 in sumx2TF1s[1:]:
        tf1.Draw("hist SAME")
    canv5.SaveAs("cat_opt_sum_mass2.png")

    objs.append( signals )
    objs.append( backgrounds )
    objs.append( normTF1s )
    objs.append( sumxTF1s )
    objs.append( sumx2TF1s )
    
    ### #########################################################################################################
    ### Figure of merit for optimization
    ###

    ### Simple counting
    ## fom       = ROOT.NaiveCutAndCountFomProvider()
    ## fom       = ROOT.PoissonCutAndCountFomProvider()

    ### ### Likelihood ratio using asymptotic approx.
    fom       = ROOT.SimpleShapeFomProvider()
    for sigModel in signals:
        sigModel.getModel().setMu(mu)
    fom.addPOI(mu)
    fom.minStrategy(2)
    for constraint in sigconstr+bkgconstr:
        print constraint
        if type(constraint) == tuple:
            var, pdf, pars = constraint
            print var,pdf,pars
            fom.addNuisance(var,pdf)
        else:
            fom.addNuisance(constraint)
    
    ## fom.minimizer("Minuit2")
    ## fom.useRooSimultaneous()
    
    ### #########################################################################################################
    ### Run optimization
    ###
        
    for sigModel in signals:
        optimizer.addSignal( sigModel, True )
        ## optimizer.addSignal( sigModel )
    for bkgModel in backgrounds:
        optimizer.addBackground( bkgModel )
    optimizer.setFigureOfMerit( fom )
    
    summary = optmizeCats( optimizer, ws, varlist.getSize(),
                           options.range, (cutoffs,options.dry,True,), options.cont, options.reduce, options.refit )
    
    ### #########################################################################################################
    ### Some plots
    ###
    
    grS = ROOT.TGraph()
    grS.SetName("zVsNcat")
    grS.SetTitle(";n_{cat};f.o.m [A.U.]")
    for ncat,val in summary.iteritems():
        if( val["fom"] < 0. ) :
            grS.SetPoint( grS.GetN(), float(ncat), -val["fom"] )
    grS.Sort()
    mincat = grS.GetX()[0]
    maxcat = grS.GetX()[grS.GetN()-1]
    ncat = int(maxcat - mincat)

    for idim in range(ndim):
        var = varlist[idim]
        name = var.GetName()
        minX = var.getMin()
        maxX = var.getMax()
        nbinsX = var.getBinning().numBoundaries()-1
        hbound = ROOT.TH2F("hbound_%s" % name,"hbound_%s" % name,nbinsX+3,minX-1.5*(maxX-minX)/nbinsX,maxX+1.5*(maxX-minX)/nbinsX,ncat+3,mincat-1.5,maxcat+1.5)
        for jcat,val in summary.iteritems():
            nbound = int(jcat)+1
            for ib in range(nbound):
                bd = float(val["boundaries"][ib+idim*nbound])
                ## hbound.Fill(float(jcat),bd)
                hbound.Fill(bd,float(jcat))
        cbound = ROOT.TCanvas( "cat_opt_%s" % hbound.GetName(), "cat_opt_%s" % hbound.GetName() )
        cbound.cd()
        hbound.Draw("box")

        cbound_pj = ROOT.TCanvas( "cat_opt_%s_pj" % hbound.GetName(),  "cat_opt_%s_pj" %  hbound.GetName() )
        cbound_pj.cd()
        hbound_pj = hbound.Clone()
        ### hbound_pj = hbound.ProjectionY()
        ### hbound_pj.Draw()
        hbound_pj.Draw("box")
        hbound_pj.SetFillColor(ROOT.kBlack)
        hbound_pj.SetLineColor(ROOT.kBlack)
        objs.append(hbound)
        objs.append(hbound_pj)
        objs.append(cbound)
        objs.append(cbound_pj)
        maxy = 0.
        pdfs = []
        for sig in signals:
            pdf = sig.getPdf(idim)
            pdf.Scale(1./pdf.Integral())
            pdf.SetLineColor(ROOT.kBlue)
            pdfs.append(pdf)
            maxy = max(maxy,pdf.GetMaximum())
            objs.append(pdf)
        for bkg in backgrounds:
            pdf = bkg.getPdf(idim)
            pdf.Scale(1./pdf.Integral())
            pdf.SetLineColor(ROOT.kRed)
            pdfs.append(pdf)
            maxy = max(maxy,pdf.GetMaximum())
            objs.append(pdf)
            
        ### hbound_pj.Scale(maxy*hbound_pj.GetMaximum())
        ### hbound_pj.GetYaxis().SetRangeUser(0.,1.1*maxy)
        ### cbound_pj.RedrawAxis()
        ### cbound_pj.Update()
        for pdf in pdfs:
            pdf.Scale( (ncat+3.)/maxy )
            pdf.Draw("hist same")
        hbound_pj.GetYaxis().SetNdivisions(500+ncat+3)
        cbound_pj.SetGridy()
        hbound_pj.Draw("box same")
        
        for fmt in "png", "C":
            cbound.SaveAs("%s.%s" % (cbound.GetName(),fmt) )
            cbound_pj.SaveAs("%s.%s" % (cbound_pj.GetName(),fmt) )
            
    for isel in range(sellist.getSize()):
        var = sellist[isel]
        name = var.GetName()
        minX = var.getMin()
        maxX = var.getMax()
        nbinsX = var.getBinning().numBoundaries()-1
        print var, name, minX, maxX, nbinsX
        hsel = ROOT.TH2F("hsel_%s" % name,"hsel_%s" % name,nbinsX+3,minX-1.5*(maxX-minX)/nbinsX,maxX+1.5*(maxX-minX)/nbinsX,ncat+3,mincat-1.5,maxcat+1.5)
        for jcat,val in summary.iteritems():
            bd = float(val["selections"][isel])
            hsel.Fill(bd,float(jcat))
        csel = ROOT.TCanvas( "cat_opt_%s" % hsel.GetName(), "cat_opt_%s" % hsel.GetName() )
        csel.cd()
        hsel.Draw("box")

        csel_pj = ROOT.TCanvas( "cat_opt_%s_pj" % hsel.GetName(),  "cat_opt_%s_pj" %  hsel.GetName() )
        csel_pj.cd()
        hsel_pj = hsel.Clone()
        ### hsel_pj = hsel.ProjectionY()
        ### hsel_pj.Draw()
        hsel_pj.Draw("box")
        hsel_pj.SetFillColor(ROOT.kBlack)
        hsel_pj.SetLineColor(ROOT.kBlack)
        objs.append(hsel)
        objs.append(hsel_pj)
        objs.append(csel)
        objs.append(csel_pj)
        maxy = 0.
        pdfs = []
        for sig in signals:
            pdf = sig.getPdf(ndim+isel)
            pdf.SetLineColor(ROOT.kBlue)
            pdf.Scale(1./pdf.Integral())
            pdfs.append(pdf)
            maxy = max(maxy,pdf.GetMaximum())
            objs.append(pdf)
        for bkg in backgrounds:
            pdf = bkg.getPdf(ndim+isel)
            pdf.Scale(1./pdf.Integral())
            pdf.SetLineColor(ROOT.kRed)
            pdfs.append(pdf)
            maxy = max(maxy,pdf.GetMaximum())
            objs.append(pdf)
            
        for pdf in pdfs:
            pdf.Scale( (ncat+1.5)/maxy/pdf.Integral() )
            pdf.Draw("l same")
        hsel_pj.GetYaxis().SetNdivisions(500+ncat+3)
        csel_pj.SetGridy()
        hsel_pj.Draw("box same")


        for fmt in "png", "C":
            csel.SaveAs("%s.%s" % (csel.GetName(),fmt) )
            csel_pj.SaveAs("%s.%s" % (csel.GetName(),fmt) )

            
    canv9 = ROOT.TCanvas("canv9","canv9")
    canv9.SetGridx()
    canv9.SetGridy()
    canv9.cd()
    grS.SetMarkerStyle(ROOT.kFullCircle)
    grS.Draw("AP")
    
    canv9.SaveAs("cat_opt_fom.png")
    canv9.SaveAs("cat_opt_fom.C")
    
    ## tmp.Close()
    return ws
Example #16
0
#!/usr/bin/python

import stl
import socket
import argparse
import os

username = os.getusername()
personal_log = '/home/' + username +'/.stl/personal-stats-' + socket.gethostname() + '.log'
work_log = '/home/' + username + '/.stl/work-stats-' + socket.gethostname() + '.log'

projects = {
     'stl' : {
           'path' : '/home/' + username + '/projects/stl/docs/source/',
           'target' : 'projects',
           'emacs' : True,
           'quiet' : True,
           'log' : personal_log,
           'ext': 'txt'
      },
}

def set_quiet(force, project_quiet):
    if force is True:
        quiet = False
    else:
        quiet = project_quiet

    return quiet

def main():
Example #17
0
    def __init__(self, host, port, user, directory, options):
        extra = shlex.split(options.get('extra', ''))
        remote = str((user or os.getusername()) + '@' + (host or 'localhost'))

        self.directory = directory
        self.tunnel = ['ssh', '-T', '-p', str(port or 22)] + extra + [remote]
Example #18
0
  def from_assigned_task(self, assigned_task, sandbox):
    """
    :param assigned_task:
    :param sandbox:
    :return: Instance of a HealthChecker.
    """
    mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
    health_check_config = mesos_task.health_check_config().get()
    health_checker = health_check_config.get('health_checker', {})
    timeout_secs = health_check_config.get('timeout_secs')
    if SHELL_HEALTH_CHECK in health_checker:
      shell_command = health_checker.get(SHELL_HEALTH_CHECK, {}).get('shell_command')

      # Filling in variables e.g. thermos.ports[http] that could have been passed in as part of
      # shell_command.
      interpolated_command = HealthCheckerProvider.interpolate_cmd(
        task=assigned_task,
        cmd=shell_command)

      # If we do not want the health check to execute as the user from the job's role
      # --nosetuid-health-checks should be passed as an argument to the executor.
      demote_to_job_role_user = None
      if not self._nosetuid_health_checks and not sandbox.is_filesystem_image:
        pw_entry = pwd.getpwnam(assigned_task.task.job.role)
        def demote_to_job_role_user():
          os.setgid(pw_entry.pw_gid)
          os.setuid(pw_entry.pw_uid)

      # If the task is executing in an isolated filesystem we'll want to wrap the health check
      # command within a mesos-containerizer invocation so that it's executed within that
      # filesystem.
      wrapper = None
      if sandbox.is_filesystem_image:
        health_check_user = (os.getusername() if self._nosetuid_health_checks
            else assigned_task.task.job.role)
        def wrapper(cmd):
          return wrap_with_mesos_containerizer(
              cmd,
              health_check_user,
              sandbox.container_root,
              self._mesos_containerizer_path)

      shell_signaler = ShellHealthCheck(
        cmd=interpolated_command,
        preexec_fn=demote_to_job_role_user,
        timeout_secs=timeout_secs,
        wrapper_fn=wrapper)
      a_health_checker = lambda: shell_signaler()
    else:
      portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
      if 'health' not in portmap:
        log.warning('No health-checks defined, will use a no-op health-checker.')
        return NoopHealthChecker()
      http_config = health_checker.get(HTTP_HEALTH_CHECK, {})
      http_endpoint = http_config.get('endpoint')
      http_expected_response = http_config.get('expected_response')
      http_expected_response_code = http_config.get('expected_response_code')

      http_signaler = HttpSignaler(
        portmap['health'],
        timeout_secs=timeout_secs)
      a_health_checker = lambda: http_signaler(
        endpoint=http_endpoint,
        expected_response=http_expected_response,
        expected_response_code=http_expected_response_code
      )

    health_checker = HealthChecker(
      a_health_checker,
      sandbox,
      interval_secs=health_check_config.get('interval_secs'),
      grace_period_secs=health_check_config.get('initial_interval_secs'),
      max_consecutive_failures=health_check_config.get('max_consecutive_failures'),
      min_consecutive_successes=health_check_config.get('min_consecutive_successes'))

    return health_checker
Example #19
0
def optimizeMultiDim(options, args):

    signals = options.signals
    backgrounds = options.backgrounds

    variables = options.variables
    observable = options.observable
    selection = options.selection
    cutoff = options.cutoff

    selectioncuts = options.selectioncuts

    ndim = len(variables)
    ws = None

    cutoffs = numpy.array([cutoff] * ndim)

    obs, obsalias = defineVariables([observable], options.label)
    obs = obs[0]
    mu = ROOT.RooRealVar("mu", "mu", 1., 0., 10.)

    varlist, aliases = defineVariables(variables, options.label)
    sellist, selaliases = defineVariables(selectioncuts, options.label)

    print "---------------------------------------------"
    print "Observables "
    obs.Print("")

    print
    print "Variables"
    varlist.Print("V")

    print
    print "Selection cuts"
    sellist.Print("V")

    aliases.extend(obsalias + selaliases)
    print
    print "Aliases"
    pprint(aliases)
    print

    if options.infile == "":
        options.infile = args[0]
    fin = ROOT.TFile.Open(options.infile)

    if options.cont:
        if os.path.exists(os.path.abspath(options.cont)):
            options.cont = os.path.abspath(options.cont)
        else:
            options.cont = os.path.abspath(
                os.path.join(os.path.dirname(options.outdir), options.cont))
    if not os.path.exists(options.outdir):
        os.mkdir(options.outdir)
    os.chdir(options.outdir)

    if not os.path.exists("/tmp/%s" % os.getusername()):
        os.mkdir("/tmp/%s" % os.getusername())
    tmp = ROOT.TFile.Open(
        "/tmp/%s/categoryOptimizationMultiDim.root" % os.getusername(),
        "recreate")
    tmp.cd()

    ### ##########################################################################################################
    ### Minimizer and optimizer
    ###
    minimizer = ROOT.TMinuitMinimizer("Minimize")
    ## minimizer.SetPrintLevel(999)
    ## minimizer = ROOT.Minuit2.Minuit2Minimizer()
    optimizer = ROOT.CategoryOptimizer(minimizer, ndim)
    for isel in range(sellist.getSize()):
        sel = sellist[isel]
        if sel.GetName() in options.fix:
            optimizer.addFixedOrthoCut(sel.GetName(), sel.getVal())
        else:
            optimizer.addFloatingOrthoCut(sel.GetName(), sel.getVal(),
                                          (sel.getMax() - sel.getMin()) /
                                          sel.getBins(), sel.getMin(),
                                          sel.getMax())
    optimizer.absoluteBoundaries(
    )  ## Float absolut boundaries instead of telescopic ones
    for opt in options.settings:
        if isinstance(opt, types.StringTypes):
            getattr(optimizer, opt)()
        else:
            name, args = opt
            getattr(optimizer, name)(*args)

    ### ##########################################################################################################
    ### Model builders
    ###
    print "---------------------------------------------"
    print "Reading inputs"
    print
    sigTrees = [
        mergeTrees(fin, selection, name, trees, aliases)
        for name, trees in signals.iteritems()
    ]
    bkgTrees = [
        mergeTrees(fin, selection, name, trees, aliases)
        for name, trees in backgrounds.iteritems()
    ]

    for tree in sigTrees + bkgTrees:
        tree.Write()

    fin.Close()

    print "---------------------------------------------"
    print "Booking model buildes"
    print
    signals, sigconstr = modelBuilders(sigTrees, ROOT.AbsModel.sig, obs,
                                       varlist, sellist,
                                       getattr(options, "weights", {}),
                                       getattr(options, "shapes", {}), {}, {})
    backgrounds, bkgconstr = modelBuilders(
        bkgTrees,
        ROOT.AbsModel.bkg,
        obs,
        varlist,
        sellist,
        getattr(options, "weights", {}),
        getattr(options, "shapes", {}),
        getattr(
            options,
            "minevents",
            {},
        ),
        getattr(options, "constrained", {}),
    )

    if options.saveCompactTree:
        for model in signals + backgrounds:
            model.getTree().Write()

    normTF1s = []
    sumxTF1s = []
    sumx2TF1s = []
    for model in backgrounds:
        norm = model.getTF1N()
        x = model.getTF1X()
        x2 = model.getTF1X2()
        norm.SetLineColor(ROOT.kRed)
        x.SetLineColor(ROOT.kRed)
        x2.SetLineColor(ROOT.kRed)
        normTF1s.append(norm)
        sumxTF1s.append(x)
        sumx2TF1s.append(x2)

    for model in signals:
        norm = model.getTF1N()
        x = model.getTF1X()
        x2 = model.getTF1X2()
        norm.SetLineColor(ROOT.kBlue)
        x.SetLineColor(ROOT.kBlue)
        x2.SetLineColor(ROOT.kBlue)
        normTF1s.append(norm)
        sumxTF1s.append(x)
        sumx2TF1s.append(x2)

    canv2 = ROOT.TCanvas("canv2", "canv2")
    canv2.cd()
    normTF1s[0].Draw("")
    for tf1 in normTF1s[1:]:
        tf1.Draw("SAME")
    canv2.SaveAs("cat_opt_cdf.png")

    ### canv3 = ROOT.TCanvas("canv3","canv3")
    ### canv3.cd()
    ### hbkgMass.SetLineColor(ROOT.kRed)
    ### hbkgMass.Draw("hist")
    ### for hsigMass in hsigsMass:
    ###     hsigMass.SetLineColor(ROOT.kBlue)
    ###     hsigMass.Draw("hist SAME")
    ### canv3.SaveAs("cat_opt_mass.png")

    canv4 = ROOT.TCanvas("canv4", "canv4")
    canv4.cd()
    sumxTF1s[0].Draw("hist")
    for tf1 in sumxTF1s[1:]:
        tf1.Draw("hist SAME")
    canv4.SaveAs("cat_opt_sum_mass.png")

    canv5 = ROOT.TCanvas("canv5", "canv5")
    canv5.cd()
    sumx2TF1s[0].Draw("hist")
    for tf1 in sumx2TF1s[1:]:
        tf1.Draw("hist SAME")
    canv5.SaveAs("cat_opt_sum_mass2.png")

    objs.append(signals)
    objs.append(backgrounds)
    objs.append(normTF1s)
    objs.append(sumxTF1s)
    objs.append(sumx2TF1s)

    ### #########################################################################################################
    ### Figure of merit for optimization
    ###

    ### Simple counting
    ## fom       = ROOT.NaiveCutAndCountFomProvider()
    ## fom       = ROOT.PoissonCutAndCountFomProvider()

    ### ### Likelihood ratio using asymptotic approx.
    fom = ROOT.SimpleShapeFomProvider()
    for sigModel in signals:
        sigModel.getModel().setMu(mu)
    fom.addPOI(mu)
    fom.minStrategy(2)
    for constraint in sigconstr + bkgconstr:
        print constraint
        if type(constraint) == tuple:
            var, pdf, pars = constraint
            print var, pdf, pars
            fom.addNuisance(var, pdf)
        else:
            fom.addNuisance(constraint)

    ## fom.minimizer("Minuit2")
    ## fom.useRooSimultaneous()

    ### #########################################################################################################
    ### Run optimization
    ###

    for sigModel in signals:
        optimizer.addSignal(sigModel, True)
        ## optimizer.addSignal( sigModel )
    for bkgModel in backgrounds:
        optimizer.addBackground(bkgModel)
    optimizer.setFigureOfMerit(fom)

    summary = optmizeCats(optimizer, ws, varlist.getSize(), options.range, (
        cutoffs,
        options.dry,
        True,
    ), options.cont, options.reduce, options.refit)

    ### #########################################################################################################
    ### Some plots
    ###

    grS = ROOT.TGraph()
    grS.SetName("zVsNcat")
    grS.SetTitle(";n_{cat};f.o.m [A.U.]")
    for ncat, val in summary.iteritems():
        if (val["fom"] < 0.):
            grS.SetPoint(grS.GetN(), float(ncat), -val["fom"])
    grS.Sort()
    mincat = grS.GetX()[0]
    maxcat = grS.GetX()[grS.GetN() - 1]
    ncat = int(maxcat - mincat)

    for idim in range(ndim):
        var = varlist[idim]
        name = var.GetName()
        minX = var.getMin()
        maxX = var.getMax()
        nbinsX = var.getBinning().numBoundaries() - 1
        hbound = ROOT.TH2F("hbound_%s" % name, "hbound_%s" % name, nbinsX + 3,
                           minX - 1.5 * (maxX - minX) / nbinsX,
                           maxX + 1.5 * (maxX - minX) / nbinsX, ncat + 3,
                           mincat - 1.5, maxcat + 1.5)
        for jcat, val in summary.iteritems():
            nbound = int(jcat) + 1
            for ib in range(nbound):
                bd = float(val["boundaries"][ib + idim * nbound])
                ## hbound.Fill(float(jcat),bd)
                hbound.Fill(bd, float(jcat))
        cbound = ROOT.TCanvas("cat_opt_%s" % hbound.GetName(),
                              "cat_opt_%s" % hbound.GetName())
        cbound.cd()
        hbound.Draw("box")

        cbound_pj = ROOT.TCanvas("cat_opt_%s_pj" % hbound.GetName(),
                                 "cat_opt_%s_pj" % hbound.GetName())
        cbound_pj.cd()
        hbound_pj = hbound.Clone()
        ### hbound_pj = hbound.ProjectionY()
        ### hbound_pj.Draw()
        hbound_pj.Draw("box")
        hbound_pj.SetFillColor(ROOT.kBlack)
        hbound_pj.SetLineColor(ROOT.kBlack)
        objs.append(hbound)
        objs.append(hbound_pj)
        objs.append(cbound)
        objs.append(cbound_pj)
        maxy = 0.
        pdfs = []
        for sig in signals:
            pdf = sig.getPdf(idim)
            pdf.Scale(1. / pdf.Integral())
            pdf.SetLineColor(ROOT.kBlue)
            pdfs.append(pdf)
            maxy = max(maxy, pdf.GetMaximum())
            objs.append(pdf)
        for bkg in backgrounds:
            pdf = bkg.getPdf(idim)
            pdf.Scale(1. / pdf.Integral())
            pdf.SetLineColor(ROOT.kRed)
            pdfs.append(pdf)
            maxy = max(maxy, pdf.GetMaximum())
            objs.append(pdf)

        ### hbound_pj.Scale(maxy*hbound_pj.GetMaximum())
        ### hbound_pj.GetYaxis().SetRangeUser(0.,1.1*maxy)
        ### cbound_pj.RedrawAxis()
        ### cbound_pj.Update()
        for pdf in pdfs:
            pdf.Scale((ncat + 3.) / maxy)
            pdf.Draw("hist same")
        hbound_pj.GetYaxis().SetNdivisions(500 + ncat + 3)
        cbound_pj.SetGridy()
        hbound_pj.Draw("box same")

        for fmt in "png", "C":
            cbound.SaveAs("%s.%s" % (cbound.GetName(), fmt))
            cbound_pj.SaveAs("%s.%s" % (cbound_pj.GetName(), fmt))

    for isel in range(sellist.getSize()):
        var = sellist[isel]
        name = var.GetName()
        minX = var.getMin()
        maxX = var.getMax()
        nbinsX = var.getBinning().numBoundaries() - 1
        print var, name, minX, maxX, nbinsX
        hsel = ROOT.TH2F("hsel_%s" % name, "hsel_%s" % name, nbinsX + 3,
                         minX - 1.5 * (maxX - minX) / nbinsX,
                         maxX + 1.5 * (maxX - minX) / nbinsX, ncat + 3,
                         mincat - 1.5, maxcat + 1.5)
        for jcat, val in summary.iteritems():
            bd = float(val["selections"][isel])
            hsel.Fill(bd, float(jcat))
        csel = ROOT.TCanvas("cat_opt_%s" % hsel.GetName(),
                            "cat_opt_%s" % hsel.GetName())
        csel.cd()
        hsel.Draw("box")

        csel_pj = ROOT.TCanvas("cat_opt_%s_pj" % hsel.GetName(),
                               "cat_opt_%s_pj" % hsel.GetName())
        csel_pj.cd()
        hsel_pj = hsel.Clone()
        ### hsel_pj = hsel.ProjectionY()
        ### hsel_pj.Draw()
        hsel_pj.Draw("box")
        hsel_pj.SetFillColor(ROOT.kBlack)
        hsel_pj.SetLineColor(ROOT.kBlack)
        objs.append(hsel)
        objs.append(hsel_pj)
        objs.append(csel)
        objs.append(csel_pj)
        maxy = 0.
        pdfs = []
        for sig in signals:
            pdf = sig.getPdf(ndim + isel)
            pdf.SetLineColor(ROOT.kBlue)
            pdf.Scale(1. / pdf.Integral())
            pdfs.append(pdf)
            maxy = max(maxy, pdf.GetMaximum())
            objs.append(pdf)
        for bkg in backgrounds:
            pdf = bkg.getPdf(ndim + isel)
            pdf.Scale(1. / pdf.Integral())
            pdf.SetLineColor(ROOT.kRed)
            pdfs.append(pdf)
            maxy = max(maxy, pdf.GetMaximum())
            objs.append(pdf)

        for pdf in pdfs:
            pdf.Scale((ncat + 1.5) / maxy / pdf.Integral())
            pdf.Draw("l same")
        hsel_pj.GetYaxis().SetNdivisions(500 + ncat + 3)
        csel_pj.SetGridy()
        hsel_pj.Draw("box same")

        for fmt in "png", "C":
            csel.SaveAs("%s.%s" % (csel.GetName(), fmt))
            csel_pj.SaveAs("%s.%s" % (csel.GetName(), fmt))

    canv9 = ROOT.TCanvas("canv9", "canv9")
    canv9.SetGridx()
    canv9.SetGridy()
    canv9.cd()
    grS.SetMarkerStyle(ROOT.kFullCircle)
    grS.Draw("AP")

    canv9.SaveAs("cat_opt_fom.png")
    canv9.SaveAs("cat_opt_fom.C")

    ## tmp.Close()
    return ws