def backup(SD_NAME=None, session=None, erase=False): if not os.path.exists('raw'): os.makedirs('raw') SD_NAME = 'test' SD_ROOT = '/media/' + os.getusername() + '/' + SD_NAME DATE = time.strftime('%Y-%m-%d_%H-%M-%S') if not session: BACKUP_DIR = 'video_' + DATE else: BACKUP_DIR = session os.chdir('raw') if not os.path.exists(BACKUP_DIR): os.makedirs(BACKUP_DIR) os.chdir(BACKUP_DIR) command = [ 'rsync', '-a', '-r', '--stats', '--progress', SD_ROOT + '/PRIVATE/AVCHD/BDMV/STREAM', '.' ] if subprocess.check_call(command) == 0: if erase: shutil.rmtree(SD_ROOT) print("Done") else: print("Error while running rsync") for filename in glob.glob('00*.MTS'): os.rename(filename, DATE + filename) os.chdir('../..')
def postInstall(fromVersion, fromRelease, toVersion, toRelease): for _file, perms in permissions.items(): # The list above is general, some paths may not exist depending on the configuration if os.path.exists(_file): os.system("/bin/chown -R %s %s" % (perms[1], _file)) os.system("/bin/chmod %s %s" % (perms[0], _file)) os.system("groupadd libvirt") os.system("usermod -G libvirt %s" % os.getusername())
def postInstall(fromVersion, fromRelease, toVersion, toRelease): try: os.system("/usr/sbin/groupadd %s" % OUR_NAME) os.system("usermod -a -G docker %s" % os.getusername()) os.system("/bin/touch %s" % logfile) os.system("/bin/chown root:docker %s" % logfile) os.system("chmod 0644 %s" % logfile) except: pass
def prepare_transmit( self, job, resume ): # NOW get the data self.file_to_send = job.get_attr( iftfile.JOB_ATTR_SRC_NAME ) self.remote_host = job.get_attr( iftfile.JOB_ATTR_DEST_HOST ) self.remote_path = job.get_attr( iftfile.JOB_ATTR_DEST_NAME ) self.remote_user = job.get_attr( IFTSCP_REMOTE_LOGIN ) if self.remote_user == None: self.remote_user = os.getusername() return 0
def postInstall(fromVersion, fromRelease, toVersion, toRelease): try: os.system("usr/sbin/groupadd -g %d %s" % (OUR_ID, OUR_NAME)) os.system("usermod -a -G docker %s" % os.getusername()) os.system("/bin/touch %s" % logfile) os.system("/bin/chown root:docker %s" % logfile) os.system("chmod 0644 %s" % logfile) os.system("rc-update add docker default") except: pass
def copyConfigAndConnect(hosts): print( "Connecting to remote hosts and copying config files, tapes and snapshots" ) #create a list of files to be copied across all the remote hosts file_list = [] for h in hosts: for r in h['runs']: if (r.isFromSnapshot): file_list.append(r.snapshotFile) else: file_list.append(r.tapeFilename) file_list.append(main.__file__) for host in hosts: if (host['name'] != socket.gethostname() ): #if I am not the computer named in host name try: username = host['username'] except: username = os.getusername( ) #default username is current user user's name try: port = host['port'] except: port = 22 #default ssh port rm = Remote.Connection(hostname=host['address'], username=username, port=port) rm.connect() # print ("Sendind file:"+main.__file__) if ('remotebasepath' in host): remote_dir = host['remotebasepath'] else: remote_dir = 'trisurf_simulations' rm.send_multiple_files_in_directory(file_list, remote_dir) # rm.send_file(main.__file__,'remote_control.py') # for run in host['runs']: # try: # rm.send_file(run.tapeFile,run.tapeFile) # except: # pass # try: # rm.send_file(run.snapshotFile,run.snapshotFile) # except: # pass host['_conn'] = rm # we are connected to all hosts... return hosts
def get_users() -> list: if not requirement.are_presents(['win32net', 'win32netcon']): if requirement.is_present('os'): return [os.getusername()] else: return [] filter = win32netcon.FILTER_NORMAL_ACCOUNT resume_handle = 0 user_list = [] while True: result = win32net.NetUserEnum(None, 0, filter, resume_handle) user_list += [user['name'] for user in result[0]] resume_handle = result[2] if not resume_handle: break user_list.sort() return user_list
def callback_notes(self): notes_path = os.path.join(self.sim_dir, "notes.txt") if os.path.isfile(notes_path) == False: out = open(notes_path, 'w') out.write("Notes on the simulation: " + os.path.basenam(self.sim_dir) + "\n") out.write("Date: " + datetime.datetime.today().strftime('%Y-%m-%d') + "\n") out.write("Generated by: " + os.getusername() + "\n") out.write("\n") out.close() else: out = open(notes_path, 'a') out.write("\n" + datetime.datetime.today().strftime('%Y-%m-%d') + ":\n") out.close() desktop_open(notes_path)
def __openConnection(self): try: self.client_pre.load_system_host_keys() self.client_pre.set_missing_host_key_policy( paramiko.WarningPolicy()) if not self.username: self.username = os.getusername() self.client_pre.connect(self.hostname, port=self.port, username=self.username, password=self.password, allow_agent=False, look_for_keys=False) self.client = self.client_pre.invoke_shell() output = self.client.recv(65535) print(output) finally: print(" Connection to {host} successfully set up. ".format( host=self.hostname).center(80, '*'))
def backup(SD_NAME=None, session=None, erase=False): if not os.path.exists('raw'): os.makedirs('raw') SD_NAME = 'test' SD_ROOT = '/media/' + os.getusername() + '/' + SD_NAME DATE = time.strftime('%Y-%m-%d_%H-%M-%S') if not session: BACKUP_DIR = 'video_' + DATE else: BACKUP_DIR = session os.chdir('raw') if not os.path.exists(BACKUP_DIR): os.makedirs(BACKUP_DIR) os.chdir(BACKUP_DIR) command = ['rsync', '-a', '-r', '--stats', '--progress', SD_ROOT+'/PRIVATE/AVCHD/BDMV/STREAM', '.'] if subprocess.check_call(command) == 0: if erase: shutil.rmtree(SD_ROOT) print("Done") else: print("Error while running rsync") for filename in glob.glob('00*.MTS'): os.rename(filename,DATE+filename) os.chdir('../..')
def from_assigned_task(self, assigned_task, sandbox): """ :param assigned_task: :param sandbox: :return: Instance of a HealthChecker. """ mesos_task = mesos_task_instance_from_assigned_task(assigned_task) health_check_config = mesos_task.health_check_config().get() health_checker = health_check_config.get('health_checker', {}) timeout_secs = health_check_config.get('timeout_secs') if SHELL_HEALTH_CHECK in health_checker: shell_command = health_checker.get(SHELL_HEALTH_CHECK, {}).get('shell_command') # Filling in variables e.g. thermos.ports[http] that could have been passed in as part of # shell_command. interpolated_command = HealthCheckerProvider.interpolate_cmd( task=assigned_task, cmd=shell_command) # If we do not want the health check to execute as the user from the job's role # --nosetuid-health-checks should be passed as an argument to the executor. demote_to_job_role_user = None if not self._nosetuid_health_checks and not sandbox.is_filesystem_image: pw_entry = pwd.getpwnam(assigned_task.task.job.role) def demote_to_job_role_user(): os.setgid(pw_entry.pw_gid) os.setuid(pw_entry.pw_uid) # If the task is executing in an isolated filesystem we'll want to wrap the health check # command within a mesos-containerizer invocation so that it's executed within that # filesystem. wrapper = None if sandbox.is_filesystem_image: health_check_user = (os.getusername() if self._nosetuid_health_checks else assigned_task.task.job.role) def wrapper(cmd): return wrap_with_mesos_containerizer( cmd, health_check_user, sandbox.container_root, self._mesos_containerizer_path) shell_signaler = ShellHealthCheck( cmd=interpolated_command, preexec_fn=demote_to_job_role_user, timeout_secs=timeout_secs, wrapper_fn=wrapper) a_health_checker = lambda: shell_signaler() else: portmap = resolve_ports(mesos_task, assigned_task.assignedPorts) if 'health' not in portmap: return None http_config = health_checker.get(HTTP_HEALTH_CHECK, {}) http_endpoint = http_config.get('endpoint') http_expected_response = http_config.get('expected_response') http_expected_response_code = http_config.get('expected_response_code') http_signaler = HttpSignaler( portmap['health'], timeout_secs=timeout_secs) a_health_checker = lambda: http_signaler( endpoint=http_endpoint, expected_response=http_expected_response, expected_response_code=http_expected_response_code ) health_checker = HealthChecker( a_health_checker, sandbox, interval_secs=health_check_config.get('interval_secs'), initial_interval_secs=health_check_config.get('initial_interval_secs'), max_consecutive_failures=health_check_config.get('max_consecutive_failures')) return health_checker
def __init__ (self, host, port, user, directory, options): extra = shlex.split (options.get ('extra', '')) remote = str ((user or os.getusername ()) + '@' + (host or 'localhost')) self.directory = directory self.tunnel = ['ssh', '-T', '-p', str (port or 22)] + extra + [remote]
# Print out the command line arguments in sys.argv, one per line: import sys a = sys.argv[1] b = sys.argv[2] sum = a + b # print("sum of a and b is {}".format(int(a)+ int(b))) # Print out the OS platform you're using: print(sys.platform) # Print out the version of Python you're using: print(sys.version) import os # See the docs for the OS module: https://docs.python.org/3.7/library/os.html os.name # Print the current process ID print(os.getpid()) # Print the current working directory (cwd): print(os.getcwd()) # get current working directory print(os.listdir(".")) # files in the current direcotry # Print out your machine's login name os.getusername() # The above didn't work so I used the followinf import instead import getpass getpass.getuser()
def optimizeMultiDim(options,args): signals = options.signals backgrounds = options.backgrounds variables = options.variables observable = options.observable selection = options.selection cutoff = options.cutoff selectioncuts = options.selectioncuts ndim = len(variables) ws = None cutoffs = numpy.array([cutoff]*ndim) obs,obsalias = defineVariables( [observable], options.label ) obs = obs[0] mu = ROOT.RooRealVar("mu","mu",1.,0.,10.) varlist,aliases = defineVariables( variables, options.label ) sellist,selaliases = defineVariables( selectioncuts, options.label ) print "---------------------------------------------" print "Observables " obs.Print("") print print "Variables" varlist.Print("V") print print "Selection cuts" sellist.Print("V") aliases.extend(obsalias+selaliases) print print "Aliases" pprint(aliases) print if options.infile == "": options.infile = args[0] fin = ROOT.TFile.Open(options.infile) if options.cont: if os.path.exists(os.path.abspath(options.cont)): options.cont = os.path.abspath(options.cont) else: options.cont = os.path.abspath(os.path.join(os.path.dirname(options.outdir),options.cont)) if not os.path.exists(options.outdir): os.mkdir(options.outdir) os.chdir(options.outdir) if not os.path.exists("/tmp/%s" % os.getusername()): os.mkdir("/tmp/%s" % os.getusername()) tmp = ROOT.TFile.Open("/tmp/%s/categoryOptimizationMultiDim.root" % os.getusername() ,"recreate") tmp.cd() ### ########################################################################################################## ### Minimizer and optimizer ### minimizer = ROOT.TMinuitMinimizer("Minimize") ## minimizer.SetPrintLevel(999) ## minimizer = ROOT.Minuit2.Minuit2Minimizer() optimizer = ROOT.CategoryOptimizer( minimizer, ndim ) for isel in range(sellist.getSize()): sel = sellist[isel] if sel.GetName() in options.fix: optimizer.addFixedOrthoCut(sel.GetName(),sel.getVal()) else: optimizer.addFloatingOrthoCut(sel.GetName(),sel.getVal(),(sel.getMax()-sel.getMin())/sel.getBins(),sel.getMin(),sel.getMax()) optimizer.absoluteBoundaries() ## Float absolut boundaries instead of telescopic ones for opt in options.settings: if isinstance(opt, types.StringTypes): getattr(optimizer,opt)() else: name, args = opt getattr(optimizer,name)(*args) ### ########################################################################################################## ### Model builders ### print "---------------------------------------------" print "Reading inputs" print sigTrees = [ mergeTrees(fin,selection,name,trees,aliases) for name,trees in signals.iteritems() ] bkgTrees = [ mergeTrees(fin,selection,name,trees,aliases) for name,trees in backgrounds.iteritems() ] for tree in sigTrees+bkgTrees: tree.Write() fin.Close() print "---------------------------------------------" print "Booking model buildes" print signals,sigconstr = modelBuilders( sigTrees, ROOT.AbsModel.sig, obs, varlist, sellist, getattr(options,"weights",{}), getattr(options,"shapes",{}), {}, {} ) backgrounds,bkgconstr = modelBuilders( bkgTrees, ROOT.AbsModel.bkg, obs, varlist, sellist, getattr(options,"weights",{}), getattr(options,"shapes",{}), getattr(options,"minevents",{},), getattr(options,"constrained",{}), ) if options.saveCompactTree: for model in signals+backgrounds: model.getTree().Write() normTF1s = [] sumxTF1s = [] sumx2TF1s = [] for model in backgrounds: norm = model.getTF1N() x = model.getTF1X() x2 = model.getTF1X2() norm.SetLineColor(ROOT.kRed) x.SetLineColor(ROOT.kRed) x2.SetLineColor(ROOT.kRed) normTF1s.append(norm) sumxTF1s.append(x) sumx2TF1s.append(x2) for model in signals: norm = model.getTF1N() x = model.getTF1X() x2 = model.getTF1X2() norm.SetLineColor(ROOT.kBlue) x.SetLineColor(ROOT.kBlue) x2.SetLineColor(ROOT.kBlue) normTF1s.append(norm) sumxTF1s.append(x) sumx2TF1s.append(x2) canv2 = ROOT.TCanvas("canv2","canv2") canv2.cd() normTF1s[0].Draw("") for tf1 in normTF1s[1:]: tf1.Draw("SAME") canv2.SaveAs("cat_opt_cdf.png") ### canv3 = ROOT.TCanvas("canv3","canv3") ### canv3.cd() ### hbkgMass.SetLineColor(ROOT.kRed) ### hbkgMass.Draw("hist") ### for hsigMass in hsigsMass: ### hsigMass.SetLineColor(ROOT.kBlue) ### hsigMass.Draw("hist SAME") ### canv3.SaveAs("cat_opt_mass.png") canv4 = ROOT.TCanvas("canv4","canv4") canv4.cd() sumxTF1s[0].Draw("hist") for tf1 in sumxTF1s[1:]: tf1.Draw("hist SAME") canv4.SaveAs("cat_opt_sum_mass.png") canv5 = ROOT.TCanvas("canv5","canv5") canv5.cd() sumx2TF1s[0].Draw("hist") for tf1 in sumx2TF1s[1:]: tf1.Draw("hist SAME") canv5.SaveAs("cat_opt_sum_mass2.png") objs.append( signals ) objs.append( backgrounds ) objs.append( normTF1s ) objs.append( sumxTF1s ) objs.append( sumx2TF1s ) ### ######################################################################################################### ### Figure of merit for optimization ### ### Simple counting ## fom = ROOT.NaiveCutAndCountFomProvider() ## fom = ROOT.PoissonCutAndCountFomProvider() ### ### Likelihood ratio using asymptotic approx. fom = ROOT.SimpleShapeFomProvider() for sigModel in signals: sigModel.getModel().setMu(mu) fom.addPOI(mu) fom.minStrategy(2) for constraint in sigconstr+bkgconstr: print constraint if type(constraint) == tuple: var, pdf, pars = constraint print var,pdf,pars fom.addNuisance(var,pdf) else: fom.addNuisance(constraint) ## fom.minimizer("Minuit2") ## fom.useRooSimultaneous() ### ######################################################################################################### ### Run optimization ### for sigModel in signals: optimizer.addSignal( sigModel, True ) ## optimizer.addSignal( sigModel ) for bkgModel in backgrounds: optimizer.addBackground( bkgModel ) optimizer.setFigureOfMerit( fom ) summary = optmizeCats( optimizer, ws, varlist.getSize(), options.range, (cutoffs,options.dry,True,), options.cont, options.reduce, options.refit ) ### ######################################################################################################### ### Some plots ### grS = ROOT.TGraph() grS.SetName("zVsNcat") grS.SetTitle(";n_{cat};f.o.m [A.U.]") for ncat,val in summary.iteritems(): if( val["fom"] < 0. ) : grS.SetPoint( grS.GetN(), float(ncat), -val["fom"] ) grS.Sort() mincat = grS.GetX()[0] maxcat = grS.GetX()[grS.GetN()-1] ncat = int(maxcat - mincat) for idim in range(ndim): var = varlist[idim] name = var.GetName() minX = var.getMin() maxX = var.getMax() nbinsX = var.getBinning().numBoundaries()-1 hbound = ROOT.TH2F("hbound_%s" % name,"hbound_%s" % name,nbinsX+3,minX-1.5*(maxX-minX)/nbinsX,maxX+1.5*(maxX-minX)/nbinsX,ncat+3,mincat-1.5,maxcat+1.5) for jcat,val in summary.iteritems(): nbound = int(jcat)+1 for ib in range(nbound): bd = float(val["boundaries"][ib+idim*nbound]) ## hbound.Fill(float(jcat),bd) hbound.Fill(bd,float(jcat)) cbound = ROOT.TCanvas( "cat_opt_%s" % hbound.GetName(), "cat_opt_%s" % hbound.GetName() ) cbound.cd() hbound.Draw("box") cbound_pj = ROOT.TCanvas( "cat_opt_%s_pj" % hbound.GetName(), "cat_opt_%s_pj" % hbound.GetName() ) cbound_pj.cd() hbound_pj = hbound.Clone() ### hbound_pj = hbound.ProjectionY() ### hbound_pj.Draw() hbound_pj.Draw("box") hbound_pj.SetFillColor(ROOT.kBlack) hbound_pj.SetLineColor(ROOT.kBlack) objs.append(hbound) objs.append(hbound_pj) objs.append(cbound) objs.append(cbound_pj) maxy = 0. pdfs = [] for sig in signals: pdf = sig.getPdf(idim) pdf.Scale(1./pdf.Integral()) pdf.SetLineColor(ROOT.kBlue) pdfs.append(pdf) maxy = max(maxy,pdf.GetMaximum()) objs.append(pdf) for bkg in backgrounds: pdf = bkg.getPdf(idim) pdf.Scale(1./pdf.Integral()) pdf.SetLineColor(ROOT.kRed) pdfs.append(pdf) maxy = max(maxy,pdf.GetMaximum()) objs.append(pdf) ### hbound_pj.Scale(maxy*hbound_pj.GetMaximum()) ### hbound_pj.GetYaxis().SetRangeUser(0.,1.1*maxy) ### cbound_pj.RedrawAxis() ### cbound_pj.Update() for pdf in pdfs: pdf.Scale( (ncat+3.)/maxy ) pdf.Draw("hist same") hbound_pj.GetYaxis().SetNdivisions(500+ncat+3) cbound_pj.SetGridy() hbound_pj.Draw("box same") for fmt in "png", "C": cbound.SaveAs("%s.%s" % (cbound.GetName(),fmt) ) cbound_pj.SaveAs("%s.%s" % (cbound_pj.GetName(),fmt) ) for isel in range(sellist.getSize()): var = sellist[isel] name = var.GetName() minX = var.getMin() maxX = var.getMax() nbinsX = var.getBinning().numBoundaries()-1 print var, name, minX, maxX, nbinsX hsel = ROOT.TH2F("hsel_%s" % name,"hsel_%s" % name,nbinsX+3,minX-1.5*(maxX-minX)/nbinsX,maxX+1.5*(maxX-minX)/nbinsX,ncat+3,mincat-1.5,maxcat+1.5) for jcat,val in summary.iteritems(): bd = float(val["selections"][isel]) hsel.Fill(bd,float(jcat)) csel = ROOT.TCanvas( "cat_opt_%s" % hsel.GetName(), "cat_opt_%s" % hsel.GetName() ) csel.cd() hsel.Draw("box") csel_pj = ROOT.TCanvas( "cat_opt_%s_pj" % hsel.GetName(), "cat_opt_%s_pj" % hsel.GetName() ) csel_pj.cd() hsel_pj = hsel.Clone() ### hsel_pj = hsel.ProjectionY() ### hsel_pj.Draw() hsel_pj.Draw("box") hsel_pj.SetFillColor(ROOT.kBlack) hsel_pj.SetLineColor(ROOT.kBlack) objs.append(hsel) objs.append(hsel_pj) objs.append(csel) objs.append(csel_pj) maxy = 0. pdfs = [] for sig in signals: pdf = sig.getPdf(ndim+isel) pdf.SetLineColor(ROOT.kBlue) pdf.Scale(1./pdf.Integral()) pdfs.append(pdf) maxy = max(maxy,pdf.GetMaximum()) objs.append(pdf) for bkg in backgrounds: pdf = bkg.getPdf(ndim+isel) pdf.Scale(1./pdf.Integral()) pdf.SetLineColor(ROOT.kRed) pdfs.append(pdf) maxy = max(maxy,pdf.GetMaximum()) objs.append(pdf) for pdf in pdfs: pdf.Scale( (ncat+1.5)/maxy/pdf.Integral() ) pdf.Draw("l same") hsel_pj.GetYaxis().SetNdivisions(500+ncat+3) csel_pj.SetGridy() hsel_pj.Draw("box same") for fmt in "png", "C": csel.SaveAs("%s.%s" % (csel.GetName(),fmt) ) csel_pj.SaveAs("%s.%s" % (csel.GetName(),fmt) ) canv9 = ROOT.TCanvas("canv9","canv9") canv9.SetGridx() canv9.SetGridy() canv9.cd() grS.SetMarkerStyle(ROOT.kFullCircle) grS.Draw("AP") canv9.SaveAs("cat_opt_fom.png") canv9.SaveAs("cat_opt_fom.C") ## tmp.Close() return ws
#!/usr/bin/python import stl import socket import argparse import os username = os.getusername() personal_log = '/home/' + username +'/.stl/personal-stats-' + socket.gethostname() + '.log' work_log = '/home/' + username + '/.stl/work-stats-' + socket.gethostname() + '.log' projects = { 'stl' : { 'path' : '/home/' + username + '/projects/stl/docs/source/', 'target' : 'projects', 'emacs' : True, 'quiet' : True, 'log' : personal_log, 'ext': 'txt' }, } def set_quiet(force, project_quiet): if force is True: quiet = False else: quiet = project_quiet return quiet def main():
def __init__(self, host, port, user, directory, options): extra = shlex.split(options.get('extra', '')) remote = str((user or os.getusername()) + '@' + (host or 'localhost')) self.directory = directory self.tunnel = ['ssh', '-T', '-p', str(port or 22)] + extra + [remote]
def from_assigned_task(self, assigned_task, sandbox): """ :param assigned_task: :param sandbox: :return: Instance of a HealthChecker. """ mesos_task = mesos_task_instance_from_assigned_task(assigned_task) health_check_config = mesos_task.health_check_config().get() health_checker = health_check_config.get('health_checker', {}) timeout_secs = health_check_config.get('timeout_secs') if SHELL_HEALTH_CHECK in health_checker: shell_command = health_checker.get(SHELL_HEALTH_CHECK, {}).get('shell_command') # Filling in variables e.g. thermos.ports[http] that could have been passed in as part of # shell_command. interpolated_command = HealthCheckerProvider.interpolate_cmd( task=assigned_task, cmd=shell_command) # If we do not want the health check to execute as the user from the job's role # --nosetuid-health-checks should be passed as an argument to the executor. demote_to_job_role_user = None if not self._nosetuid_health_checks and not sandbox.is_filesystem_image: pw_entry = pwd.getpwnam(assigned_task.task.job.role) def demote_to_job_role_user(): os.setgid(pw_entry.pw_gid) os.setuid(pw_entry.pw_uid) # If the task is executing in an isolated filesystem we'll want to wrap the health check # command within a mesos-containerizer invocation so that it's executed within that # filesystem. wrapper = None if sandbox.is_filesystem_image: health_check_user = (os.getusername() if self._nosetuid_health_checks else assigned_task.task.job.role) def wrapper(cmd): return wrap_with_mesos_containerizer( cmd, health_check_user, sandbox.container_root, self._mesos_containerizer_path) shell_signaler = ShellHealthCheck( cmd=interpolated_command, preexec_fn=demote_to_job_role_user, timeout_secs=timeout_secs, wrapper_fn=wrapper) a_health_checker = lambda: shell_signaler() else: portmap = resolve_ports(mesos_task, assigned_task.assignedPorts) if 'health' not in portmap: log.warning('No health-checks defined, will use a no-op health-checker.') return NoopHealthChecker() http_config = health_checker.get(HTTP_HEALTH_CHECK, {}) http_endpoint = http_config.get('endpoint') http_expected_response = http_config.get('expected_response') http_expected_response_code = http_config.get('expected_response_code') http_signaler = HttpSignaler( portmap['health'], timeout_secs=timeout_secs) a_health_checker = lambda: http_signaler( endpoint=http_endpoint, expected_response=http_expected_response, expected_response_code=http_expected_response_code ) health_checker = HealthChecker( a_health_checker, sandbox, interval_secs=health_check_config.get('interval_secs'), grace_period_secs=health_check_config.get('initial_interval_secs'), max_consecutive_failures=health_check_config.get('max_consecutive_failures'), min_consecutive_successes=health_check_config.get('min_consecutive_successes')) return health_checker
def optimizeMultiDim(options, args): signals = options.signals backgrounds = options.backgrounds variables = options.variables observable = options.observable selection = options.selection cutoff = options.cutoff selectioncuts = options.selectioncuts ndim = len(variables) ws = None cutoffs = numpy.array([cutoff] * ndim) obs, obsalias = defineVariables([observable], options.label) obs = obs[0] mu = ROOT.RooRealVar("mu", "mu", 1., 0., 10.) varlist, aliases = defineVariables(variables, options.label) sellist, selaliases = defineVariables(selectioncuts, options.label) print "---------------------------------------------" print "Observables " obs.Print("") print print "Variables" varlist.Print("V") print print "Selection cuts" sellist.Print("V") aliases.extend(obsalias + selaliases) print print "Aliases" pprint(aliases) print if options.infile == "": options.infile = args[0] fin = ROOT.TFile.Open(options.infile) if options.cont: if os.path.exists(os.path.abspath(options.cont)): options.cont = os.path.abspath(options.cont) else: options.cont = os.path.abspath( os.path.join(os.path.dirname(options.outdir), options.cont)) if not os.path.exists(options.outdir): os.mkdir(options.outdir) os.chdir(options.outdir) if not os.path.exists("/tmp/%s" % os.getusername()): os.mkdir("/tmp/%s" % os.getusername()) tmp = ROOT.TFile.Open( "/tmp/%s/categoryOptimizationMultiDim.root" % os.getusername(), "recreate") tmp.cd() ### ########################################################################################################## ### Minimizer and optimizer ### minimizer = ROOT.TMinuitMinimizer("Minimize") ## minimizer.SetPrintLevel(999) ## minimizer = ROOT.Minuit2.Minuit2Minimizer() optimizer = ROOT.CategoryOptimizer(minimizer, ndim) for isel in range(sellist.getSize()): sel = sellist[isel] if sel.GetName() in options.fix: optimizer.addFixedOrthoCut(sel.GetName(), sel.getVal()) else: optimizer.addFloatingOrthoCut(sel.GetName(), sel.getVal(), (sel.getMax() - sel.getMin()) / sel.getBins(), sel.getMin(), sel.getMax()) optimizer.absoluteBoundaries( ) ## Float absolut boundaries instead of telescopic ones for opt in options.settings: if isinstance(opt, types.StringTypes): getattr(optimizer, opt)() else: name, args = opt getattr(optimizer, name)(*args) ### ########################################################################################################## ### Model builders ### print "---------------------------------------------" print "Reading inputs" print sigTrees = [ mergeTrees(fin, selection, name, trees, aliases) for name, trees in signals.iteritems() ] bkgTrees = [ mergeTrees(fin, selection, name, trees, aliases) for name, trees in backgrounds.iteritems() ] for tree in sigTrees + bkgTrees: tree.Write() fin.Close() print "---------------------------------------------" print "Booking model buildes" print signals, sigconstr = modelBuilders(sigTrees, ROOT.AbsModel.sig, obs, varlist, sellist, getattr(options, "weights", {}), getattr(options, "shapes", {}), {}, {}) backgrounds, bkgconstr = modelBuilders( bkgTrees, ROOT.AbsModel.bkg, obs, varlist, sellist, getattr(options, "weights", {}), getattr(options, "shapes", {}), getattr( options, "minevents", {}, ), getattr(options, "constrained", {}), ) if options.saveCompactTree: for model in signals + backgrounds: model.getTree().Write() normTF1s = [] sumxTF1s = [] sumx2TF1s = [] for model in backgrounds: norm = model.getTF1N() x = model.getTF1X() x2 = model.getTF1X2() norm.SetLineColor(ROOT.kRed) x.SetLineColor(ROOT.kRed) x2.SetLineColor(ROOT.kRed) normTF1s.append(norm) sumxTF1s.append(x) sumx2TF1s.append(x2) for model in signals: norm = model.getTF1N() x = model.getTF1X() x2 = model.getTF1X2() norm.SetLineColor(ROOT.kBlue) x.SetLineColor(ROOT.kBlue) x2.SetLineColor(ROOT.kBlue) normTF1s.append(norm) sumxTF1s.append(x) sumx2TF1s.append(x2) canv2 = ROOT.TCanvas("canv2", "canv2") canv2.cd() normTF1s[0].Draw("") for tf1 in normTF1s[1:]: tf1.Draw("SAME") canv2.SaveAs("cat_opt_cdf.png") ### canv3 = ROOT.TCanvas("canv3","canv3") ### canv3.cd() ### hbkgMass.SetLineColor(ROOT.kRed) ### hbkgMass.Draw("hist") ### for hsigMass in hsigsMass: ### hsigMass.SetLineColor(ROOT.kBlue) ### hsigMass.Draw("hist SAME") ### canv3.SaveAs("cat_opt_mass.png") canv4 = ROOT.TCanvas("canv4", "canv4") canv4.cd() sumxTF1s[0].Draw("hist") for tf1 in sumxTF1s[1:]: tf1.Draw("hist SAME") canv4.SaveAs("cat_opt_sum_mass.png") canv5 = ROOT.TCanvas("canv5", "canv5") canv5.cd() sumx2TF1s[0].Draw("hist") for tf1 in sumx2TF1s[1:]: tf1.Draw("hist SAME") canv5.SaveAs("cat_opt_sum_mass2.png") objs.append(signals) objs.append(backgrounds) objs.append(normTF1s) objs.append(sumxTF1s) objs.append(sumx2TF1s) ### ######################################################################################################### ### Figure of merit for optimization ### ### Simple counting ## fom = ROOT.NaiveCutAndCountFomProvider() ## fom = ROOT.PoissonCutAndCountFomProvider() ### ### Likelihood ratio using asymptotic approx. fom = ROOT.SimpleShapeFomProvider() for sigModel in signals: sigModel.getModel().setMu(mu) fom.addPOI(mu) fom.minStrategy(2) for constraint in sigconstr + bkgconstr: print constraint if type(constraint) == tuple: var, pdf, pars = constraint print var, pdf, pars fom.addNuisance(var, pdf) else: fom.addNuisance(constraint) ## fom.minimizer("Minuit2") ## fom.useRooSimultaneous() ### ######################################################################################################### ### Run optimization ### for sigModel in signals: optimizer.addSignal(sigModel, True) ## optimizer.addSignal( sigModel ) for bkgModel in backgrounds: optimizer.addBackground(bkgModel) optimizer.setFigureOfMerit(fom) summary = optmizeCats(optimizer, ws, varlist.getSize(), options.range, ( cutoffs, options.dry, True, ), options.cont, options.reduce, options.refit) ### ######################################################################################################### ### Some plots ### grS = ROOT.TGraph() grS.SetName("zVsNcat") grS.SetTitle(";n_{cat};f.o.m [A.U.]") for ncat, val in summary.iteritems(): if (val["fom"] < 0.): grS.SetPoint(grS.GetN(), float(ncat), -val["fom"]) grS.Sort() mincat = grS.GetX()[0] maxcat = grS.GetX()[grS.GetN() - 1] ncat = int(maxcat - mincat) for idim in range(ndim): var = varlist[idim] name = var.GetName() minX = var.getMin() maxX = var.getMax() nbinsX = var.getBinning().numBoundaries() - 1 hbound = ROOT.TH2F("hbound_%s" % name, "hbound_%s" % name, nbinsX + 3, minX - 1.5 * (maxX - minX) / nbinsX, maxX + 1.5 * (maxX - minX) / nbinsX, ncat + 3, mincat - 1.5, maxcat + 1.5) for jcat, val in summary.iteritems(): nbound = int(jcat) + 1 for ib in range(nbound): bd = float(val["boundaries"][ib + idim * nbound]) ## hbound.Fill(float(jcat),bd) hbound.Fill(bd, float(jcat)) cbound = ROOT.TCanvas("cat_opt_%s" % hbound.GetName(), "cat_opt_%s" % hbound.GetName()) cbound.cd() hbound.Draw("box") cbound_pj = ROOT.TCanvas("cat_opt_%s_pj" % hbound.GetName(), "cat_opt_%s_pj" % hbound.GetName()) cbound_pj.cd() hbound_pj = hbound.Clone() ### hbound_pj = hbound.ProjectionY() ### hbound_pj.Draw() hbound_pj.Draw("box") hbound_pj.SetFillColor(ROOT.kBlack) hbound_pj.SetLineColor(ROOT.kBlack) objs.append(hbound) objs.append(hbound_pj) objs.append(cbound) objs.append(cbound_pj) maxy = 0. pdfs = [] for sig in signals: pdf = sig.getPdf(idim) pdf.Scale(1. / pdf.Integral()) pdf.SetLineColor(ROOT.kBlue) pdfs.append(pdf) maxy = max(maxy, pdf.GetMaximum()) objs.append(pdf) for bkg in backgrounds: pdf = bkg.getPdf(idim) pdf.Scale(1. / pdf.Integral()) pdf.SetLineColor(ROOT.kRed) pdfs.append(pdf) maxy = max(maxy, pdf.GetMaximum()) objs.append(pdf) ### hbound_pj.Scale(maxy*hbound_pj.GetMaximum()) ### hbound_pj.GetYaxis().SetRangeUser(0.,1.1*maxy) ### cbound_pj.RedrawAxis() ### cbound_pj.Update() for pdf in pdfs: pdf.Scale((ncat + 3.) / maxy) pdf.Draw("hist same") hbound_pj.GetYaxis().SetNdivisions(500 + ncat + 3) cbound_pj.SetGridy() hbound_pj.Draw("box same") for fmt in "png", "C": cbound.SaveAs("%s.%s" % (cbound.GetName(), fmt)) cbound_pj.SaveAs("%s.%s" % (cbound_pj.GetName(), fmt)) for isel in range(sellist.getSize()): var = sellist[isel] name = var.GetName() minX = var.getMin() maxX = var.getMax() nbinsX = var.getBinning().numBoundaries() - 1 print var, name, minX, maxX, nbinsX hsel = ROOT.TH2F("hsel_%s" % name, "hsel_%s" % name, nbinsX + 3, minX - 1.5 * (maxX - minX) / nbinsX, maxX + 1.5 * (maxX - minX) / nbinsX, ncat + 3, mincat - 1.5, maxcat + 1.5) for jcat, val in summary.iteritems(): bd = float(val["selections"][isel]) hsel.Fill(bd, float(jcat)) csel = ROOT.TCanvas("cat_opt_%s" % hsel.GetName(), "cat_opt_%s" % hsel.GetName()) csel.cd() hsel.Draw("box") csel_pj = ROOT.TCanvas("cat_opt_%s_pj" % hsel.GetName(), "cat_opt_%s_pj" % hsel.GetName()) csel_pj.cd() hsel_pj = hsel.Clone() ### hsel_pj = hsel.ProjectionY() ### hsel_pj.Draw() hsel_pj.Draw("box") hsel_pj.SetFillColor(ROOT.kBlack) hsel_pj.SetLineColor(ROOT.kBlack) objs.append(hsel) objs.append(hsel_pj) objs.append(csel) objs.append(csel_pj) maxy = 0. pdfs = [] for sig in signals: pdf = sig.getPdf(ndim + isel) pdf.SetLineColor(ROOT.kBlue) pdf.Scale(1. / pdf.Integral()) pdfs.append(pdf) maxy = max(maxy, pdf.GetMaximum()) objs.append(pdf) for bkg in backgrounds: pdf = bkg.getPdf(ndim + isel) pdf.Scale(1. / pdf.Integral()) pdf.SetLineColor(ROOT.kRed) pdfs.append(pdf) maxy = max(maxy, pdf.GetMaximum()) objs.append(pdf) for pdf in pdfs: pdf.Scale((ncat + 1.5) / maxy / pdf.Integral()) pdf.Draw("l same") hsel_pj.GetYaxis().SetNdivisions(500 + ncat + 3) csel_pj.SetGridy() hsel_pj.Draw("box same") for fmt in "png", "C": csel.SaveAs("%s.%s" % (csel.GetName(), fmt)) csel_pj.SaveAs("%s.%s" % (csel.GetName(), fmt)) canv9 = ROOT.TCanvas("canv9", "canv9") canv9.SetGridx() canv9.SetGridy() canv9.cd() grS.SetMarkerStyle(ROOT.kFullCircle) grS.Draw("AP") canv9.SaveAs("cat_opt_fom.png") canv9.SaveAs("cat_opt_fom.C") ## tmp.Close() return ws