def ParseFor(self,c): v = variable.variable() if self.t.checkNext() != "(": self.error("error for (") else: self.t.getNext() if self.t.checkNext() != ";": self.Expression(v, ";") l = len(v.x.code.c) else: self.t.getNext() if self.t.checkNext() != ";": self.Expression(v, ";") oc = code.jrf() v.pushcode(oc) b = len(v.x.code.c) else: self.t.getNext() x = variable.variable() if self.t.checkNext() != ")": self.Expression(x, ")") else: self.t.getNext() self.Statement(v) if x.x.code: v.x.code.c += x.x.code.c v.pushcode(code.jr(l-len(v.x.code.c)-1)) if oc: oc.j = len(v.x.code.c)-b c.x.code.c += v.x.code.c
def __init__(self, num, dim, config): dtype = config.dtype scope = config.scope initializer = config.initializer use_bias, b_initializer = tuple(config.bias) params = [] with variable_scope(scope): emb = variable("embedding", (num, dim), initializer, dtype) params.append(emb) if use_bias: bias = variable("bias", (dim,), b_initializer, dtype) params.append(bias) def forward(indices): values = embedding_lookup(emb, indices) if not use_bias: return values else: return values + bias self.scope = scope self.config = config self.forward = forward self.parameter = params self.embedding = emb
def read_meta_file(filepath = None): """ Read data's meta information from file. Method tries to be generic in the way that it reads what keys are present in settings.META_COLUMNS and constructs instances of variable- class from those keys so that each key is set as an instance attribute. """ if filepath is None: filepath = settings.META_FILE f = None try: f = open(filepath, "r") print "Reading meta file at %s." % (filepath) except: print "%s could not open \"%s\" in %s" % (inspect.stack()[0][3], filepath, __file__) sys.exit(0) lines = f.readlines() col_info = settings.META_COLUMNS keys = [c[0] for c in col_info] id_col = settings.META_ID_COLUMN variables = dict() i = 0 for l in lines: i += 1 meta = l.split() if len(meta) < len(col_info): print "line %d: Too few variables in line. Expected at least %d, found %d." % (i, len(col_info), len(meta)) print "Leaving line %d out." % (i) continue else: ID = meta[id_col] if ID not in variables: variables[ID] = dict() for x in range(0, len(col_info)): # Convert possible number values value = convert(meta[x], col_info[x][2]) if keys[x] not in settings.OMIT_COLUMNS: if not keys[x] in variables[ID]: variables[ID][keys[x]] = [] variables[ID][keys[x]].append(value) else: if col_info[x][1] == 'set': if value not in variables[ID][keys[x]]: variables[ID][keys[x]].append(value) elif col_info[x][1] == 'multiset': variables[ID][keys[x]].append(value) variables_with_meta = [] for value in variables.values(): current = variable(dictionary = value, id = keys[id_col]) variables_with_meta.append(current) #print current print "File reading done. Found %d variables with meta information." % (len(variables_with_meta)) return variables_with_meta
def addVarFunc(self, func, name, tipo, direccion, isArray, arrSize=-1): func.appendVar(variable( name, tipo, direccion, isArray, arrSize, ), self)
def addAtributo(self, clase, varName, tipo, direccion, isArray, isPublic, arrSize): clase.appendAtributo( variable( varName, tipo, direccion, isArray, arrSize, ), isPublic, self)
def dec_variable(self, name, type_, num=0, line=0): start_addr = None if num > 0 and type_[-1] == '*': for i in range(num): name_ = name + '[' + str(i) + ']' new_var_ = variable.variable(name_, type_[:-1], line=line) if i == 0: start_addr = new_var_.addr self.names[new_var_.name] = new_var_ if start_addr is None: new_var = variable.variable(name, type_, line=line) else: new_var = variable.variable(name, type_, value=start_addr, num=num, line=line) if name in self.names: new_var.parent = self.names[name] self.names[name] = new_var
def ParseIf(self,c): v = variable.variable() if self.t.checkNext() != "(": self.error("error if (") else: self.t.getNext() self.Expression(v, ")") if v.x.code: oc = code.jrf() v.pushcode(oc) b = len(v.x.code.c) self.Statement(v) if oc: oc.j = len(v.x.code.c) - b c.x.code.c += v.x.code.c
def test(): a = variable.variable('a', 'int') new_env = environment(a.addr) new_env.dec_variable('b', 'int') new_env.set_variable('b', 4) b, env = new_env.find_name('b') another_env = environment(b.addr, new_env) another_env.dec_variable('c', 'float') another_env.dec_variable('b', 'int') another_env.set_address(b.addr, 8) another_env.set_variable('b', 4) env = another_env.return_func(7) print(env) man.show_bitmap()
def Statement(self,c): n = self.t.checkNext() if (n == ";"): self.t.getNext() return elif (n == "{"): self.t.getNext() self.ParseBlock(c) return elif (n == "identifier"): self.t.getNext() if self.t.data == "if": self.ParseIf(c) return elif self.t.data == "for": self.ParseFor(c) return else: name = self.t.data if self.t.checkNext() == "(": self.t.getNext() arg = variable.variable() if self.t.checkNext() == ")": self.t.getNext() arg.pushcode(code.push_null()) else: self.Expression(arg, ")") if self.t.checkNext() == "{": self.t.getNext() arg.pushcode(code.argument()) self.ParseBlock(arg) c.index(name).assignment(arg) return c.pushcode(code.pop()) c.pushcode(code.variable(name)) c.x.code.c += arg.x.code.c c.pushcode(code.call()) else: c.pushcode(code.pop()) c.pushcode(code.variable(name)) self.getSuff(c) self.Expression(c, ";", True) return c.pushcode(code.pop()) self.Expression(c, ";")
def standardize_apart(lc, previous_variable=None): if previous_variable is None: previous_variable = set() if type(lc) is UniversalQuantifier or type(lc) is ExistentialQualifier: variables = set(lc.variables()) variables_to_change = previous_variable.intersection(variables) new_variable_name = [ variable(randomString(), static_value=var.value, constants=var.constants) for var in variables_to_change ] proposition = None # TODO return UniversalQuantifier( tuple(new_variable_name + list(variables - variables_to_change)), proposition)
def checkForEqual(self, text): text = text.replace(" ", "") varObject = None if "=" in text: textList = text.split("=") print(self.varList) varObject = self.checkForBrackets(textList[1]) print(textList) if self.varList[textList[0]] is not None: self.varList[textList[0]].setValue(varObject) else: v = var.variable(textList[0], varObject.getOutputTyp(), self) v.setValue(varObject) self.varList[textList[0]] = v #create Variable #v = var.variable(,varObject.,self) print("create variable") else: print("Ungültige Eingabe")
if not os.path.exists("rootFiles/" + flag + "_cut" + cutStr): # if there si already one it does not delete it os.makedirs("rootFiles/" + flag + "_cut" + cutStr) # Compute the fake rate with ID method if computeFR: print("") print("Computing fake rate with mediumId...") data_fr = sample_all["data"].copy() #already with preselection inv_mass(data_fr) var = [] var.append( variable("inv_mass", "m(3#mu) with mediumId=1", "m(3#mu)", "[GeV]", 25, 5., 5.45)) var.append(variable("inv_mass", "m(3#mu)", "m(3#mu)", "[GeV]", 30, 5, 5.45)) sample_dic["data"].df = selMediumID(data_fr, 1).copy() histo_medium1, t1_int = createHisto(sample_dic["data"], var[0], over=False) makeSinglePlot(histo_medium1, var[0], sample_dic["data"], "medium1_" + flag) medium1_gaus_int = gaus_fit(histo_medium1, var[0], "medium1_" + flag, pathDir + "FR_plots", [None, 5.28, 0.07, None, None, None], pol="pol2") sample_dic["data"].df = data_fr
def __init__(self,code): self.line = 0 self.code = code self.next = None self.local = variable.variable() self.static = variable.variable()
def __init__(self): self.stack = [] self.scope = None self.glob = variable.variable()
import code import parser import variable import environment from variable import fromFunction p = parser.Parser("fizzbuzz(n){for (i = 1; i < n; i += 1){s = '';if (i % 3 == 0)s += 'Fizz';if (i % 5 == 0)s += 'Buzz';if (s == '')s = i;print(s);}}fizzbuzz(100);1;") c = variable.variable() p.parse(c) if p.errcount: print "compile error" exit() def Print(s): print s.toString() env = environment.Environment() env.glob = c env.glob.index("print").substitution(fromFunction(Print)) c.prepare(env) r = env.run()
def __init__(self, pdf, data=None, params=None): # Set the pdf self.pdf = pdf self.args = {} # Use inspection to find all arguments of the pdf func_spec = inspect.getargspec(self.pdf) (all_arguments, all_defaults) = (func_spec.args, func_spec.defaults) # Get the defaults for any arguments that have them default_dict = {} for name, val in zip( reversed(all_arguments), reversed(all_defaults) ): default_dict[name] = val # Determine the data var and save an internal 'variable' # If none is supplied, assume it's the 0th argument # If a string is supplied, create a new variable if data == None: data = all_arguments[0] if isinstance(data, variable): self.data = data else: self.data = variable(data) if self.data.name not in all_arguments: print "Error: Supplied data var is not an argument of the supplied pdf", print all_arguments raise Exception("InvalidDataVar") # And create an attribute for easy access if data.name in default_dict: setattr(self, data.name, default_dict[data.name]) else: setattr(self, data.name, 0.0) #self.param_dict[data.name] = data # Get the list of parameters, # create variables based on them, # and store that list of variables self.param_dict = {} param_list = [arg for arg in func_spec.args if arg != self.data.name] for param in param_list: # Check if the parameter matches one in the # supplied list of parameter 'variables' matching_var_list = [var for var in params if var.name==param] param_var = None if len(matching_var_list)==0: self.logging.debug("Creating new param with name: " + param) param_var = variable(param) # If the parameter has a default in the function definition, # Set that default here # MUST SET: param_var.val = param.defaults elif len(matching_var_list)==1: param_var = matching_var_list[0] else: print "Error: More than one parameter variable supplied ", print "with the name: ", param raise Exception("ParamVariable") self.param_dict[param_var.name] = param_var # And create an attribute for easy access if param_var.name in default_dict: setattr(self, param_var.name, default_dict[param_var.name]) else: setattr(self, param_var.name, 0.0) setattr(self, param_var.name + "_var", param_var) self.args.update(self.param_dict) self.args[self.data.name] = self.data self.norm = 1.0 self.normalization_cache = {} self.minimization_cache = {} self.nll_cache = {}
def add_variable(self, lens, type_, **kwargs): '''add a variable to the merit function''' self.vars.append(variable(self.num_vars, lens, type_, **kwargs)) self.num_vars += 1
from variable import variable from makePlot import makeSinglePlot from makePlot import makeStack from sample import sample_coll #from create_datacard import create_datacard_5 ROOT.gROOT.SetBatch() weight = True asimov = False PassFail = True # file with input histos name from bash #f=ROOT.TFile("/work/friti/new/CMSSW_10_2_15/src/HiggsAnalysis/PassFail/prova/fitDiagnostics.root","r") f=ROOT.TFile("/work/friti/new/CMSSW_10_2_15/src/HiggsAnalysis/Q_sq_2020Nov02_cut04/fitDiagnostics.root","r") var = variable("Q_sq", "Q_sq", "Q^{2}", "[GeV]", 12, 1, 10) #var = variable("pt_miss", "pt_miss", "p_{T}^{miss}", "[GeV]", 15, 0, 20) #var = variable("tau", "tau", "prob_{#tau}", "", 10, 0, 0.7) #var = variable("BE_mu_star","E_{#mu}^{*}","E_{#mu}^{*}","[GeV]",12,0.2,2.2) #var = variable("m_miss_sq", "m_miss_sq", "m_{miss}^2", "[GeV^{2}]",10,0,9) #var = variable("toy", "toy", "toy", "", 2, 0, 2) #var = variable("Bmass","m(3#mu)","m(3#mu)","[GeV]",10,3,7) #var = variable("Bmass","m(3#mu)","m(3#mu)","[GeV]",10,3,13) if not PassFail: his_m = f.Get("shapes_fit_s/control/mc_mu") histo_m = ROOT.TH1F("mc_mu","mc_mu", var.nbins, var.xmin, var.xmax) for i in range(1,his_m.GetNbinsX()+1): histo_m.SetBinContent(i,his_m.GetBinContent(i)) histo_m.SetBinError(i,his_m.GetBinError(i)) print("mu",histo_m.Integral())
def __init__(self, input_size, output_size, config=linear_config()): dtype = config.dtype scope = config.scope concat = config.concat multibias = config.multibias use_bias, b_initializer = tuple(config.bias) output_major, w_initializer = tuple(config.weight) if not isinstance(input_size, (list, tuple)): input_size = [input_size] params = [] weights = [] biases = [] with variable_scope(scope): if concat: input_size = sum(input_size) if output_major: shape = [output_size, input_size] else: shape = [input_size, output_size] weight = variable("weight", shape, w_initializer, dtype) params.append(weight) weights.append(weight) else: for i in range(len(input_size)): if output_major: shape = [output_size, input_size[i]] else: shape = [input_size[i], output_size] weight = variable("weight", shape, w_initializer, dtype) params.append(weight) weights.append(weight) if use_bias: shape = [output_size] if not concat and multibias: for i in range(len(input_size)): bias = variable("bias", shape, b_initializer, dtype) params.append(bias) biases.append(bias) else: bias = variable("bias", shape, b_initializer, dtype) params.append(bias) biases.append(bias) def forward(x): if not isinstance(x, (list, tuple)): x = [x] if len(x) != len(input_size): raise RuntimeError("unmatched inputs and weights") outs = [] n = len(x) if concat: if n == 1: x = x[0] else: x = theano.tensor.concatenate(x, -1) if output_major: outs.append(theano.dot(x, weights[0].transpose())) else: outs.append(theano.dot(x, weights[0])) if use_bias: outs.append(biases[0]) else: for i in range(n): if output_major: outs.append(theano.dot(x[i], weights[i].transpose())) else: outs.append(theano.dot(x[i], weights[i])) if use_bias and multibias: outs.append(biases[i]) if use_bias and not multibias: outs.append(biases[0]) y = reduce(theano.tensor.add, outs) return y self.name = scope self.config = config self.parameter = params self.forward = forward
def add_variable(self, type_, min_val, max_val, **kwargs): '''add a variable to the tolerancer''' self.vars.append( variable(self.num_vars, self.lens, type_, min_val, max_val, **kwargs)) self.num_vars += 1
def buildLoopIORegisters(myLoopStorage, myTraceFileName): ''' Build I/O parameters from register. Such parameters are defined as bytes manipulated by a same instruction in a loop body *and* in the same register at this moment. In contrary to memory I/O parameters, we set their values here. ''' for k in myLoopStorage.keys(): myLoop = myLoopStorage[k] # for all instances of the loop for instanceCounter in myLoop.instances.keys(): # Only take into account the valid instances if myLoop.instances[instanceCounter].valid == 0x0: continue registerInputBytes = dict() # addr -> value registerOutputBytes = dict() insCounter = 0x0 time = myLoop.instances[instanceCounter].startTime f = open(myTraceFileName, 'r') lineCounter = 1 # Go to the first line while lineCounter != time + 1: f.readline() lineCounter += 1 firsTurn = 1 while time <= myLoop.instances[instanceCounter].endTime: myIns = executionTrace.lineConnector(f.readline()) if myIns is None: # print "continue" time += 1 continue # Jump over nested loops # The nested loop can turn a different number of times at each turn of the big loop if str(myLoop.body[insCounter]).startswith('+L'): # Goal : move the time over the loop in the trace # What loop ? loopId = int(str(myLoop.body[insCounter])[2:]) # Look for the associated instance (could we make the assumption that the key == ID ?) found = 0x0 for k in myLoopStorage.keys(): if myLoopStorage[k].ID == loopId: found = 1 # Look for the instance with the right start time foundBis = 0x0 for kk in myLoopStorage[k].instances.keys(): if myLoopStorage[k].instances[kk].startTime \ == time: # carry out the instance length in the trace lengthToJump = \ myLoopStorage[k].instances[kk].endTime \ - myLoopStorage[k].instances[kk].startTime + 1 foundBis = 1 myLoop.instances[ instanceCounter].imbricatedInstanceID.append( [k, kk]) break if foundBis == 0x0: print 'Fail to find the instance!' print 'Loop ' + str(myLoop.ID) print 'We look for ' + str(loopId) \ + ' at time ' + str(time) return if found == 0x0: print 'Fail to find the loop !!!' return time += lengthToJump lineCounter = 0x0 # Go to the first line while lineCounter != lengthToJump - 1: f.readline() lineCounter += 1 else: # Read count = 0x0 for rReg in myIns.registersRead: countAddr = 0x0 for addr in utilities.registersAddress(rReg): if addr not in registerOutputBytes.keys(): registerInputBytes[addr] = \ (myIns.registersReadValue[count])[countAddr * 2:countAddr * 2 + 2] countAddr += 1 count += 1 # Write count = 0x0 for wReg in myIns.registersWrite: countAddr = 0x0 for addr in utilities.registersAddress(wReg): registerOutputBytes[addr] = \ (myIns.registersWriteValue[count])[countAddr * 2:countAddr * 2 + 2] countAddr += 1 count += 1 time += 1 insCounter = (insCounter + 1) % len(myLoop.body) if insCounter == 0x0: firsTurn = 0x0 f.close() # Input register parameters for reg in utilities.GPR32: # Check which parts of the reg have been used existL = 0x0 # AL, BL... existH = 0x0 # AH, BH... existX = 0x0 # EAX, EBX... if reg + '3' in registerInputBytes.keys(): existL = 1 if reg + '2' in registerInputBytes.keys(): existH = 1 if reg + '1' in registerInputBytes.keys(): existX = 1 if existX: # 32 bytes register var = variable.variable(0x0, 4, reg) var.value[0x0] = registerInputBytes[reg + '0'] var.value[1] = registerInputBytes[reg + '1'] var.value[2] = registerInputBytes[reg + '2'] var.value[3] = registerInputBytes[reg + '3'] myLoop.instances[ instanceCounter].inputRegisterParameters.append(var) elif existH and existL: # 16 bytes register var = variable.variable(0x0, 2, reg[1:]) var.value[0x0] = registerInputBytes[reg + '2'] var.value[1] = registerInputBytes[reg + '3'] myLoop.instances[ instanceCounter].inputRegisterParameters.append(var) elif existH: # 8 bytes register (AH, BH...) var = variable.variable(0x0, 1, reg[1:2] + 'h') var.value[0x0] = registerInputBytes[reg + '2'] myLoop.instances[ instanceCounter].inputRegisterParameters.append(var) elif existL: # 8 bytes register (AL, BL...) var = variable.variable(0x0, 1, reg[1:2] + 'l') var.value[0x0] = registerInputBytes[reg + '3'] myLoop.instances[ instanceCounter].inputRegisterParameters.append(var) # Output register parameters for reg in utilities.GPR32: # Check which parts of the reg have been used existL = 0x0 # AL, BL... existH = 0x0 # AH, BH... existX = 0x0 # EAX, EBX... if reg + '3' in registerOutputBytes.keys(): existL = 1 if reg + '2' in registerOutputBytes.keys(): existH = 1 if reg + '1' in registerOutputBytes.keys(): existX = 1 if existX: # 32 bytes register var = variable.variable(0x0, 4, reg) var.value[0x0] = registerOutputBytes[reg + '0'] var.value[1] = registerOutputBytes[reg + '1'] var.value[2] = registerOutputBytes[reg + '2'] var.value[3] = registerOutputBytes[reg + '3'] myLoop.instances[ instanceCounter].outputRegisterParameters.append(var) elif existH and existL: # 16 bytes register var = variable.variable(0x0, 2, reg[1:]) var.value[0x0] = registerOutputBytes[reg + '2'] var.value[1] = registerOutputBytes[reg + '3'] myLoop.instances[ instanceCounter].outputRegisterParameters.append(var) elif existH: # 8 bytes register (AH, BH...) var = variable.variable(0x0, 1, reg[1:2] + 'h') var.value[0x0] = registerOutputBytes[reg + '2'] myLoop.instances[ instanceCounter].outputRegisterParameters.append(var) elif existL: # 8 bytes register (AL, BL...) var = variable.variable(0x0, 1, reg[1:2] + 'l') var.value[0x0] = registerOutputBytes[reg + '3'] myLoop.instances[ instanceCounter].outputRegisterParameters.append(var)
def buildLoopIOConstants(myLoopStorage, myTraceFileName): ''' Build constant parameters for loops, e.g. 0x42 in MOV EAX, 0x42. In particular, these are only input parameters. It actually only works for 4-byte contants (cf. TODO list). ''' constantAddr = 0x0 for k in myLoopStorage.keys(): myLoop = myLoopStorage[k] # For all instances of the loop for instanceCounter in myLoop.instances.keys(): # Only take into account the valid instances if myLoop.instances[instanceCounter].valid == 0x0: continue constantSet = set() insCounter = 0x0 time = myLoop.instances[instanceCounter].startTime f = open(myTraceFileName, 'r') lineCounter = 1 # Go to the first line while lineCounter != time + 1: f.readline() lineCounter += 1 firsTurn = 1 while time <= myLoop.instances[instanceCounter].endTime: myIns = executionTrace.lineConnector(f.readline()) if myIns is None: # print "continue" time += 1 continue # Jump over nested loops # The nested loop can turn a different number of times at each turn of the big loop if str(myLoop.body[insCounter]).startswith('+L'): # Goal : move the time over the loop in the trace # What loop ? loopId = int(str(myLoop.body[insCounter])[2:]) # Look for the associated instance (could we make the assumption that the key == ID ?) found = 0x0 for k in myLoopStorage.keys(): if myLoopStorage[k].ID == loopId: found = 1 # Look for the instance with the right start time foundBis = 0x0 for kk in myLoopStorage[k].instances.keys(): if myLoopStorage[k].instances[kk].startTime \ == time: # Carry out the instance length in the trace lengthToJump = myLoopStorage[k].instances[kk].endTime \ - myLoopStorage[k].instances[kk].startTime + 1 foundBis = 1 myLoop.instances[ instanceCounter].imbricatedInstanceID.append( [k, kk]) break if foundBis == 0x0: print 'Fail to find the instance!' print 'Loop ' + str(myLoop.ID) print 'We look for ' + str(loopId) \ + ' at time ' + str(time) return if found == 0x0: print 'Fail to find the loop !!!' return time += lengthToJump lineCounter = 0x0 # Go to the first line while lineCounter != lengthToJump - 1: f.readline() lineCounter += 1 else: for cte in myIns.constants: if cte not in constantSet: constantSet.add(cte) # Due to the way we represent parameters, we have # to attribute fake addresses to constant parameters. These # adresses need to be unique, in order to not # influence the data-flow building step. var = variable.variable(constantAddr, 4) var.constant = 1 constantAddr += 4 for i in range(0x0, 4): var.value[i] = cte[i * 2:i * 2 + 2] myLoop.instances[ instanceCounter].constantParameter.append(var) time += 1
def add_compensator(self, type_, **kwargs): '''add a compensator (variable used in optimization)''' self.comps.append(variable(self.num_comps, lens, type_, **kwargs)) self.num_comps += 1
def get_variable(self, name, shape=None, dtype=None, initializer=None, regularizer=None, reuse=None, trainable=True): initializing_from_value = False if initializer is not None and not callable(initializer): initializing_from_value = True if shape is not None and initializing_from_value: raise ValueError("if initializer is a constant, " "do not specify shape.") should_check = reuse is not None dtype = dtype or theano.config.floatX # name already defined if name in self._vars: if should_check and not reuse: raise ValueError("variable %s already exists, disallowed." % name) found_var = self._vars[name] found_shape = found_var.get_value().shape if not is_compatible_shape(shape, found_shape): raise ValueError("trying to share variable %s, " "but specified shape %s and found shape %s." % (name, shape, found_shape)) if dtype and dtype != found_var.dtype: raise ValueError("trying to share variable %s, but specified " "dtype %s and found dtype %s." % (name, dtype, found_var.dtype)) return found_var # creating a new variable if should_check and reuse: raise ValueError("variable %s does not exist, or was not created " "with get_variable()." % name) # get default initializer if initializer is None: if is_floating_dtype(dtype): initializer = uniform_unit_scaling_initializer() initializing_from_value = False elif is_integer_dtype(dtype): initializer = zeros_initializer() initializer = initializer(shape=shape, dtype=dtype) initializing_from_value = True else: raise ValueError("a initializer for variable %s of %s " "is required" % (name, dtype)) if initializing_from_value: init_val = initializer else: init_val = lambda: initializer(shape, dtype=dtype) # create variable v = variable(initial_value=init_val, name=name, trainable=trainable, dtype=dtype) self._vars[name] = v if regularizer: with name_scope_op(name + "/regularizer/"): loss = regularizer(v) if loss is not None: add_regularization_loss(loss) return v
from makePlot import makeSinglePlot from makePlot import makeStack import os #from create_datacard import create_datacard_5 ROOT.gROOT.SetBatch() flag = 'v3' weight = False saveonwww = False cutStr = '04' # file with input histos name from bash path_combine = "/work/friti/combine/CMSSW_10_2_13/src/HiggsAnalysis/CombinedLimit/Q_sq_2020Dec13_cut04/" f = ROOT.TFile(path_combine + "fitDiagnostics.root", "r") var = variable("Q_sq", "Q_sq", "Q^{2}", "[GeV]", 15, 3, 11) #var = variable("pt_miss", "pt_miss", "p_{T}^{miss}", "[GeV]", 15, 0, 20) #var = variable("tau", "tau", "prob_{#tau}", "", 10, 0, 0.7) #var = variable("BE_mu_star","E_{#mu}^{*}","E_{#mu}^{*}","[GeV]",12,0.2,2.2) #var = variable("m_miss_sq", "m_miss_sq", "m_{miss}^2", "[GeV^{2}]",10,0,3.5) #var = variable("toy", "toy", "toy", "", 2, 0, 2) #var = variable("Bmass","m(3#mu)","m(3#mu)","[GeV]",10,3,7) #var = variable("Bmass","m(3#mu)","m(3#mu)","[GeV]",10,3,13) if weight: scale = 1.333 else: scale = 1 sample_names = [
def buildLoopIOConstants(myLoopStorage, myTraceFileName): ''' Build constant parameters for loops, e.g. 0x42 in MOV EAX, 0x42. In particular, these are only input parameters. It actually only works for 4-byte contants (cf. TODO list). ''' constantAddr = 0x0 for k in myLoopStorage.keys(): myLoop = myLoopStorage[k] # For all instances of the loop for instanceCounter in myLoop.instances.keys(): # Only take into account the valid instances if myLoop.instances[instanceCounter].valid == 0x0: continue constantSet = set() insCounter = 0x0 time = myLoop.instances[instanceCounter].startTime f = open(myTraceFileName, 'r') lineCounter = 1 # Go to the first line while lineCounter != time + 1: f.readline() lineCounter += 1 firsTurn = 1 while time <= myLoop.instances[instanceCounter].endTime: myIns = executionTrace.lineConnector(f.readline()) if myIns is None: # print "continue" time += 1 continue # Jump over nested loops # The nested loop can turn a different number of times at each turn of the big loop if str(myLoop.body[insCounter]).startswith('+L'): # Goal : move the time over the loop in the trace # What loop ? loopId = int(str(myLoop.body[insCounter])[2:]) # Look for the associated instance (could we make the assumption that the key == ID ?) found = 0x0 for k in myLoopStorage.keys(): if myLoopStorage[k].ID == loopId: found = 1 # Look for the instance with the right start time foundBis = 0x0 for kk in myLoopStorage[k].instances.keys(): if myLoopStorage[k].instances[kk].startTime \ == time: # Carry out the instance length in the trace lengthToJump = myLoopStorage[k].instances[kk].endTime \ - myLoopStorage[k].instances[kk].startTime + 1 foundBis = 1 myLoop.instances[instanceCounter].imbricatedInstanceID.append([k,kk]) break if foundBis == 0x0: print 'Fail to find the instance!' print 'Loop ' + str(myLoop.ID) print 'We look for ' + str(loopId) \ + ' at time ' + str(time) return if found == 0x0: print 'Fail to find the loop !!!' return time += lengthToJump lineCounter = 0x0 # Go to the first line while lineCounter != lengthToJump - 1: f.readline() lineCounter += 1 else: for cte in myIns.constants: if cte not in constantSet: constantSet.add(cte) # Due to the way we represent parameters, we have # to attribute fake addresses to constant parameters. These # adresses need to be unique, in order to not # influence the data-flow building step. var = variable.variable(constantAddr, 4) var.constant = 1 constantAddr += 4 for i in range(0x0, 4): var.value[i] = cte[i * 2:i * 2 + 2] myLoop.instances[instanceCounter].constantParameter.append(var) time += 1
def addParameter(self, func, name, tipo, direccion, isArray): func.appendParam(variable(name, tipo, direccion, isArray, -1))
tau_df = tau.copy() tau_df = selSoftID(tau_df,1) onia_df = onia.copy() onia_df = selSoftID(onia_df,1) data_df = data.copy() data_df = selSoftID(data_df,1) # Compute the fake rate with soft ID method if computeFR: print("") print("Computing fake rate with softId...") data_fr = data.copy() #already with ID selection on jpsi muons inv_mass(data_fr) var=[] var.append(variable("inv_mass","m(3#mu) with softId=1","m(3#mu)","[GeV]", 30, 5., 5.45 )) var.append(variable("inv_mass","m(3#mu)","m(3#mu)", "[GeV]",30, 5, 5.45 )) var.append(variable("inv_mass","m(3#mu) with softId=0","m(3#mu)", "[GeV]",60, 3., 8. )) var.append(variable("inv_mass","m(3#mu) with softId=0","m(3#mu)", "[GeV]",60, 5, 5.45 )) sampleColl[-1].df = selSoftID(data_fr,1).copy() histo_soft1, t1_int = createHisto(sampleColl[-1],var[0]) makeSinglePlot(histo_soft1,var[0],sampleColl[-1],"soft1_" + flag) soft1_gaus_int=gaus_fit(histo_soft1,var[0],"soft1_" + flag,[None,5.28,0.07,None,None,None],pol="pol2") sampleColl[-1].df = data_fr histo_softAll, tAll_int = createHisto(sampleColl[-1],var[1] ) makeSinglePlot(histo_softAll,var[1],sampleColl[-1],"softAll_" + flag) softall_gaus_int=gaus_fit(histo_softAll,var[1],"softAll_" + flag,[None,5.29,0.07,None,None,None])
def buildLoopIOMemory(myLoopStorage, myTraceFileName): ''' Build I/O parameters from memory. A parameter is defined as a set of bytes adjacent in memory *and* used (R|W) by a same instruction in a loop body. The actual values of these parameters will be set later during the loop data flow graph building. ''' for k in myLoopStorage.keys(): myLoop = myLoopStorage[k] # For all instances of the loop for instanceCounter in myLoop.instances.keys(): # Only take into account the valid instances if myLoop.instances[instanceCounter].valid == 0x0: continue model = list() for ins in range(0x0, len(myLoop.body)): model.append(list()) model[ins].append(set()) # Read memory addresses model[ins].append(set()) # Written memory addresses writtenAddresses = set() insCounter = 0x0 time = myLoop.instances[instanceCounter].startTime f = open(myTraceFileName, 'r') lineCounter = 1 # Go to the first line while lineCounter != time + 1: f.readline() lineCounter += 1 firsTurn = 1 while time <= myLoop.instances[instanceCounter].endTime: myIns = executionTrace.lineConnector(f.readline()) if myIns is None: time += 1 continue # Jump over nested loops # The nested loop can turn a different number of times at each turn of the big loop if str(myLoop.body[insCounter]).startswith('+L'): # Goal : move the time over the loop in the trace loopId = int(str(myLoop.body[insCounter])[2:]) # Look for the associated instance based on its starttime found = 0x0 for k in myLoopStorage.keys(): if myLoopStorage[k].ID == loopId: found = 1 foundBis = 0x0 for kk in myLoopStorage[k].instances.keys(): if myLoopStorage[k].instances[kk].startTime \ == time: # carry out the instance length in the trace lengthToJump = \ myLoopStorage[k].instances[kk].endTime \ - myLoopStorage[k].instances[kk].startTime + 1 foundBis = 1 myLoop.instances[instanceCounter].imbricatedInstanceID.append([k, kk]) break if foundBis == 0x0: print 'Fail to find the instance!' print 'Loop ' + str(myLoop.ID) print 'We look for ' + str(loopId) \ + ' at time ' + str(time) return if found == 0x0: print 'Fail to find the loop !!!' return time += lengthToJump lineCounter = 0x0 # Go to the first line while lineCounter != lengthToJump - 1: f.readline() lineCounter += 1 else: # Read count = 0x0 for rAddr in myIns.memoryReadAddress: for b in range(0x0, myIns.memoryReadSize[count]): addr = int(rAddr, 16) + b # Not previously written ? if addr not in writtenAddresses: model[insCounter][0x0].add(addr) count += 1 # Write count = 0x0 for wAddr in myIns.memoryWriteAddress: for b in range(0x0, myIns.memoryWriteSize[count]): addr = int(wAddr, 16) + b model[insCounter][1].add(addr) writtenAddresses.add(addr) count += 1 time += 1 insCounter = (insCounter + 1) % len(myLoop.body) if insCounter == 0x0: firsTurn = 0x0 f.close() # Build input variables inputVar = list() for ins in range(0x0, len(myLoop.body)): if len(model[ins][0x0]) != 0x0: var = variable.variable(0x0, 0x0) for addr in range(min(model[ins][0x0]), max(model[ins][0x0]) + 1): if addr in model[ins][0x0]: if var.startAddress == 0x0: var.startAddress = addr var.incrementSize() else: # Close existing var if there is one if var.startAddress != 0x0: inputVar.append(var) var = variable.variable(0x0, 0x0) # Close last one if var.startAddress != 0x0: inputVar.append(var) var = variable.variable(0x0, 0x0) # Build output variables outputVar = list() for ins in range(0x0, len(myLoop.body)): if len(model[ins][1]) != 0x0: var = variable.variable(0x0, 0x0) for addr in range(min(model[ins][1]), max(model[ins][1]) + 1): if addr in model[ins][1]: if var.startAddress == 0x0: var.startAddress = addr var.incrementSize() else: # Close existing var if there is one if var.startAddress != 0x0: outputVar.append(var) var = variable.variable(0x0, 0x0) # Close last one if var.startAddress != 0x0: outputVar.append(var) var = variable.variable(0x0, 0x0) # Deal with doublons # Input vars i = 0x0 while i < len(inputVar): for j in range(i + 1, len(inputVar)): if inputVar[i].contains(inputVar[j]): del inputVar[j] i = -1 break if inputVar[i].intersects(inputVar[j]): inputVar[i] = inputVar[i].merge(inputVar[j]) del inputVar[j] i = -1 break i += 1 # # Output vars i = 0x0 while i < len(outputVar): for j in range(i + 1, len(outputVar)): if outputVar[i].contains(outputVar[j]): del outputVar[j] i = -1 break if outputVar[i].intersects(outputVar[j]): outputVar[i] = outputVar[i].merge(outputVar[j]) del outputVar[j] i = -1 break i += 1 myLoop.instances[instanceCounter].inputMemoryParameters = inputVar myLoop.instances[instanceCounter].outputMemoryParameters = outputVar
def buildLoopIOMemory(myLoopStorage, myTraceFileName): ''' Build I/O parameters from memory. A parameter is defined as a set of bytes adjacent in memory *and* used (R|W) by a same instruction in a loop body. The actual values of these parameters will be set later during the loop data flow graph building. ''' for k in myLoopStorage.keys(): myLoop = myLoopStorage[k] # For all instances of the loop for instanceCounter in myLoop.instances.keys(): # Only take into account the valid instances if myLoop.instances[instanceCounter].valid == 0x0: continue model = list() for ins in range(0x0, len(myLoop.body)): model.append(list()) model[ins].append(set()) # Read memory addresses model[ins].append(set()) # Written memory addresses writtenAddresses = set() insCounter = 0x0 time = myLoop.instances[instanceCounter].startTime f = open(myTraceFileName, 'r') lineCounter = 1 # Go to the first line while lineCounter != time + 1: f.readline() lineCounter += 1 firsTurn = 1 while time <= myLoop.instances[instanceCounter].endTime: myIns = executionTrace.lineConnector(f.readline()) if myIns is None: time += 1 continue # Jump over nested loops # The nested loop can turn a different number of times at each turn of the big loop if str(myLoop.body[insCounter]).startswith('+L'): # Goal : move the time over the loop in the trace loopId = int(str(myLoop.body[insCounter])[2:]) # Look for the associated instance based on its starttime found = 0x0 for k in myLoopStorage.keys(): if myLoopStorage[k].ID == loopId: found = 1 foundBis = 0x0 for kk in myLoopStorage[k].instances.keys(): if myLoopStorage[k].instances[kk].startTime \ == time: # carry out the instance length in the trace lengthToJump = \ myLoopStorage[k].instances[kk].endTime \ - myLoopStorage[k].instances[kk].startTime + 1 foundBis = 1 myLoop.instances[ instanceCounter].imbricatedInstanceID.append( [k, kk]) break if foundBis == 0x0: print 'Fail to find the instance!' print 'Loop ' + str(myLoop.ID) print 'We look for ' + str(loopId) \ + ' at time ' + str(time) return if found == 0x0: print 'Fail to find the loop !!!' return time += lengthToJump lineCounter = 0x0 # Go to the first line while lineCounter != lengthToJump - 1: f.readline() lineCounter += 1 else: # Read count = 0x0 for rAddr in myIns.memoryReadAddress: for b in range(0x0, myIns.memoryReadSize[count]): addr = int(rAddr, 16) + b # Not previously written ? if addr not in writtenAddresses: model[insCounter][0x0].add(addr) count += 1 # Write count = 0x0 for wAddr in myIns.memoryWriteAddress: for b in range(0x0, myIns.memoryWriteSize[count]): addr = int(wAddr, 16) + b model[insCounter][1].add(addr) writtenAddresses.add(addr) count += 1 time += 1 insCounter = (insCounter + 1) % len(myLoop.body) if insCounter == 0x0: firsTurn = 0x0 f.close() # Build input variables inputVar = list() for ins in range(0x0, len(myLoop.body)): if len(model[ins][0x0]) != 0x0: var = variable.variable(0x0, 0x0) for addr in range(min(model[ins][0x0]), max(model[ins][0x0]) + 1): if addr in model[ins][0x0]: if var.startAddress == 0x0: var.startAddress = addr var.incrementSize() else: # Close existing var if there is one if var.startAddress != 0x0: inputVar.append(var) var = variable.variable(0x0, 0x0) # Close last one if var.startAddress != 0x0: inputVar.append(var) var = variable.variable(0x0, 0x0) # Build output variables outputVar = list() for ins in range(0x0, len(myLoop.body)): if len(model[ins][1]) != 0x0: var = variable.variable(0x0, 0x0) for addr in range(min(model[ins][1]), max(model[ins][1]) + 1): if addr in model[ins][1]: if var.startAddress == 0x0: var.startAddress = addr var.incrementSize() else: # Close existing var if there is one if var.startAddress != 0x0: outputVar.append(var) var = variable.variable(0x0, 0x0) # Close last one if var.startAddress != 0x0: outputVar.append(var) var = variable.variable(0x0, 0x0) # Deal with doublons # Input vars i = 0x0 while i < len(inputVar): for j in range(i + 1, len(inputVar)): if inputVar[i].contains(inputVar[j]): del inputVar[j] i = -1 break if inputVar[i].intersects(inputVar[j]): inputVar[i] = inputVar[i].merge(inputVar[j]) del inputVar[j] i = -1 break i += 1 # # Output vars i = 0x0 while i < len(outputVar): for j in range(i + 1, len(outputVar)): if outputVar[i].contains(outputVar[j]): del outputVar[j] i = -1 break if outputVar[i].intersects(outputVar[j]): outputVar[i] = outputVar[i].merge(outputVar[j]) del outputVar[j] i = -1 break i += 1 myLoop.instances[instanceCounter].inputMemoryParameters = inputVar myLoop.instances[ instanceCounter].outputMemoryParameters = outputVar
def buildLoopIORegisters(myLoopStorage, myTraceFileName): ''' Build I/O parameters from register. Such parameters are defined as bytes manipulated by a same instruction in a loop body *and* in the same register at this moment. In contrary to memory I/O parameters, we set their values here. ''' for k in myLoopStorage.keys(): myLoop = myLoopStorage[k] # for all instances of the loop for instanceCounter in myLoop.instances.keys(): # Only take into account the valid instances if myLoop.instances[instanceCounter].valid == 0x0: continue registerInputBytes = dict() # addr -> value registerOutputBytes = dict() insCounter = 0x0 time = myLoop.instances[instanceCounter].startTime f = open(myTraceFileName, 'r') lineCounter = 1 # Go to the first line while lineCounter != time + 1: f.readline() lineCounter += 1 firsTurn = 1 while time <= myLoop.instances[instanceCounter].endTime: myIns = executionTrace.lineConnector(f.readline()) if myIns is None: # print "continue" time += 1 continue # Jump over nested loops # The nested loop can turn a different number of times at each turn of the big loop if str(myLoop.body[insCounter]).startswith('+L'): # Goal : move the time over the loop in the trace # What loop ? loopId = int(str(myLoop.body[insCounter])[2:]) # Look for the associated instance (could we make the assumption that the key == ID ?) found = 0x0 for k in myLoopStorage.keys(): if myLoopStorage[k].ID == loopId: found = 1 # Look for the instance with the right start time foundBis = 0x0 for kk in myLoopStorage[k].instances.keys(): if myLoopStorage[k].instances[kk].startTime \ == time: # carry out the instance length in the trace lengthToJump = \ myLoopStorage[k].instances[kk].endTime \ - myLoopStorage[k].instances[kk].startTime + 1 foundBis = 1 myLoop.instances[instanceCounter].imbricatedInstanceID.append([k, kk]) break if foundBis == 0x0: print 'Fail to find the instance!' print 'Loop ' + str(myLoop.ID) print 'We look for ' + str(loopId) \ + ' at time ' + str(time) return if found == 0x0: print 'Fail to find the loop !!!' return time += lengthToJump lineCounter = 0x0 # Go to the first line while lineCounter != lengthToJump - 1: f.readline() lineCounter += 1 else: # Read count = 0x0 for rReg in myIns.registersRead: countAddr = 0x0 for addr in utilities.registersAddress(rReg): if addr not in registerOutputBytes.keys(): registerInputBytes[addr] = \ (myIns.registersReadValue[count])[countAddr * 2:countAddr * 2 + 2] countAddr += 1 count += 1 # Write count = 0x0 for wReg in myIns.registersWrite: countAddr = 0x0 for addr in utilities.registersAddress(wReg): registerOutputBytes[addr] = \ (myIns.registersWriteValue[count])[countAddr * 2:countAddr * 2 + 2] countAddr += 1 count += 1 time += 1 insCounter = (insCounter + 1) % len(myLoop.body) if insCounter == 0x0: firsTurn = 0x0 f.close() # Input register parameters for reg in utilities.GPR32: # Check which parts of the reg have been used existL = 0x0 # AL, BL... existH = 0x0 # AH, BH... existX = 0x0 # EAX, EBX... if reg + '3' in registerInputBytes.keys(): existL = 1 if reg + '2' in registerInputBytes.keys(): existH = 1 if reg + '1' in registerInputBytes.keys(): existX = 1 if existX: # 32 bytes register var = variable.variable(0x0, 4, reg) var.value[0x0] = registerInputBytes[reg + '0'] var.value[1] = registerInputBytes[reg + '1'] var.value[2] = registerInputBytes[reg + '2'] var.value[3] = registerInputBytes[reg + '3'] myLoop.instances[instanceCounter].inputRegisterParameters.append(var) elif existH and existL: # 16 bytes register var = variable.variable(0x0, 2, reg[1:]) var.value[0x0] = registerInputBytes[reg + '2'] var.value[1] = registerInputBytes[reg + '3'] myLoop.instances[instanceCounter].inputRegisterParameters.append(var) elif existH: # 8 bytes register (AH, BH...) var = variable.variable(0x0, 1, reg[1:2] + 'h') var.value[0x0] = registerInputBytes[reg + '2'] myLoop.instances[instanceCounter].inputRegisterParameters.append(var) elif existL: # 8 bytes register (AL, BL...) var = variable.variable(0x0, 1, reg[1:2] + 'l') var.value[0x0] = registerInputBytes[reg + '3'] myLoop.instances[instanceCounter].inputRegisterParameters.append(var) # Output register parameters for reg in utilities.GPR32: # Check which parts of the reg have been used existL = 0x0 # AL, BL... existH = 0x0 # AH, BH... existX = 0x0 # EAX, EBX... if reg + '3' in registerOutputBytes.keys(): existL = 1 if reg + '2' in registerOutputBytes.keys(): existH = 1 if reg + '1' in registerOutputBytes.keys(): existX = 1 if existX: # 32 bytes register var = variable.variable(0x0, 4, reg) var.value[0x0] = registerOutputBytes[reg + '0'] var.value[1] = registerOutputBytes[reg + '1'] var.value[2] = registerOutputBytes[reg + '2'] var.value[3] = registerOutputBytes[reg + '3'] myLoop.instances[instanceCounter].outputRegisterParameters.append(var) elif existH and existL: # 16 bytes register var = variable.variable(0x0, 2, reg[1:]) var.value[0x0] = registerOutputBytes[reg + '2'] var.value[1] = registerOutputBytes[reg + '3'] myLoop.instances[instanceCounter].outputRegisterParameters.append(var) elif existH: # 8 bytes register (AH, BH...) var = variable.variable(0x0, 1, reg[1:2] + 'h') var.value[0x0] = registerOutputBytes[reg + '2'] myLoop.instances[instanceCounter].outputRegisterParameters.append(var) elif existL: # 8 bytes register (AL, BL...) var = variable.variable(0x0, 1, reg[1:2] + 'l') var.value[0x0] = registerOutputBytes[reg + '3'] myLoop.instances[instanceCounter].outputRegisterParameters.append(var)
for sname in sample_names[:-2]: #except mis-id bkg sample_df[sname] = sample_all[sname].copy() sample_df[sname] = selSoftID(sample_df[sname],1) sample_df["data"] = sample_all["data"].copy() sample_df["data"] = selSoftID(sample_df["data"],1) # Compute the fake rate with soft ID method if computeFR: print("") print("Computing fake rate with mediumId...") data_fr = sample_all["data"].copy() #already with ID selection on jpsi muons inv_mass(data_fr) var=[] var.append(variable("inv_mass","m(3#mu) with mediumId=1","m(3#mu)","[GeV]", 40, 5., 5.45 )) var.append(variable("inv_mass","m(3#mu)","m(3#mu)", "[GeV]",40, 5, 5.45 )) sample_dic["data"].df = selMediumID(data_fr,1).copy() histo_medium1, t1_int = createHisto(sample_dic["data"],var[0],over = False) makeSinglePlot(histo_medium1,var[0],sample_dic["data"],"medium1_" + flag) medium1_gaus_int=gaus_fit(histo_medium1,var[0],"medium1_" + flag,[None,5.28,0.07,None,None,None],pol="pol1") sample_dic["data"].df = data_fr histo_mediumAll, tAll_int = createHisto(sample_dic["data"],var[1] ,over = False) makeSinglePlot(histo_mediumAll,var[1],sample_dic["data"],"mediumAll_" + flag) mediumall_gaus_int=gaus_fit(histo_mediumAll,var[1],"mediumAll_" + flag,[None,5.29,0.07,None,None,None]) fr_mediumId = medium1_gaus_int/mediumall_gaus_int
label=labels, weights=weights, stacked=True, bins=v.binning, log=v.logy) plt.ylabel(v.ylabel) plt.xlabel(v.xlabel) plt.legend() plt.draw() plt.savefig(v.varname + '_' + sel.replace('==', 'eq').replace('>', 'gt') + '.png', dpi=250) return var_HT = var.variable('HT', [0, 250, 500, 750, 1000, 1500, 3000, 10000], r'$H_T$ [GeV]', 'Entries', True) var_MET = var.variable('MET', [0, 250, 500, 750, 1000, 1500, 3000, 10000], r'$E^{miss}_T$ [GeV]', 'Entries', True) var_NJET = var.variable('njet', np.linspace(0.5, 20.5, 21), r'$N_{jets}$', 'Entries', True) var_NBJET = var.variable('nbjet', np.linspace(0.5, 10.5, 11), r'$N_{b-jets}$', 'Entries', True) var_SRi = var.variable('ChIn', np.linspace(0.5, 7.5, 8), r'Channel index', 'Entries', True) var_array = [var_HT, var_MET, var_NJET, var_NBJET, var_SRi] plot(var_array, 'ht>500e3') plot(var_array, 'nbjet==4') tend = default_timer() print ' --> All happened in {:.2f}s\n'.format(tend - t0)