def init_kafka(self):
if self.testing:
self._messages = []
else:
bootstrap_servers = getenv("KAFKA_BOOTSTRAP_SERVERS",
type=list,
separator=",")
kafka_args = {"bootstrap_servers": bootstrap_servers}
if self.ssl_security_protocol:
kafka_args.update({
"ssl_cafile":
getenv("KAFKA_SSL_CAFILE", type=str),
"ssl_certfile":
getenv("KAFKA_SSL_CERTFILE", type=str),
"ssl_keyfile":
getenv("KAFKA_SSL_KEYFILE", type=str),
"security_protocol":
"SSL",
})
logger.debug(f"bootstrap servers: {bootstrap_servers}")
self._producer = KafkaProducer(value_serializer=msgpack.dumps,
**kafka_args)
self._consumer = KafkaConsumer(
self.name,
group_id=getenv("KIRBY_SUPERVISOR_GROUP_ID", type=str),
enable_auto_commit=True,
value_deserializer=msgpack.loads,
**kafka_args,
)
def topic_sender():
args = {
"client_id": "topic_sender",
"bootstrap_servers": getenv(
"KAFKA_BOOTSTRAP_SERVERS", type=list, separator=","
),
"value_serializer": msgpack.dumps,
}
if getenv("KAFKA_USE_TLS", type=bool):
args.update(
{
"security_protocol": "SSL",
"ssl_cafile": os.environ["KAFKA_SSL_CAFILE"],
"ssl_certfile": os.environ["KAFKA_SSL_CERTFILE"],
"ssl_keyfile": os.environ["KAFKA_SSL_KEYFILE"],
}
)
producer = KafkaProducer(**args)
@tenacity.retry(**kafka_retry_args)
def send(topic_name, data, submitted=None):
if submitted is None:
submitted = datetime.datetime.utcnow()
timestamp_ms = int(submitted.timestamp() * 1000)
producer.send(topic_name, value=data, timestamp_ms=timestamp_ms)
producer.flush()
yield send
producer.close()
def test_getenv_default(self):
"""
If environment variable does not exist:
ensure getenv returns None if default value is not specified,
ensure getenv returns default value if it is specified,
ensure getenv does not cast default value to desired type.
"""
self.assertEqual(getenv(self.test_var_name), None)
self.assertEqual(getenv(self.test_var_name, default='default string value'), 'default string value')
self.assertEqual(getenv(self.test_var_name, type=int, default='default string value'), 'default string value')
def test_getenv_type_bool(self):
"""
If environment variable exists and desired type is bool, ensure getenv returns bool.
"""
os.environ[self.test_var_name] = 'true'
self.assertEqual(getenv(self.test_var_name, type=bool), True)
os.environ[self.test_var_name] = 'True'
self.assertEqual(getenv(self.test_var_name, type=bool), True)
os.environ[self.test_var_name] = '1'
self.assertEqual(getenv(self.test_var_name, type=bool), True)
os.environ[self.test_var_name] = 'absolutely not a boolean'
self.assertEqual(getenv(self.test_var_name, type=bool), True)
os.environ[self.test_var_name] = ' '
self.assertEqual(getenv(self.test_var_name, type=bool), True)
os.environ[self.test_var_name] = 'false'
self.assertEqual(getenv(self.test_var_name, type=bool), False)
os.environ[self.test_var_name] = 'False'
self.assertEqual(getenv(self.test_var_name, type=bool), False)
os.environ[self.test_var_name] = '0'
self.assertEqual(getenv(self.test_var_name, type=bool), False)
os.environ[self.test_var_name] = ''
self.assertEqual(getenv(self.test_var_name, type=bool), False)
def kafka_topic_factory():
from smart_getenv import getenv
from contextlib import contextmanager
from kafka import KafkaAdminClient
from kafka.admin import NewTopic
from kafka.errors import UnknownTopicOrPartitionError
import logging
logger = logging.getLogger(__name__)
bootstrap_servers = getenv(
"KAFKA_BOOTSTRAP_SERVERS", type=list, separator=","
)
if bootstrap_servers:
args = {"bootstrap_servers": bootstrap_servers}
if getenv("KAFKA_USE_TLS", type=bool):
args.update(
{
"security_protocol": "SSL",
"ssl_cafile": getenv("KAFKA_SSL_CAFILE"),
"ssl_certfile": getenv("KAFKA_SSL_CERTFILE"),
"ssl_keyfile": getenv("KAFKA_SSL_KEYFILE"),
}
)
admin = tenacity.retry(**kafka_retry_args)(KafkaAdminClient)(**args)
@tenacity.retry(**kafka_retry_args)
@contextmanager
def create_kafka_topic(topic_name, timeout_ms=1500):
try:
admin.delete_topics([topic_name])
except UnknownTopicOrPartitionError:
pass
admin.create_topics(
[NewTopic(topic_name, 1, 1)], timeout_ms=timeout_ms
)
yield
admin.delete_topics([topic_name])
yield create_kafka_topic
admin.close()
else:
logger.warning(
f"There is no KAFKA_BOOTSTRAP_SERVERS. "
"Creation of kafka_topic skipped."
)
yield
def read_topic(name):
"""
debug function to consume the messages in a kafka topic
:param name: name of the topic
"""
from kafka import KafkaConsumer
import msgpack
import json
from pprint import pformat
bootstrap_servers = getenv("KAFKA_BOOTSTRAP_SERVERS",
type=list,
separator=",")
consumer = KafkaConsumer(
name,
bootstrap_servers=bootstrap_servers,
auto_offset_reset="earliest",
enable_auto_commit=True,
value_deserializer=msgpack.loads,
)
for idx, message in enumerate(consumer, start=1):
message = message.value
try:
message = json.loads(message.decode("utf-8"))
logger.debug(pformat(message))
except Exception:
logger.debug(message)
def test_getenv_type_float(self):
"""
If environment variable exists and desired type is float:
ensure getenv returns float,
ensure getenv excepts if value can not be casted to float.
"""
os.environ[self.test_var_name] = '123.4'
self.assertEqual(getenv(self.test_var_name, type=float), 123.4)
os.environ[self.test_var_name] = 'absolutely not a float'
try:
getenv(self.test_var_name, type=float)
self.fail('Calling getenv_int on a environment variable with'
' non-castable to float value should fail with exception!')
except ValueError:
pass
def runIsoOrder(analysis, sample, channel, count, num, mid1, mid2,cutMapper,numFilesPerCycle) :
#if svFitPost == 'true' : SVF = True
#else : SVF = False
save = '%s_%i_%s' % (sample, count, channel)
#print "save",save
print "%5i %20s %10s %3i: ====>>> START Iso Order <<<====" % (num, sample, channel, count)
''' 2. Rename branches, Tau and Iso order legs '''
print "%5i %20s %10s %3i: Started Iso Ordering" % (num, sample, channel, count)
isoQty = renameBranches( analysis, mid1, mid2, save, channel, count )
### FF values for data events
doFF = getenv('doFF', type=bool)
if doFF and channel == 'tt' :
from util.applyFakeFactors import fillFakeFactorValues
fillFakeFactorValues( analysis, mid2, save, channel )
#output.put( '%s%s/%s.root' % (analysis, mid2, save) )
print "%5i %20s %10s %3i: Finished Iso Ordering" % (num, sample, channel, count)
#output.put((num, sample, channel, count, initialQty, postCutQty, isoQty ))
print "%5i %20s %10s %3i: ====>>> DONE Iso Order <<<====" % (num, sample, channel, count)
return (num, sample, channel, count, isoQty )
def get(self, key, _type=None):
kwargs = {"name": key, "default": self._config.get(key)}
if _type:
kwargs["type"] = _type
return getenv(**kwargs)
def fetch_jobs(self):
url = getenv("KIRBY_SCHEDULE_ENDPOINT", type=str)
try:
response = requests.get(url)
return response.text
except requests.exceptions.ConnectionError:
logger.exception("unable to fetch jobs: ")
def test_getenv_type_list(self):
"""
If environment variable exists and desired type is list:
ensure getenv returns list,
ensure getenv default separator is ',',
ensure getenv supports custom separator.
"""
os.environ[self.test_var_name] = 'abc'
self.assertEqual(getenv(self.test_var_name, type=list), ['abc'])
os.environ[self.test_var_name] = 'a,b,c'
self.assertEqual(getenv(self.test_var_name, type=list), ['a', 'b', 'c'])
os.environ[self.test_var_name] = ',a,b,c,'
self.assertEqual(getenv(self.test_var_name, type=list, separator=','), ['', 'a', 'b', 'c', ''])
os.environ[self.test_var_name] = 'a:b:c'
self.assertEqual(getenv(self.test_var_name, type=list, separator=':'), ['a', 'b', 'c'])
def __getattr__(self, item):
signature = get_signature().get(item, {})
attr = getenv(item, **signature)
if attr:
return attr
else:
raise MissingEnvironmentVariable(
f"The environment variable {item} hasn't been initialized."
)
def test_getenv_type_tuple(self):
"""
If environment variable exists and desired type is tuple:
ensure getenv returns tuple,
ensure getenv default separator is ',',
ensure getenv supports custom separator.
"""
os.environ[self.test_var_name] = 'abc'
self.assertEqual(getenv(self.test_var_name, type=tuple), ('abc',))
os.environ[self.test_var_name] = 'a,b,c'
self.assertEqual(getenv(self.test_var_name, type=tuple), ('a', 'b', 'c'))
os.environ[self.test_var_name] = ',a,b,c,'
self.assertEqual(getenv(self.test_var_name, type=tuple, separator=','), ('', 'a', 'b', 'c', ''))
os.environ[self.test_var_name] = 'a:b:c'
self.assertEqual(getenv(self.test_var_name, type=tuple, separator=':'), ('a', 'b', 'c'))
def app_maker(config=None):
app = Flask("kirby")
app.config["TESTING"] = getenv("TESTING", type=bool)
app.config["SQLALCHEMY_DATABASE_URI"] = getenv("SQLALCHEMY_DATABASE_URI")
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = getenv(
"SQLALCHEMY_TRACK_MODIFICATIONS")
app.config["SECRET_KEY"] = getenv("SECRET_KEY")
app.config["SECURITY_PASSWORD_SALT"] = getenv("SECURITY_PASSWORD_SALT")
if config: # pragma: no cover
app.config.update(config)
if not app.config.get("SQLALCHEMY_DATABASE_URI"):
raise ConfigException(
"Database is not defined. Please specify 'SQLALCHEMY_DATABASE_URI'"
)
db.init_app(app)
admin.init_app(app)
api.init_app(app)
security._state = security.init_app(app=app,
datastore=user_datastore,
login_form=LoginForm)
@security.context_processor
def security_context_processor():
return dict(
admin_base_template=admin.base_template,
admin_view=admin.index_view,
h=admin_helpers,
get_url=url_for,
)
@app.before_first_request
def create_models():
db.create_all()
user_datastore.find_or_create_role(UserRoles.ADMIN.value)
db.session.commit()
return app
def ensure_environment(self, venvs_directory=None):
if not venvs_directory:
venvs_directory = getenv(
"KIRBY_VENV_DIRECTORY",
default=expanduser("~/.kirby/virtualenvs"),
)
venv_path = os.path.join(venvs_directory, self.venv_name)
logging.info(f"creating venv for {self.venv_name} at {venv_path}")
env = VirtualEnvironment(venv_path)
env.install(self.package_name)
return env
def create_kirby_topic(topic_name, *args, timeout_ms=1500, **kargs):
if bootstrap_servers:
kargs.update(
use_tls=getenv("KAFKA_USE_TLS", type=bool, default=True))
with kafka_topic_factory(topic_name, timeout_ms=timeout_ms):
with Topic(topic_name, *args, **kargs) as kirby_topic:
yield kirby_topic
else:
logger.warning(f"There is no KAFKA_BOOTSTRAP_SERVERS. "
"Topic will be created in testing mode")
kargs.update(testing=True)
with Topic(topic_name, *args, **kargs) as kirby_topic:
yield kirby_topic
def test_getenv_type_dict(self):
"""
If environment variable exists and desired type is dict:
ensure getenv returns dict,
ensure getenv supports custom separator.
"""
os.environ[self.test_var_name] = '{"key": "value"}'
self.assertEqual(getenv(self.test_var_name, type=dict), {'key': 'value'})
os.environ[self.test_var_name] = '{ "key": "value" }'
self.assertEqual(getenv(self.test_var_name, type=dict), {'key': 'value'})
os.environ[self.test_var_name] = '{ "key": "value" }'
self.assertEqual(getenv(self.test_var_name, type=dict), {'key': 'value'})
os.environ[self.test_var_name] = 'absolutely not a dict'
try:
getenv(self.test_var_name, type=dict)
self.fail('Calling getenv with desired type of dict on a environment variable with'
' non-castable to dict value should fail with exception!')
except TypeError:
pass
except SyntaxError:
pass
def kirby_topic(kirby_app, kafka_topic_factory):
import logging
logger = logging.getLogger(__name__)
bootstrap_servers = getenv("KAFKA_BOOTSTRAP_SERVERS",
type=list,
separator=",")
if bootstrap_servers:
with kafka_topic_factory(TOPIC_NAME):
with Topic(kirby_app, TOPIC_NAME) as topic:
yield topic
else:
logger.warning(f"There is no KAFKA_BOOTSTRAP_SERVERS. "
"Topic will be created in testing mode")
with Topic(kirby_app, TOPIC_NAME, testing=True) as topic:
yield topic
def test_it_can_create_a_queue_integration(kafka_topic_factory):
start = datetime.datetime(year=2019, month=7, day=18, hour=15, minute=39)
offset = datetime.timedelta(seconds=5)
with kafka_topic_factory("kirby-test-integration"):
q = Queue(
"kirby-test-integration",
init_time=start - 4 * offset,
use_tls=getenv("KAFKA_USE_TLS", type=bool, default=True),
)
q.append("too early", submitted=start - 2 * offset)
q.append("hello world", submitted=start)
q.append("too late", submitted=start + 2 * offset)
messages = q.between(start, start + offset)
assert messages == ["hello world"]
def topic_sender():
use_tls = getenv("KAFKA_USE_TLS", type=bool, default=False)
topic_config = TopicConfig(
name=None, group_id=None, use_tls=use_tls, raw_records=True
)
producers = {}
@topic_retry_decorator
def send(topic_name, data, **kargs):
if topic_name not in producers.keys():
producers[topic_name] = Producer(
topic_config._replace(name=topic_name)
)
producers[topic_name].send(data, **kargs)
yield send
for p in producers.values():
p.close()
def start_session(self):
if not is_authenticated(current_user):
return redirect(url_for("security.login", next=request.url))
else:
session_id = self.get_session_id()
self.sessions[session_id] = {
"log_reader":
LogReader(use_tls=getenv(
"KAFKA_USE_TLS",
type=bool,
default=True,
group_id=session_id,
)),
"last_seen":
datetime.datetime.utcnow(),
}
return json.dumps({
"status_code": 200,
"message": "The session has been started."
})
def run_supervisor(name, window, wakeup):
server = Redis()
queue = Queue(name=getenv(
"KIRBY_TOPIC_JOB_OFFERS", type=str, default=".kirby.job-offers"))
scheduler = Scheduler(queue=queue, wakeup=wakeup)
with Election(identity=name, server=server, check_ttl=window) as me:
while True:
checkpoint = perf_counter()
if me.is_leader():
content = scheduler.fetch_jobs()
if content is not None:
jobs = scheduler.parse_jobs(content)
for job in jobs:
scheduler.queue_job(job)
else:
logger.debug("not the leader, do nothing")
drift = perf_counter() - checkpoint
next_wakeup = wakeup - drift
logger.debug("waking up in {:.2f}s".format(next_wakeup))
sleep(next_wakeup)
def kirby_topic_factory(kafka_topic_factory):
import logging
logger = logging.getLogger(__name__)
bootstrap_servers = getenv("KAFKA_BOOTSTRAP_SERVERS",
type=list,
separator=",")
@contextmanager
def create_kirby_topic(topic_name, *args, timeout_ms=1500, **kargs):
if bootstrap_servers:
kargs.update(
use_tls=getenv("KAFKA_USE_TLS", type=bool, default=True))
with kafka_topic_factory(topic_name, timeout_ms=timeout_ms):
with Topic(topic_name, *args, **kargs) as kirby_topic:
yield kirby_topic
else:
logger.warning(f"There is no KAFKA_BOOTSTRAP_SERVERS. "
"Topic will be created in testing mode")
kargs.update(testing=True)
with Topic(topic_name, *args, **kargs) as kirby_topic:
yield kirby_topic
yield create_kirby_topic
def makeDataCards( analysis, inSamples, channels, folderDetails, **kwargs ) :
assert( type(inSamples) == type(OrderedDict())
or type(inSamples) == type({}) ), "Provide a samples list which \
is a dict or OrderedDict"
# Get expanded list of samples with gen appended names
samples = dataCardGenMatchedSamples( inSamples )
#for key in samples :
# print key, samples[key]
ops = {
'useQCDMakeName' : 'x',
'qcdSF' : 1.0,
'mssm' : False,
'category' : 'inclusive',
'fitShape' : 'm_visCor',
'sync' : False,
'redBkg' : False,
'allShapes' : False,}
for key in kwargs :
#print "another keyword arg: %s: %s" % (key, kwargs[key])
if key in ops.keys() :
ops[key] = kwargs[key]
print ops
ROOT.gROOT.SetBatch(True)
tdr.setTDRStyle()
with open('meta/NtupleInputs_%s/samples.json' % analysis) as sampFile :
sampDict = json.load( sampFile )
# Change or alter samples based on flags
if analysis == 'azh' :
samples['ZZ4l'] = 'ZZ'
for era in eras :
samples['RedBkgShape-%s' % era] = 'RedBkg'
# Use FF built QCD backgrounds
doFF = getenv('doFF', type=bool)
# value saved to renormalize the FF shape uncertainties
jetFakesNomYield = -999.
if doFF :
eras = []
for s in samples :
if 'dataTT' in s : eras.append( s.split('-').pop() )
if 'QCD' in samples.keys() :
del samples['QCD']
for era in eras :
samples['QCD-%s' % era] = 'jetFakes'
# Build list of names/samples which will be used
# for a given final channel. This differs for some HTT
# channels
nameArray = []
for samp in samples :
if samples[samp] not in nameArray : nameArray.append( samples[samp] )
# the shape names are changed to reflect the fact that they have the
# jet -> tau fakes removed
if doFF :
ffRenameMap = {
'W' : 'W_rest',
'ZJ' : 'ZJ_rest',
'TTJ' : 'TTJ_rest',
'VVJ' : 'VVJ_rest'
}
for samp, val in samples.iteritems() :
if val in ffRenameMap.keys() :
print "Val in ffRenameMap", val
samples[samp] = ffRenameMap[ val ]
if val in nameArray :
nameArray.remove( val )
nameArray.append( ffRenameMap[ val ] )
print "Samples to use and their mapping"
for key in samples :
print key, samples[key]
extra = ''
checkDir( '%sShapes' % analysis )
if ops['mssm'] :
checkDir( '%sShapes/mssm' % analysis )
extra = 'mssm'
if analysis == 'azh' :
extra = 'azh'
else :
checkDir( '%sShapes/htt' % analysis )
extra = 'htt'
for channel in channels :
if channel == 'tt' :
for sample in samples.keys() :
if '-ZLL' in sample :
del samples[ sample ]
if 'ZLL' in nameArray : nameArray.remove('ZLL')
#if channel == 'em' :
# for sample in samples.keys() :
# if sample[-3:] == '-ZL' or '-ZJ' in sample :
# del samples[ sample ]
# if 'ZJ' in nameArray : nameArray.remove('ZJ')
# if 'ZL' in nameArray : nameArray.remove('ZL')
print "Name Array"
print nameArray
print channel
newVarMapUnsorted = analysisPlots.getHistoDict( analysis, channel )
newVarMap = returnSortedDict( newVarMapUnsorted )
baseVar = ops['fitShape']
#if 'data' in sample : print "Fitting",baseVar
appendMap = {
'm_sv' : 'svFitMass2D',
'pt_sv:m_sv' : 'svFitMass2D',
'mjj:m_sv' : 'svFitMass2D',
'm_visCor' : 'visMass2D',
'Higgs_PtCor:m_sv' : 'svFitMass2D',
'Higgs_PtCor:m_visCor' : 'visMass2D',
'mjj:m_visCor' : 'visMass2D',
'Mass' : '4LMass',
}
#if '0jet2D' in ops['category'] :
# if ops['fitShape'] == 'm_sv' :
# Append = '_svFitMass2D'
# else :
# Append = '_visMass2D'
#else :
# Append = '_'+appendMap[baseVar]
Append = '_'+appendMap[baseVar]
if ops['mssm'] :
# if not var == baseVar+'_mssm' : continue
mid = 'mssm'
else :
mid = 'sm'
nameChan = 'tt' if analysis != 'azh' else 'zh'
shapeFile = ROOT.TFile('%sShapes/%s/htt_%s.inputs-%s-13TeV%s.root' % (analysis, extra, nameChan, mid, Append), 'UPDATE')
# We have two pathways to create tt_0jet and need to maintain their seperate root files for 1D vs 2D
# so we need this override that renames 0jet2D -> 0jet and places in the unrolled root file
if '0jet2D' in ops['category'] :
cr = '_qcd_cr' if '_qcd_cr' in ops['category'] else ''
shapeDir = shapeFile.mkdir( channel + '_0jet'+cr, channel + '_0jet'+cr )
else :
shapeDir = shapeFile.mkdir( channel + '_%s' % ops['category'], channel + '_%s' % ops['category'] )
assert( shapeDir != None ), "It looks like the directory already exists, remove the old root file and start again: rm httShapes/htt/htt_tt.inputs-sm-13TeV_ ..."
for var in newVarMap.keys() :
print var
if not baseVar in var : continue
if ops['fitShape'] == 'm_sv' and ':' in var : continue # Get rid of the 2D shapes in 0jet
if ops['fitShape'] == 'm_visCor' and ':' in var : continue # Get rid of the 2D shapes in 0jet
print "\n\n=============================================================="
if ops['allShapes'] :
print "All Shapes Applied: %s" % var
if doFF :
if not (('_energyScale' in var) or ('_zPt' in var) or \
('ffSyst' in var) or ('ffStat' in var) or ('_topPt' in var) or \
('_metUnclustered' in var) or ('_metClustered' in var) \
or ('_JES' in var) or ('_JetToTau' in var) or \
('_Zmumu' in var) or ('_ggH' in var) or ('_topQuarkggH' in var) \
or ('_ffSub' in var) or (baseVar == var)) :
continue
else :
if not (('_energyScale' in var) or ('_zPt' in var) or ('_topPt' in var) \
or ('_JES' in var) or ('_ggH' in var) or ('_topQuarkggH' in var) or ('_JetToTau' in var) \
or ('_metUnclustered' in var) or ('_metClustered' in var) \
or ('_Zmumu' in var) or ('_tauPt' in var) or (baseVar == var)) :
print "Did we fail?"
continue
else :
if not var == baseVar : continue
# Defined out here for large scope
print "\nVar: ",var
binArray = array( 'd', [] )
#if ops['mssm'] :
# #if doBTagging == True :
# if 'ZTT' in folderDetails :
# print "Inclusive"
# binArray = array( 'd', [0,10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160,170,180,190,200,225,250,275,300,325,350,400,500,700,900,1100,1300,1500,1700,1900,2100,2300,2500,2700,2900,3100,3300,3500,3700,3900] )
# elif 'NoBTL' in ops['folderDetails'] :
# print "No-BTAGGING"
# binArray = array( 'd', [0,10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160,170,180,190,200,225,250,275,300,325,350,400,500,700,900,1100,1300,1500,1700,1900,2100,2300,2500,2700,2900,3100,3300,3500,3700,3900] )
# elif 'NoBTL' not in ops['folderDetails'] :
# print "BTAGGING"
# binArray = array( 'd', [0,20,40,60,80,100,120,140,160,180,200,250,300,350,400,500,700,900,1100,1300,1500,1700,1900,2100,2300,2500,2700,2900,3100,3300,3500,3700,3900] )
#elif var == 'mt_tot' :
# binArray = array( 'd', [0.0,10.0,20.0,30.0,40.0,50.0,60.0,70.0,\
# 80.0,90.0,100.0,110.0,120.0,130.0,140.0,150.0,160.0,\
# 170.0,180.0,190.0,200.0,225.0,250.0,275.0,300.0,325.0,\
# 350.0,400.0,500.0,700.0,900.0, 1100.0,1300.0,1500.0,\
# 1700.0,1900.0,2100.0,2300.0,2500.0,2700.0,2900.0,3100.0,\
# 3300.0,3500.0,3700.0,3900.0] )
if ops['sync'] :
binArray = array( 'd', [i*20 for i in range( 11 )] )
# This is the proposed binning for ZTT 2015 paper
elif doFF and ('m_sv' in var or 'm_visCor' in var) :
if ":" in var : binArray = array( 'd', [i for i in range( 49 )] )
else : binArray = array( 'd', [i*10 for i in range( 31 )] )
else :
if ":" in var : binArray = array( 'd', [i for i in range( 49 )] )
elif ops['category'] in ['1jet_low', '1jet_high'] :
binArray = array( 'd', [0,40,60,70,80,90,100,110,120,130,150,200,250] )
elif 'vbf' in ops['category'] :
binArray = array( 'd', [0,40,60,80,100,120,150,200,250] )
else :
#binArray = array( 'd', [i*10 for i in range( 31 )] )
binArray = array( 'd', [0,50,60,70,80,90,100,110,120,130,\
140,150,160,170,180,190,200,210,220,230,240,250,\
260,270,280,290,300] )
numBins = len( binArray ) - 1
#print binArray
#print numBins
histos = OrderedDict()
for name in nameArray :
title = name
if ops['allShapes'] :
if '_' in var and ('Up' in var or 'Down' in var) :
systName = var.split('_')[-1]
histos[ name ] = ROOT.TH1D( name+systName, name+systName, numBins, binArray )
else :
histos[ name ] = ROOT.TH1D( name, name, numBins, binArray )
else :
histos[ name ] = ROOT.TH1D( name, name, numBins, binArray )
histos[ name ].Sumw2()
for sample in samples:
''' Skip plotting unused shape systematics '''
if skipSystShapeVar( var, sample, channel ) : continue
if '_topPt' in var : print "Top Pt still in Var: "+var+" sample: "+sample
# Skip looping over nonsense channel / sample combos
if skipChanDataCombo( channel, sample, analysis ) : continue
#if sample == 'DYJetsLow' : continue
#if 'HtoTauTau' in sample : continue
#print sample
if sample == 'dataEM' :
tFile = ROOT.TFile('%s%s/%s_em.root' % (analysis, folderDetails, sample), 'READ')
elif 'dataTT' in sample :
tFile = ROOT.TFile('%s%s/%s_tt.root' % (analysis, folderDetails, sample), 'READ')
elif 'QCD' in sample :
if ops['useQCDMakeName'] != 'x' :
print "Use QCD MAKE NAME: ",ops['useQCDMakeName']
tFile = ROOT.TFile('meta/%sBackgrounds/%s_qcdShape_%s.root' % (analysis, channel, ops['useQCDMakeName']), 'READ')
elif doFF :
tFile = ROOT.TFile('%s%s/%s_%s.root' % (analysis, folderDetails, sample, channel), 'READ')
#print " \n### Using Fake Factor QCD Shape !!! ###\n"
else :
print " \n\n ### SPECIFY A QCD SHAPE !!! ### \n\n"
elif ops['redBkg'] and 'RedBkgShape' in sample :
tFileYield = ROOT.TFile('%s%s/%s_%s.root' % (analysis, folderDetails, sample.replace('Shape','Yield'), channel), 'READ')
tFile = ROOT.TFile('%s%s/%s_%s.root' % (analysis, folderDetails, sample, channel), 'READ')
else :
tFile = ROOT.TFile('%s%s/%s_%s.root' % (analysis, folderDetails, sample, channel), 'READ')
dic = tFile.Get("%s_Histos" % channel )
if not doFF :
hist = dic.Get( "%s" % var )
if doFF :
if 'QCD' in sample :
hist = dic.Get( "%s_ffSub" % var )
else :
hist = dic.Get( "%s" % var )
# This is where we subtract off the fakes from MC from jetFakes
if '-ZJ' in sample or '-TTJ' in sample or 'WJets' in sample or '-VVJ' in sample :
if not '_ffSub' in var :
ffSubHist = dic.Get( var+'_ffSub' )
#print sample," FF Sub int",ffSubHist.Integral()
if ":" in var :
ffSubHist2 = unroll2D( ffSubHist )
else :
ffSubHist2 = ffSubHist.Rebin( numBins, "rebinned", binArray )
ffSubHist2.GetXaxis().SetRangeUser( binArray[0], binArray[-1] )
#if "DYJets" in sample and "ZTT" in sample :
# ffSubHist2.Scale( zttScaleTable[ops['category']] )
histos[ 'jetFakes' ].Add( ffSubHist2, -1.0 )
hist.SetDirectory( 0 )
#print "Hist yield before scaling ",hist.Integral()
""" Scale reducible bkg shape by yield estimate """
if ops['redBkg'] and 'RedBkgShape' in sample :
redBkgYield = tFileYield.Get('%s_Histos/%s' % (channel, var)).Integral()
if hist.Integral() != 0 :
hist.Scale( redBkgYield / hist.Integral() )
''' Scale Histo based on cross section ( 1000 is for 1 fb^-1 of data ),
QCD gets special scaling from bkg estimation, see qcdYield[channel] above for details '''
#print "PRE Sample: %s Int: %f" % (sample, hist.Integral() )
if sample == 'QCD' and ops['useQCDMakeName'] != 'x' :
#print "Using QCD SCALE FACTOR <<<< NEW >>>>"
qcdScale = ops['qcdSF']
assert( qcdScale > 0. ), "\nQCD Scale is wrong, you probably need to rerun all channels together\n"
print "Skip rebin; Scale QCD shape by %f" % qcdScale
#print "QCD yield Pre: %f" % hist.Integral()
hist.Scale( qcdScale )
#print "QCD yield Post Scale: %f" % hist.Integral()
#if 'QCD' not in sample :
# #hist.Rebin( 10 )
# #print "hist # bins pre: %i" % hist.GetXaxis().GetNbins()
# hNew = hist.Rebin( numBins, "new%s" % sample, binArray )
# #print "hist # bins post: %i" % hNew.GetXaxis().GetNbins()
# histos[ samples[ sample ] ].Add( hNew )
#else :
# #print "hist # bins pre: %i" % hist.GetXaxis().GetNbins()
# hNew = hist.Rebin( numBins, "new%s" % sample, binArray )
# #print "hist # bins post: %i" % hNew.GetXaxis().GetNbins()
# histos[ samples[ sample ] ].Add( hNew )
if ":" in var :
hNew = unroll2D( hist )
#print "nbinsX",hNew.GetNbinsX() ,hNew.GetBinLowEdge(1),hNew.GetBinLowEdge( hNew.GetNbinsX()+1 )
else :
hNew = hist.Rebin( numBins, "new%s" % sample, binArray )
# If using the qcd CR, we want a single bin for all
if '_qcd_cr' in ops['category'] :
nBins = hNew.GetNbinsX()
hNew.Rebin( nBins )
# If histos haven't been Rebinned, do it
nBinsHistos = histos[ samples[ sample ] ].GetNbinsX()
if nBins == nBinsHistos :
histos[ samples[ sample ] ].Rebin( nBins )
histos[ samples[ sample ] ].Add( hNew )
#XXX LOTS OF PRINT OUTS if ops['mssm'] and not 'ggH_htt' in sample and not 'bbH' in sample :
#XXX LOTS OF PRINT OUTS print "SampleName: %20s Hist yield %.2f" % (sample, hist.Integral())
#XXX LOTS OF PRINT OUTS else :
#XXX LOTS OF PRINT OUTS print "SampleName: %20s Hist yield %.2f" % (sample, hist.Integral())
#hist2 = hist.Rebin( 18, 'rebinned', binArray )
#histos[ samples[ sample ] ].Add( hist2 )
tFile.Close()
# All files looped through, now renormalize jetFakes
# background if doing Fake Factor method
if doFF :
if var == baseVar :
jetFakesNomYield = histos[ 'jetFakes' ].Integral()
print " -- FF Normalization: var == basevar %s %s yield: %.2f" % (baseVar,var,jetFakesNomYield)
elif ('ffSyst' in var or 'ffStat' in var) :
assert( jetFakesNomYield > 0. ), "Why didn't you loop over the base variable yet?"
preYield = histos[ 'jetFakes' ].Integral()
histos[ 'jetFakes' ].Scale( jetFakesNomYield / preYield )
postYield = histos[ 'jetFakes' ].Integral()
print " -- FF Normalization: var is a ff shape: %s %s pre: %.2f post: %.2f" % (baseVar,var,preYield,postYield)
print ".............................................................."
shapeDir.cd()
# We need to check for any total bkg bins which have 0
# Recall ROOT TH1 bin numbers start at 1 for the useful bins
# Don't worry about this in QCD CR
setVal = 0.00001
if not '_qcd_cr' in ops['category'] :
#dataArray = []
bkgArray = [0] * numBins
for name in histos :
if 'ggH_htt' in name or 'qqH_htt' in name or 'ZH_htt' in name or 'WH_htt' in name :
continue # we don't care about signal here
elif name == 'data_obs' : continue # can use this later if need be
# for bin_id in range( histos[ name ].GetNbinsX() ) :
# dataArray.append( histos[ name ].GetBinContent( bin_id+1 ) )
else : # All backgrounds
for bin_id in range( histos[ name ].GetNbinsX() ) :
bkgArray[bin_id] = bkgArray[bin_id] + histos[ name ].GetBinContent( bin_id+1 )
#print dataArray
#print bkgArray
#assert( len(dataArray) == len(bkgArray) ), "Zero bin check is not working, are you missing data_obs?"
# Find problem bins
problemBins = []
print "Checking for problem bins"
for bin_id in range( len(bkgArray) ) :
#if dataArray[bin_id] > 0. and bkgArray[bin_id] == 0. :
print bin_id+1, bkgArray[bin_id], " QCD val: ",histos[ 'QCD' ].GetBinContent( bin_id+1 )
if bkgArray[bin_id] == 0. :
problemBins.append( bin_id+1 ) # +1 here gets us to ROOT coords
# Apply correction and set an empyt bin to QCD = 1e-5
if 'QCD' in histos.keys() :
for problemBin in problemBins :
print "Setting QCD bin_id %i to %s" % (problemBin, setVal)
histos[ 'QCD' ].SetBinContent( problemBin, setVal )
# Poissonian error for 0
histos[ 'QCD' ].SetBinError( problemBin, 1.8 )
elif len(problemBins) > 0 :
print "\nQCD Bkg not included so zero bins are not being corrected properly"
print problemBins
print "\n\n\n"
# Check QCD
#for bin_id in range( 1, histos[ 'QCD' ].GetNbinsX()+1 ) :
# print bin_id, histos[ 'QCD' ].GetBinContent( bin_id )
for name in histos :
#print "name: %s Yield Pre: %f" % (name, histos[ name ].Integral() )
# First, if we are doing Fake Factor and jetFakes is negative
# zero it
if histos[ name ].Integral() < 0.0 and name == 'jetFakes' :
for bin_ in range( 1, histos[ name ].GetXaxis().GetNbins()+1 ) :
histos[ name ].SetBinContent( bin_, setVal )
# Make sure we have no negative bins
for bin_ in range( 1, histos[ name ].GetXaxis().GetNbins()+1 ) :
if name == 'QCD' : # Set all QCD 0.0 and negative vals to 1e-5
if histos[ name ].GetBinContent( bin_ ) == 0. :
histos[ name ].SetBinContent( bin_, setVal )
# Poissonian error for 0
histos[ name ].SetBinError( bin_, 1.8 )
print "name: %s Set bin %i to value: %s" % (name, bin_, setVal)
elif histos[ name ].GetBinContent( bin_ ) < 0. :
# Don't change the uncertainty on the bin
# it was set by the negative content, leave it as is
histos[ name ].SetBinContent( bin_, setVal )
print "name: %s Set bin %i to value: %s" % (name, bin_, setVal)
if histos[ name ].GetBinContent( bin_ ) < 0 :
# Don't change the uncertainty on the bin
# it was set by the negative content, leave it as is
histos[ name ].SetBinContent( bin_, setVal )
print "name: %s Set bin %i to value: %f" % (name, bin_, setVal)
if histos[ name ].Integral() != 0.0 :
print "DataCard Name: %10s Yield Post: %.2f" % (name, histos[ name ].Integral() )
#if not ops['mssm'] :
# histos[ name ].GetXaxis().SetRangeUser( 0, 350 )
# Proper naming of output histos
if ops['allShapes'] and ('_energyScale' in var or '_tauPt' in var or '_zPt' in var \
or '_JES' in var or '_topPt' in var or '_ggH' in var or '_JetToTau' in var or '_Zmumu' in var \
or '_metUnclustered' in var or '_metClustered' in var or '_topQuarkggH' in var \
or 'ffSyst' in var or 'ffStat' in var) :
# Systematics naming removes CRs
category = ops['category'].strip('_qcd_cr')
if name in ['data_obs','QCD'] : continue
if name == 'jetFakes' and not doFF : continue
if name == 'jetFakes' and not ('ffSyst' in var or 'ffStat' in var) : continue
if ('ffSyst' in var or 'ffStat' in var) and name != 'jetFakes' : continue
if '_ggH' in var and not name in ['ggH_htt110', 'ggH_htt120','ggH_htt125','ggH_htt130', 'ggH_htt140'] : continue
if '_topQuarkggH' in var and not name in ['ggH_htt110', 'ggH_htt120','ggH_htt125','ggH_htt130', 'ggH_htt140'] : continue
if '_JetToTau' in var and not name in ['W', 'TTJ', 'ZJ', 'VVJ',
'VVJ_rest', 'W_rest', 'TTJ_rest', 'ZJ_rest'] : continue
if '_Zmumu' in var and (name not in ['ZTT', 'ZL', 'ZJ', 'ZJ_rest', 'EWKZ'] or \
category != 'vbf') : continue # Shape only used in vbf category atm
lep = 'x'
if channel == 'tt' : lep = 't'
if channel == 'em' : lep = 'e'
shiftDir = ''
shiftVar = var.replace('_ffStat','').replace('_ffSyst','')
#print "shiftVar1:",shiftVar
if shiftVar[-2:] == 'Up' : shiftDir = 'Up'
if shiftVar[-4:] == 'Down' : shiftDir = 'Down'
assert( shiftDir == 'Up' or shiftDir == 'Down' ), "Is var a +/- shift? %s" % var
# JES Breakdown
if '_JES' in var :
jesUnc = var.split('_')[-1]
jesUnc = jesUnc.replace('JES', '')
if 'Up' in jesUnc[-2:] : jesUnc = jesUnc[:-2]
if 'Down' in jesUnc[-4:] : jesUnc = jesUnc[:-4]
if jesUnc == '' : jesUnc = '13TeV'+shiftDir
# Keep a normal shift included for checks at Combine level
elif 'Total' in jesUnc and 'Sub' not in jesUnc : jesUnc = '13TeV'+shiftDir
else : jesUnc += '_13TeV'+shiftDir
# For naming conventions for systematics
if category == 'vbf' : tmpCat = 'VBF'
else : tmpCat = category
if '_zPt' in var :
if name not in ['ZTT','ZL','ZJ','ZLL','ZJ_rest'] : continue
elif '_zPt' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_dyShape_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_dyShape_13TeV'+shiftDir )
elif '_topPt' in var :
if name not in ['TTT','TTJ','TTJ_rest'] : continue
elif '_topPt' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_ttbarShape_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_ttbarShape_13TeV'+shiftDir )
#elif name in ['TTT','TTJ'] : continue # this is to catch TT when it's not wanted
elif '_energyScaleAll' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_scale_'+lep+'_'+channel+'_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_scale_'+lep+'_'+channel+'_13TeV'+shiftDir )
altName = histos[ name ].Clone( name.strip('_')+'_CMS_scale_'+lep+'_13TeV'+shiftDir )
altName.SetTitle( name.strip('_')+'_CMS_scale_'+lep+'_13TeV'+shiftDir )
altName.Write()
del altName
elif '_energyScaleDM0' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_scale_'+lep+'_1prong_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_scale_'+lep+'_1prong_13TeV'+shiftDir )
elif '_energyScaleDM10' in var : # Gotta do this one first to elif DM10 and DM1
histos[ name ].SetTitle( name.strip('_')+'_CMS_scale_'+lep+'_3prong_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_scale_'+lep+'_3prong_13TeV'+shiftDir )
elif '_energyScaleDM1' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_scale_'+lep+'_1prong1pizero_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_scale_'+lep+'_1prong1pizero_13TeV'+shiftDir )
elif '_JES' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_scale_j_'+jesUnc )
histos[ name ].SetName( name.strip('_')+'_CMS_scale_j_'+jesUnc )
elif '_JetToTau' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_jetToTauFake_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_jetToTauFake_13TeV'+shiftDir )
elif '_tauPt' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_eff_t_High_'+channel+'_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_eff_t_High_'+channel+'_13TeV'+shiftDir )
elif '_ggH' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_scale_gg_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_scale_gg_13TeV'+shiftDir )
elif '_topQuarkggH' in var :
histos[ name ].SetTitle( name.strip('_')+'_TopMassTreatment_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_TopMassTreatment_13TeV'+shiftDir )
elif '_Zmumu' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_zmumuShape_'+tmpCat+'_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_zmumuShape_'+tmpCat+'_13TeV'+shiftDir )
elif '_metClustered' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_scale_met_clustered_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_scale_met_clustered_13TeV'+shiftDir )
elif '_metUnclustered' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_scale_met_unclustered_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_scale_met_unclustered_13TeV'+shiftDir )
### For these Fake Factor shapes, we need 2 copies with slightly different names
elif 'ffSyst' in var :
if 'qcdffSyst' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_norm_ff_qcd_tt_syst_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_norm_ff_qcd_tt_syst_13TeV'+shiftDir )
elif 'ttbarffSyst' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_norm_ff_ttbar_tt_syst_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_norm_ff_ttbar_tt_syst_13TeV'+shiftDir )
elif 'wjetsffSyst' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_norm_ff_w_tt_syst_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_norm_ff_w_tt_syst_13TeV'+shiftDir )
else : assert (2 + 2 == 5), "This shouldn't happen. Problem in your FF shapes."
elif 'ffStat' in var :
if '0jet1prongffStat' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_norm_ff_qcd_1prong_njet0_tt_stat_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_norm_ff_qcd_1prong_njet0_tt_stat_13TeV'+shiftDir )
elif '0jet3prongffStat' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_norm_ff_qcd_3prong_njet0_tt_stat_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_norm_ff_qcd_3prong_njet0_tt_stat_13TeV'+shiftDir )
elif '1jet1prongffStat' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_norm_ff_qcd_1prong_njet1_tt_stat_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_norm_ff_qcd_1prong_njet1_tt_stat_13TeV'+shiftDir )
elif '1jet3prongffStat' in var :
histos[ name ].SetTitle( name.strip('_')+'_CMS_htt_norm_ff_qcd_3prong_njet1_tt_stat_13TeV'+shiftDir )
histos[ name ].SetName( name.strip('_')+'_CMS_htt_norm_ff_qcd_3prong_njet1_tt_stat_13TeV'+shiftDir )
else : assert (2 + 2 == 5), "This shouldn't happen. Problem in your FF shapes."
histos[ name ].Write()
else :
histos[ name ].SetTitle( name.strip('_') )
histos[ name ].SetName( name.strip('_') )
histos[ name ].Write()
shapeFile.Close()
print "\n Output shapes file: %sShapes/%s/htt_%s.inputs-%s-13TeV%s.root \n" % (analysis, extra, nameChan, mid, Append)
# -*- coding: utf-8 -*-
from smart_getenv import getenv
DEBUG = getenv('DEBUG', default=False, type=bool)
PROJECT_NAME = LOGGER_NAME = 'graphene_boilerplate'
SECRET_KEY = getenv('SECRET_KEY', default='testsecretkey')
SQLALCHEMY_DATABASE_URI = getenv(
'SQLALCHEMY_DATABASE_URI',
default=
'mysql+pymysql://root:@localhost:3306/graphene_boilerplate?charset=utf8mb4'
)
SQLALCHEMY_TRACK_MODIFICATIONS = getenv('SQLALCHEMY_TRACK_MODIFICATIONS',
default=True,
type=bool)
# -*- coding: utf-8 -*-
from smart_getenv import getenv
# Turn this off on production
DEBUG = getenv('DEBUG', type=bool, default=False)
# This is mandatory. Please define a secret key - random sequence of characters
SECRET_KEY = getenv('SECRET_KEY', default='')
SQLALCHEMY_DATABASE_URI = '{schema}://{user}:{pwd}@{host}/{dbname}'.format(
schema=getenv('DB_SCHEMA', default='sqlite'),
user=getenv('DB_USER', default=''),
pwd=getenv('DB_PASS', default=''),
host=getenv('DB_HOST', default=''),
dbname=getenv('DB_NAME', default=''))
SQLALCHEMY_BINDS = {
'factsheet':
'{schema}://{user}:{pwd}@{host}/{bindname}'.format(
schema=getenv('DB_SCHEMA', default='sqlite'),
user=getenv('DB_USER', default=''),
pwd=getenv('DB_PASS', default=''),
host=getenv('DB_HOST', default=''),
bindname=getenv('BIND_NAME', default=''))
}
SQLALCHEMY_TRACK_MODIFICATIONS = False
COLLECT_STATIC_ROOT = getenv('STATIC_ROOT', default='/static')
COLLECT_STORAGE = 'flask_collect.storage.file'
import os
from corsheaders.defaults import default_headers, default_methods
from smart_getenv import getenv
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.getenv('SECRET_KEY', '{{ secret_key }}')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = getenv('DEBUG', type=bool, default=True)
ALLOWED_HOSTS = getenv('ALLOWED_HOSTS', type=list, default=['*'])
# If the app is running behind a proxy, this variable must be set with the proxy path
# See https://docs.djangoproject.com/en/{{ docs_version }}/ref/settings/#force-script-name
FORCE_SCRIPT_NAME = os.getenv('PROXY_SCRIPT_NAME', None)
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
# -*- coding: utf-8 -*-
from smart_getenv import getenv
LOGGER_NAME = 'app'
# mysql
SQLALCHEMY_POOL_SIZE = 5
SQLALCHEMY_MAX_OVERFLOW = 0
SQLALCHEMY_POOL_TIMEOUT = 10
SQLALCHEMY_POOL_RECYCLE = 2000
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_DATABASE_URI = getenv('SQLALCHEMY_DATABASE_URI')
# secret key
SECRET_KEY = getenv('SECRET_KEY', default='test123')
try:
from .local_config import * # noqa
except ImportError:
pass
def makeLotsOfPlots( analysis, samples, channels, folderDetails, **kwargs ) :
ops = {
'qcdMakeDM' : 'x',
'useQCDMakeName' : 'x',
'isSSQCD' : False,
'addUncert' : True,
'qcdMC' : False,
'qcdSF' : 1.0,
'ratio' : True,
'blind' : True,
'text' : False,
'mssm' : False,
'log' : False,
'sync' : False,
'redBkg' : False,
'targetDir' : ''}
'''python analysis3Plots.py --folder=2June26_OSl1ml2_VTight_ZTT --channel=tt --text=True --useQCDMake=True --useQCDMakeName=OSl1ml2_VTight_LooseZTT --qcdSF=0.147 --btag=False'''
for key in kwargs :
#print "another keyword arg: %s: %s" % (key, kwargs[key])
if key in ops.keys() :
ops[key] = kwargs[key]
print ops
# Use FF built QCD backgrounds
doFF = getenv('doFF', type=bool)
vv = ['WW1l1nu2q', 'WW2l2nu', 'WZ1l1nu2q', 'WZ1l3nu',
'WZ2l2q', 'WZ3l1nu', 'ZZ2l2nu', 'ZZ2l2q', 'ZZ4l',
'VV', 'WWW', 'ZZZ', 'T-tW', 'T-tchan', 'Tbar-tW', 'Tbar-tchan']
""" Add in the gen matched DY catagorization """
if analysis == 'htt' :
genList = ['ZTT', 'ZLL', 'ZL', 'ZJ']
dyJets = ['DYJetsAMCNLO', 'DYJets', 'DYJets1', 'DYJets2', 'DYJets3', 'DYJets4', 'DYJetsLow']
newSamples = {}
for sample in samples.keys() :
#print sample
if sample in dyJets :
for gen in genList :
#print gen, sample+'-'+gen
samples[ sample+'-'+gen ] = deepcopy(samples[ sample ])
genApp = gen.lower()
samples[ sample+'-'+gen ]['group'] = genApp
if sample == 'TT' :
for gen in ['TTT', 'TTJ'] :
samples[ sample+'-'+gen ] = deepcopy(samples[ sample ])
genApp = 'ttbar'
samples[ sample+'-'+gen ]['group'] = genApp
if sample in vv :
for gen in ['VVT', 'VVJ'] :
samples[ sample+'-'+gen ] = deepcopy(samples[ sample ])
genApp = 'dib'
samples[ sample+'-'+gen ]['group'] = genApp
# Clean the samples list
if analysis == 'htt' and 'TT' in samples.keys() :
del samples[ 'TT' ]
for dyJet in dyJets :
if dyJet in samples.keys() :
del samples[ dyJet ]
for vvSamp in vv :
if vvSamp in samples.keys() :
del samples[ vvSamp ]
if not doFF :
samples[ 'QCD' ] = {'xsec' : 0.0, 'group' : 'qcd' }
# Don't plot sm higgs 120, 130
smMassesNoInclude = ['110', '120', '130', '140']
smHiggs = []
for mass in smMassesNoInclude :
smHiggs.append('ggHtoTauTau%s' % mass)
smHiggs.append('VBFHtoTauTau%s' % mass)
smHiggs.append('WMinusHTauTau%s' % mass)
smHiggs.append('WPlusHTauTau%s' % mass)
smHiggs.append('ZHTauTau%s' % mass)
for higgs in smHiggs :
if higgs in samples : del samples[ higgs ]
for sample in samples :
print sample
print "Running over %s samples" % analysis
ROOT.gROOT.SetBatch(True)
tdr.setTDRStyle()
cmsLumi = float(os.getenv('LUMI'))/1000
print "Lumi = %.1f / fb" % cmsLumi
mssmMass = 250
azhMass = 350
mssmSF = 100
higgsSF = 10
if 'vbf_high' in ops['targetDir'] : higgsSF = 2.5
azhSF = .025
with open('meta/NtupleInputs_%s/samples.json' % analysis) as sampFile :
sampDict = json.load( sampFile )
chans = {
'tt' : '#tau_{h}#tau_{h}',
'em' : 'e#mu',
'eeet' : 'eee#tau_{h}',
'eemt' : 'ee#mu#tau_{h}',
'eett' : 'ee#tau_{h}#tau_{h}',
'eeem' : 'eee#mu',
'eeee' : 'eeee',
'eemm' : 'ee#mu#mu',
'emmt' : '#mu#mue#tau_{h}',
'mmmt' : '#mu#mu#mu#tau_{h}',
'mmtt' : '#mu#mu#tau_{h}#tau_{h}',
'emmm' : '#mu#mue#mu',
'mmmm' : '#mu#mu#mu#mu',
'ZEE' : 'Z#rightarrowee',
'ZMM' : 'Z#rightarrow#mu#mu',
'ZXX' : 'Z#rightarrowee/#mu#mu',
}
sampInfo = { 'htt' : {
'dib' : [ROOT.kRed+2, 'VV'],
'ttbar' : [ROOT.kBlue-8, 't#bar{t}'],
'qcd' : [ROOT.TColor.GetColor(250,202,255), 'QCD'], #kMagenta-10
'ztt' : [ROOT.TColor.GetColor(248,206,104), 'Z#rightarrow#tau#tau'], #kOrange-4,
'zl' : [ROOT.kAzure+2, 'Z#rightarrowee (lepton)'],
'zj' : [ROOT.kGreen+2, 'Z#rightarrowee (jet)'],
'zll' : [ROOT.TColor.GetColor(100,182,232), 'Z#rightarrowee'],
'wjets' : [ROOT.kAzure+6, 'WJets'],
'higgs' : [ROOT.kBlue, 'SM Higgs(125)'],
'VH' : [ROOT.kGreen, 'SM VHiggs(125)'],
#'mssm' : [ROOT.kPink, 'MSSM(%i) x %i' % (mssmMass, mssmSF)],
'obs' : [ROOT.kBlack, 'Data'],
}, # htt
'azh' : {
'obs' : [ROOT.kBlack, 'Data'],
'zz' : [ROOT.kGreen-9, 'ZZ'],
'wz' : [ROOT.kRed-4, 'WZ'],
'dyj' : [ROOT.TColor.GetColor(248,206,104), 'ZJets'],
'top' : [ROOT.kBlue-8, 't#bar{t}'],
'redBkg' : [ROOT.kCyan, 'Reducible Bkg.'],
#'sm' : [ROOT.kGreen, 'SM Higgs (125)'],
'VH' : [ROOT.kGreen, 'SM VHiggs(125)'],
'azh' : [ROOT.kBlue, 'A#rightarrowZh M%s #sigma=%.3fpb' % (azhMass, azhSF)],
} # azh
} # sampInfo
if not ops['mssm'] : sampInfo['htt']['higgs'][1] = "SM Higgs(125) x %.1f" % higgsSF
# Make signal variable for later easy mapping
signal = ''
if analysis == 'htt' :
signal = 'higgs'
signalSF = higgsSF
if analysis == 'azh' :
#signal = 'azh'
#signalSF = azhSF
signal = 'VH'
signalSF = higgsSF
sampInfo['azh']['VH'][1] = "SM VHiggs(125) x %.1f" % higgsSF
if doFF :
sampInfo['htt']['jetFakes'] = [ROOT.TColor.GetColor(250,202,255), 'jetFakes'] #kMagenta-10
del sampInfo['htt']['qcd']
for channel in channels :
print channel
# Make an index file for web viewing
checkDir( '%sPlots/em' % analysis )
checkDir( '%sPlots/tt' % analysis )
checkDir( '%sPlotsList/em' % analysis )
checkDir( '%sPlotsList/tt' % analysis )
checkDir( '/afs/cern.ch/user/t/truggles/www/%sPlots/%s%s/' % (analysis, channel, ops['targetDir']))
checkDir( '/afs/cern.ch/user/t/truggles/www/%sPlotsList/%s%s/' % (analysis, channel, ops['targetDir']))
htmlFile = open('/afs/cern.ch/user/t/truggles/www/%sPlots/%s%s/index.html' % (analysis, channel, ops['targetDir']), 'w')
htmlFile.write( '<html><head><STYLE type="text/css">img { border:0px; }</STYLE>\n' )
htmlFile.write( '<title>Channel %s/</title></head>\n' % channel )
htmlFile.write( '<body>\n' )
newVarMapUnsorted = analysisPlots.getHistoDict( analysis, channel )
newVarMap = returnSortedDict( newVarMapUnsorted )
if doFF :
tmpDict = {}
for var, info in newVarMap.iteritems() :
tmpDict[var] = info
tmpDict[var+'_ffSub'] = list(info)
tmpDict[var+'_ffSub'][4] += ' FF Sub'
newVarMap = returnSortedDict( tmpDict )
finalQCDYield = 0.0
finalDataYield = 0.0
qcdMake = False
if ops['qcdMakeDM'] != 'x' :
qcdMake = True
finalQCDYield = 0.0
finalDataYield = 0.0
checkDir('meta/%sBackgrounds' % analysis)
print "qcdMakeDM called: ",ops['qcdMakeDM']
qcdMaker = ROOT.TFile('meta/%sBackgrounds/%s_qcdShape_%s_%s.root' % (analysis, channel, folderDetails.split('_')[0], ops['qcdMakeDM']), 'RECREATE')
qcdDir = qcdMaker.mkdir('%s_Histos' % channel)
#print newVarMap
for var, info in newVarMap.iteritems() :
# This is to speed up the Data Card making process by 2x and not
# create all the plots for SS when all we need it the yield from m_visCor
if ops['isSSQCD'] and not var == 'm_visCor' : continue
# speed up 2D plotting
if ":" in var :
# Skip 1 bin plot of 2D vars
if 'plotMe' in ops['qcdMakeDM'] : continue
elif 'vbf' in ops['qcdMakeDM'] :
if not ('mjj' in var or 'vbfMass' in var) : continue
elif 'boosted' in ops['qcdMakeDM'] :
if not ('pt_sv' in var or 'Higgs_Pt' in var) : continue
else : continue
#if 'mt_sv' in var : continue
print "Var:",var
"""
Handle variable binning and longer ranges for visible mass
"""
isVBFCat = False
is1JetCat = False
is0JetCat = False
# This was useful for lower stats sub-divided categories pre-unrolling
#if 'm_sv' in var or 'm_visCor' in var :
# if ('1jet_low' in ops['useQCDMakeName'] or '1jet_high' in ops['useQCDMakeName']\
# or '1jet_low' in ops['qcdMakeDM'] or '1jet_high' in ops['qcdMakeDM']) :
# is1JetCat = True
# if ('vbf' in ops['useQCDMakeName'] or 'vbf' in ops['qcdMakeDM']) :
# isVBFCat = True
# if not (isVBFCat or is1JetCat) : is0JetCat = True
varBinned = True
if '_mssm' in var :
if 'ZTT' in folderDetails :
print "Inclusive"
xBins = array( 'd', [0,10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160,170,180,190,200,225,250,275,300,325,350,400,500,700,900,1100,1300,1500,1700,1900,2100,2300,2500,2700,2900,3100,3300,3500,3700,3900] )
elif 'NoBTL' in folderDetails :
print "No-BTAGGING"
xBins = array( 'd', [0,10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160,170,180,190,200,225,250,275,300,325,350,400,500,700,900,1100,1300,1500,1700,1900,2100,2300,2500,2700,2900,3100,3300,3500,3700,3900] )
elif 'NoBTL' not in folderDetails :
print "BTAGGING"
xBins = array( 'd', [0,20,40,60,80,100,120,140,160,180,200,250,300,350,400,500,700,900,1100,1300,1500,1700,1900,2100,2300,2500,2700,2900,3100,3300,3500,3700,3900] )
elif var == 'mt_tot' :
xBins = array( 'd', [0.0,10.0,20.0,30.0,40.0,50.0,60.0,70.0,\
80.0,90.0,100.0,110.0,120.0,130.0,140.0,150.0,160.0,\
170.0,180.0,190.0,200.0,225.0,250.0,275.0,300.0,325.0,\
350.0,400.0,500.0,700.0,900.0, 1100.0,1300.0,1500.0,\
1700.0,1900.0,2100.0,2300.0,2500.0,2700.0,2900.0,3100.0,\
3300.0,3500.0,3700.0,3900.0] )
elif var == 'm_visCor_mssm' :
#xBins = array( 'd', [0,20,40,60,80,100,150,200,250,350,600,1000,1500,2000,2500,3500] )
xBins = array( 'd', [] )
for i in range(0, 351 ) :
xBins.append( i * 10 )
elif ops['sync'] and 'm_visCor' in var :
xBins = array( 'd', [] )
for i in range( 21 ) :
xBins.append( i * 17.5 )
# This is the proposed binning for ZTT 2015 paper
elif doFF and ('m_sv' in var or 'm_visCor' in var) :
xBins = array( 'd', [i*10 for i in range( 31 )] )
elif 'm_sv' in var or 'm_visCor' in var :
if is1JetCat :
xBins = array( 'd', [0,40,60,70,80,90,100,110,120,130,150,200,250] )
elif isVBFCat :
xBins = array( 'd', [0,40,60,80,100,120,150,200,250] )
else :
#xBins = array( 'd', [i*10 for i in range( 31 )] )
xBins = array( 'd', [0,50,60,70,80,90,100,110,120,130,\
140,150,160,170,180,190,200,210,220,230,240,250,\
260,270,280,290,300] )
else :
varBinned = False
first = info[1] * 1.
last = info[2] * 1.
totBins = ( info[0] / info[3] ) * 1.
binWidth = (last - first)/totBins
#print first, last, totBins, binWidth
xBins = array('d', [])
for i in range( 0, int(totBins)+1 ) :
if 'iso' in var :
xBins.append( round(i*binWidth+first,2) )
else :
xBins.append( round(i*binWidth+first,1) )
# Make a single bin version for QCD CR plots
if 'plotMe' in ops['qcdMakeDM'] :
xBins = array( 'd', [xBins[0], xBins[-1]] )
xNum = len( xBins ) - 1
#print "Binning scheme: ",xBins
append = var + channel
stack = ROOT.THStack("All Backgrounds stack", "%s, %s" % (channel, var) )
sampHistos = {}
if ":" not in var :
for samp in sampInfo[analysis].keys() :
# Skip some DY gen based combos
if analysis == 'htt' and channel == 'em' :
if samp in ['zl', 'zj'] : continue
if analysis == 'htt' and channel == 'tt' :
if samp == 'zll' : continue
sampHistos[samp] = ROOT.TH1D("All Backgrounds %s %s %s" % (samp, append, ops['targetDir'].strip('/')), samp, xNum, xBins )
sampHistos[samp].Sumw2()
sampHistos[samp].SetFillColor( sampInfo[analysis][samp][0] )
sampHistos[samp].SetLineColor( ROOT.kBlack )
sampHistos[samp].SetLineWidth( 2 )
sampHistos[samp].SetTitle( sampInfo[analysis][samp][1] )
sampHistos[samp].SetDirectory( 0 )
sampHistos[ signal ].SetLineColor( ROOT.kPink )
sampHistos[ signal ].SetLineWidth( 4 )
sampHistos[ signal ].SetLineStyle( 7 )
sampHistos[ signal ].SetMarkerStyle( 0 )
else :
twoDVars = analysisPlots.get2DVars( var )
for samp in sampInfo[analysis].keys() :
# Skip some DY gen based combos
if analysis == 'htt' and channel == 'em' :
if samp in ['zl', 'zj'] : continue
if analysis == 'htt' and channel == 'tt' :
if samp == 'zll' : continue
sampHistos[samp] = ROOT.TH2D("All Backgrounds %s %s %s" % (samp, append, ops['targetDir'].strip('/')),
samp, len(twoDVars[0])-1, twoDVars[0], len(twoDVars[1])-1, twoDVars[1] )
sampHistos[samp].Sumw2()
sampHistos[samp].SetDirectory( 0 )
for sample in samples.keys() :
#print sample
#print samples[sample]
''' Shape systematics are plotted with their
unshifted counterparts '''
getVar = var
if skipSystShapeVar( var, sample, channel ) :
breakUp = var.split('_')
if breakUp[-1] == 'ffSub' :
breakUp.pop()
breakUp.pop()
else : breakUp.pop()
getVar = '_'.join(breakUp)
# Remember data samples are called 'dataEE' and 'dataMM'
if channel in ['eeet', 'eemt', 'eett', 'eeem', 'eeee', 'eemm', 'ZEE'] and 'dataMM' in sample : continue
if channel in ['emmt', 'mmmt', 'mmtt', 'emmm', 'mmmm', 'ZMM'] and 'dataEE' in sample : continue
if channel == 'tt' and sample == 'dataEM' : continue
if channel == 'tt' and '-ZLL' in sample : continue
if channel == 'em' and sample == 'dataTT' : continue
if channel == 'em' and '-ZJ' in sample : continue
if channel == 'em' and '-ZL' in sample and not '-ZLL' in sample : continue
if ops['qcdMC'] and sample == 'QCD' : continue
#print sample
#if not ops['qcdMC'] and 'QCD' in sample and '-' in sample : continue # Why was this line here? QCD pt binned MC???
#if var == 'm_visCor' : print sample
#print '%s2IsoOrderAndDups/%s_%s.root' % (analysis, sample, channel)
if 'QCD' in sample :
#print "qcd in sample",sample
if doFF :
tFile = ROOT.TFile('%s%s/%s_%s.root' % (analysis, folderDetails, sample, channel), 'READ')
elif ops['useQCDMakeName'] != 'x' :
fName = 'meta/%sBackgrounds/%s_qcdShape_%s_%s.root' % (analysis, channel, folderDetails.split('_')[0], ops['useQCDMakeName'])
#print fName
tFile = ROOT.TFile(fName, 'READ')
elif ops['qcdMC'] :
print "Got QCD MC file", sample
tFile = ROOT.TFile('%s%s/%s_%s.root' % (analysis, folderDetails, sample, channel), 'READ')
print "QCD MC: %s Integral %f" % (sample, hxx.Integral() )
elif not ops['qcdMakeDM'] :
tFile = ROOT.TFile('%s%s/%s_%s.root' % (analysis, folderDetails, sample, channel), 'READ')
else :
continue
elif ops['redBkg'] and 'RedBkgShape' in sample :
tFileYield = ROOT.TFile('%s%s/%s_%s.root' % (analysis, folderDetails, sample.replace('Shape','Yield'), channel), 'READ')
tFile = ROOT.TFile('%s%s/%s_%s.root' % (analysis, folderDetails, sample, channel), 'READ')
else :
#print "File: '%s%s/%s_%s.root'" % (analysis, folderDetails, sample, channel)
tFile = ROOT.TFile('%s%s/%s_%s.root' % (analysis, folderDetails, sample, channel), 'READ')
#print tFile
dic = tFile.Get("%s_Histos" % channel )
# Require var to be in file or print note and skip
keys = dic.GetListOfKeys()
inVars = [key.GetName() for key in keys]
# inVars.append( key.GetName() )
if getVar not in inVars :
print "\n"+getVar+" not in your root files! Skipping...\n"
continue
''' Special ZTT scaling for DYJets -> ZTT samples '''
zttScaleTable = {
'inclusive' : 1.0,
'inclusiveSS' : 1.0,
'0jet' : 1.0,
'1jet' : 1.0,
'2jet' : 1.0,
'1jet_low' : 1.0,
'1jet_medium' : 1.0,
'1jet_high' : 1.0,
'2jet_vbf' : 1.2,
'1bjet' : 1.2,
'2bjet' : 1.4,
}
""" Do the Fake Factor MC - jet->tau fake MC here """
if doFF :
# Do this based on the gen appended break down (and WJets)
if '-ZJ' in sample or '-TTJ' in sample or 'WJets' in sample or \
'-VVJ' in sample :
preHist = dic.Get( getVar )
if not '_ffSub' in getVar :
ffSubHist = dic.Get( getVar+'_ffSub' )
#print sample," FF Sub int",ffSubHist.Integral()
ffSubHist2 = ffSubHist.Rebin( xNum, "rebinned", xBins )
if varBinned :
ffSubHist2.GetXaxis().SetRangeUser( xBins[0], xBins[-1] )
else :
ffSubHist2.GetXaxis().SetRangeUser( info[1], info[2] )
#if "DYJets" in sample and "ZTT" in sample :
# ffSubHist2.Scale( zttScaleTable[dirCode] )
sampHistos[ 'jetFakes' ].Add( ffSubHist2, -1.0 )
#print "FFSub Int ",sample,sampHistos[ 'jetFakes' ].Integral()
# QCD takes shape and yield from anti-isolated
# the +'_ffSub' is already appended to getVar
elif 'QCD' in sample :
if not '_ffSub' in getVar :
preHist = dic.Get( getVar+'_ffSub' )
else :
preHist = dic.Get( getVar )
# Data and all other MC treated normally
else :
preHist = dic.Get( getVar )
else :
preHist = dic.Get( getVar )
preHist.SetDirectory( 0 )
if ops['redBkg'] and 'RedBkgShape' in sample :
redBkgYield = tFileYield.Get('%s_Histos/%s' % (channel, getVar)).Integral()
#print "REd BKG Yield:",redBkgYield
if preHist.Integral() != 0 :
preHist.Scale( redBkgYield / preHist.Integral() )
hist = ROOT.TH1D( preHist )
if sample == 'QCD' and ops['useQCDMakeName'] != 'x' :
#print "Using QCD SCALE FACTOR <<<< NEW >>>>"
preHist.Scale( ops['qcdSF'] )
#print "QCD yield: %f" % preHist.Integral()
if not ":" in var :
histX = ROOT.TH1D( preHist )
else :
histX = ROOT.TH2D( preHist )
else :
if ":" in var :
histX = preHist.Rebin2D( 1, 1, "rebinned" )
else :
histX = preHist.Rebin( xNum, "rebinned", xBins )
# If plotting qcd CR, we want a single bin for all
if 'plotMe' in ops['qcdMakeDM'] :
nBins = histX.GetNbinsX()
hist = histX.Rebin( nBins )
else : hist = histX.Clone()
if var == 'm_visCor' :
if 'data' in sample and qcdMake : finalDataYield = hist.Integral()
''' Good Debugging stuff '''
#nBins = hist.GetNbinsX()
#print "sample %s # bins, %i range %i %i" % (sample, nBins, hist.GetBinLowEdge( 1 ), hist.GetBinLowEdge( nBins+1 ))
sampHistos[ samples[ sample ]['group'] ].Add( hist )
#if samples[ sample ]['group'] == 'jetFakes' :
# print "jetFakes Stack yield: %f" % hist.Integral()
# print "%s int: %.2f" % (sample, sampHistos[ samples[ sample ]['group'] ].Integral() )
tFile.Close()
''' Change bin yield to make this make sense with variable binning
this is only for viewing, the DC process is seperate '''
for samp in sampHistos.keys() :
if var == 'm_visCor' :
print "%s --- yield %f" % ( samp, sampHistos[samp].Integral() )
# # With Variable binning, need to set bin content appropriately
# if not varBinned : continue
# if samp == "qcd" : continue
# minWidth = 999.
# for bin_ in range( 1, sampHistos[samp].GetNbinsX()+1 ) :
# minTmp = sampHistos[samp].GetBinWidth(bin_)
# if minTmp < minWidth : minWidth = minTmp
# for bin_ in range( 1, sampHistos[samp].GetNbinsX()+1 ) :
# sampHistos[samp].SetBinContent( bin_, sampHistos[samp].GetBinContent( bin_ ) * ( minWidth / sampHistos[samp].GetBinWidth( bin_ ) ) )
# Some specific HTT stuff
if analysis == 'htt' :
if doFF :
stack.Add( sampHistos['jetFakes'] )
elif not qcdMake :
#print "Adding QCD: ",sampHistos['qcd'].Integral()
stack.Add( sampHistos['qcd'] )
stack.Add( sampHistos['ttbar'] )
stack.Add( sampHistos['dib'] )
stack.Add( sampHistos['wjets'] )
if channel != 'em' :
stack.Add( sampHistos['zl'] )
stack.Add( sampHistos['zj'] )
if channel == 'em' :
stack.Add( sampHistos['zll'] )
stack.Add( sampHistos['ztt'] )
# A to Zh stuff
if analysis == 'azh' :
if ops['redBkg'] :
stack.Add( sampHistos['redBkg'] )
else :
stack.Add( sampHistos['top'] )
stack.Add( sampHistos['dyj'] )
stack.Add( sampHistos['wz'] )
stack.Add( sampHistos['zz'] )
# Scale signal samples for viewing
sampHistos[ signal ].Scale( signalSF )
''' Print out yields for a given distribution '''
#sensitivityVars = ['Higgs_Pt', 'pt_1', 'pt_2', 'mjj', 'jdeta', 'pt_sv']
##if var == 'Higgs_Pt' or var == 'pt_1' or var == 'pt_2' :
#if var in sensitivityVars :
# print "\n\nX "+var+" Stack yields"
# totBkgVar = 0.
# totSig = 0.
# top = stack.GetStack().Last().GetNbinsX()+1
# for bin in range( 1, top ) :
# binBkg = stack.GetStack().Last().GetBinContent( top-bin )
# binSig = sampHistos[signal].GetBinContent( top-bin )/higgsSF
# if binBkg > 0. :
# binSensitivity = binSig / math.sqrt( binBkg+binSig )
# else : binSensitivity = 0.
# totBkgVar += binBkg
# totSig += binSig
# if totBkgVar > 0. :
# sensitivity = totSig / math.sqrt( totBkgVar+totSig )
# else : sensitivity = 0.
# edge = stack.GetStack().Last().GetBinLowEdge( top-bin )
# #print "Bin: %i sensitivity: %.2f signal %.2f bkg: %.2f" % (edge, sensitivity, totSig, totBkgVar)
# print "Bin: %.1f sensitivity: %.2f binSensitivity: %.2f" % (edge, sensitivity, binSensitivity)
"""
Calculate rough bin-by-bin uncertainties
"""
uncertNormMap = { 'htt' : {
'qcd' : .20,
'jetFakes' : .0,
'ttbar' : .15,
'dib' : .10,
'wjets' : .10,
'ztt' : .05,
'zl' : .30,
'zj' : .30,
'higgs' : .0,
'VH' : .2,
'obs' : .0,},
'azh' : {
'top' : .10,
'dyj' : .10,
'wz' : .10,
'zz' : .10,
'redBkg' : .0,
'azh' : .0,
'sm' : .0,
'VH' : .0,
'obs' : .0,}
}
binErrors = []
for k in range( stack.GetStack().Last().GetNbinsX()+1 ) :
toRoot = 0.
for samp in sampHistos.keys() :
if samp in ['azh','sm','VH'] : continue
#toRoot += (sampHistos[samp].GetBinContent(k)*\
# uncertNormMap[analysis][samp])**2
toRoot += sampHistos[samp].GetBinError(k)**2
binErrors.append( math.sqrt(toRoot) )
if qcdMake :
if not ":" in var :
qcdVar = ROOT.TH1D( var, 'qcd%s%s' % (append,var), xNum, xBins )
else :
qcdVar = ROOT.TH2D( var, 'qcd%s%s' % (append,var), len(twoDVars[0])-1, twoDVars[0], len(twoDVars[1])-1, twoDVars[1] )
qcdVar.Sumw2()
qcdVar.Add( sampHistos['obs'], stack.GetStack().Last(), 1., -1. )
qcdVar.SetFillColor( ROOT.kMagenta-10 )
qcdVar.SetLineColor( ROOT.kBlack )
qcdVar.SetLineWidth( 2 )
# Add the shape estimated here to the stack pre-scaling!!!
stack.Add( qcdVar )
if var == 'm_visCor_mssm' :
print "M_VIS_MSSM plot details: %f %f" % (info[1], info[2])
if varBinned :
qcdVar.GetXaxis().SetRangeUser( xBins[0], xBins[-1] )
else :
qcdVar.GetXaxis().SetRangeUser( info[1], info[2] )
print "qcdVar: %f mean %f" % (qcdVar.Integral(), qcdVar.GetMean() )
if var == 'm_visCor' :
finalQCDYield = qcdVar.Integral()
qcdDir.cd()
qcdVar.Write()
# Maybe make ratio hist
c1 = ROOT.TCanvas("c1","Z -> #tau#tau, %s, %s" % (channel, var), 550, 550)
if not ops['ratio'] or ":" in var :
pad1 = ROOT.TPad("pad1", "", 0, 0, 1, 1)
pad1.Draw()
pad1.cd()
stack.Draw('hist')
if not 'plotMe' in ops['qcdMakeDM'] :
sampHistos[signal].Draw('same')
sampHistos['obs'].Draw('esamex0')
# X Axis!
stack.GetXaxis().SetTitle("%s" % info[ 4 ])
else : # Do Ratio plots
smlPadSize = .25
pads = ratioPlot( c1, 1-smlPadSize )
pad1 = pads[0]
ratioPad = pads[1]
ratioPad.SetTopMargin(0.00)
ratioPad.SetBottomMargin(0.3)
pad1.SetBottomMargin(0.00)
ratioPad.SetGridy()
ratioHist = ROOT.TH1D('ratio %s' % append, 'ratio', xNum, xBins )
ratioHist.Sumw2()
ratioHist.Add( sampHistos['obs'] )
ratioHist.Divide( stack.GetStack().Last() )
ratioHist.SetMaximum( 2. )
ratioHist.SetMinimum( 0. )
if channel == 'tt' :
ratioHist.SetMaximum( 1.5 )
ratioHist.SetMinimum( 0.5 )
ratioHist.SetMarkerStyle( 21 )
ratioPad.cd()
ratioHist.Draw('ex0')
""" Add uncertainty bands on ratio """
er = ROOT.TH1D("er %s" % append, "er", xNum, xBins )
er.Sumw2()
if varBinned :
er.GetXaxis().SetRangeUser( xBins[0], xBins[-1] )
else :
er.GetXaxis().SetRangeUser( info[1], info[2] )
for k in range( er.GetNbinsX()+1 ) :
er.SetBinContent( k, 1. )
if stack.GetStack().Last().GetBinContent(k) > 0. :
er.SetBinError(k, binErrors[k]/stack.GetStack().Last().GetBinContent(k) )
#print "Qcd Error:",qcd.GetBinError(k)
er.SetLineColor( 0 )
er.SetLineWidth( 0 )
er.SetMarkerSize( 0 )
er.SetFillStyle( 3001 )
er.SetFillColor( 15 )
er.Draw('same e2')
ratioHist.Draw('esamex0')
line = ROOT.TLine( info[1], 1, info[2], 1 )
line.SetLineColor(ROOT.kBlack)
line.SetLineWidth( 1 )
line.Draw()
ratioHist.Draw('esamex0')
# X Axis!
ratioHist.GetXaxis().SetTitle("%s" % info[ 4 ])
ratioHist.GetYaxis().SetTitle("Data / MC")
ratioHist.GetYaxis().SetTitleSize( ratioHist.GetXaxis().GetTitleSize()*( (1-smlPadSize)/smlPadSize) )
ratioHist.GetYaxis().SetTitleOffset( smlPadSize*1.5 )
ratioHist.GetYaxis().SetLabelSize( ratioHist.GetYaxis().GetLabelSize()*( 1/smlPadSize) )
ratioHist.GetYaxis().SetNdivisions( 5, True )
ratioHist.GetXaxis().SetLabelSize( ratioHist.GetXaxis().GetLabelSize()*( 1/smlPadSize) )
ratioHist.GetXaxis().SetTitleSize( ratioHist.GetXaxis().GetTitleSize()*( 1/smlPadSize) )
pad1.cd()
stack.Draw('hist')
if not 'plotMe' in ops['qcdMakeDM'] :
sampHistos[signal].Draw('same')
sampHistos['obs'].Draw('esamex0')
# Set labels appropriately
if info[ 5 ] == '' :
stack.GetYaxis().SetTitle("Events")
elif not ":" in var :
if varBinned :
stack.GetYaxis().SetTitle("Events / Bin Width")
else :
width = stack.GetStack().Last().GetBinWidth(1)
stack.GetYaxis().SetTitle("Events / %.1f%s" % (width, info[ 5 ]) )
# Set axis and viewing area
stackMax = stack.GetStack().Last().GetMaximum()
dataMax = sampHistos['obs'].GetMaximum()
stack.SetMaximum( max(dataMax, stackMax) * 1.5 )
if ops['targetDir'] == '/vbf_low' :
stack.SetMaximum( max(dataMax, stackMax) * 1.8 )
if ops['log'] :
pad1.SetLogy()
stack.SetMaximum( max(dataMax, stackMax) * 10 )
stack.SetMinimum( min(dataMax, stackMax) * .005 )
''' Build the legend explicitly so we can specify marker styles '''
legend = ROOT.TLegend(0.60, 0.65, 0.95, 0.93)
legend.SetMargin(0.3)
legend.SetBorderSize(0)
legend.AddEntry( sampHistos['obs'], "Data", 'lep')
if not 'plotMe' in ops['qcdMakeDM'] :
legend.AddEntry( sampHistos[signal], sampHistos[signal].GetTitle(), 'l')
for j in range(0, stack.GetStack().GetLast() + 1) :
last = stack.GetStack().GetLast()
name_str = stack.GetStack()[last - j ].GetTitle()
if 'qcd' in name_str or 'QCD' in name_str : name_str = 'QCD'
legend.AddEntry( stack.GetStack()[ last - j ], name_str, 'f')
legend.Draw()
# Set CMS Styles Stuff
logo = ROOT.TText(.2, .88,"CMS Preliminary")
logo.SetTextSize(0.03)
logo.DrawTextNDC(.2, .89,"CMS Preliminary")
chan = ROOT.TLatex(.2, .80,"x")
chan.SetTextSize(0.05)
chan.DrawLatexNDC(.2, .84,"Channel: %s" % chans[channel] )
lumi = ROOT.TText(.7,1.05,"X fb^{-1} (13 TeV)")
lumi.SetTextSize(0.03)
lumi.DrawTextNDC(.7,.96,"%.1f / fb (13 TeV)" % cmsLumi )
''' Random print outs on plots '''
if ops['text'] and not varBinned :
text1 = ROOT.TText(.4,.6,"Data Integral: %f" % sampHistos['obs'].GetMean() )
text1.SetTextSize(0.04)
text1.DrawTextNDC(.6,.6,"Data Integral: %s" % str( round( sampHistos['obs'].Integral(), 1) ) )
text2 = ROOT.TText(.4,.55,"Data Int: %s" % str( sampHistos['obs'].Integral() ) )
text2.SetTextSize(0.04)
text2.DrawTextNDC(.6,.55,"MC Integral: %s" % str( round( stack.GetStack().Last().Integral(), 1) ) )
text3 = ROOT.TText(.4,.55,"Data Mean: %s" % str( sampHistos['obs'].GetMean() ) )
text3.SetTextSize(0.04)
text3.DrawTextNDC(.6,.50,"Diff: %s" % str( round( sampHistos['obs'].Integral() - stack.GetStack().Last().Integral(), 1) ) )
pad1.Update()
if varBinned :
stack.GetXaxis().SetRangeUser( xBins[0], xBins[-1] )
else :
stack.GetXaxis().SetRangeUser( info[1], info[2] )
if ops['ratio'] and not ":" in var :
if varBinned :
ratioHist.GetXaxis().SetRangeUser( xBins[0], xBins[-1] )
else :
ratioHist.GetXaxis().SetRangeUser( info[1], info[2] )
"""
Add uncertainty bands on background stack
"""
if ops['addUncert'] and not ":" in var :
e1 = ROOT.TH1D("e1 %s" % append, "e1", xNum, xBins )
e1.Sumw2()
if varBinned :
e1.GetXaxis().SetRangeUser( xBins[0], xBins[-1] )
else :
e1.GetXaxis().SetRangeUser( info[1], info[2] )
for k in range( e1.GetNbinsX()+1 ) :
e1.SetBinContent( k, stack.GetStack().Last().GetBinContent( k ) )
e1.SetBinError(k, binErrors[k] )
#print "Qcd Error:",qcd.GetBinError(k)
e1.SetLineColor( 0 )
e1.SetLineWidth( 0 )
e1.SetMarkerSize( 0 )
e1.SetFillStyle( 3001 )
e1.SetFillColor( 15 )
e1.Draw('same e2')
""" Blinding Data """
if ops['blind'] and not 'plotMe' in ops['qcdMakeDM'] :
if (analysis == 'htt' and ('m_visCor' in var or 'm_sv' in var or 'mt_sv' in var\
or 'mt_tot' in var) ) or\
(analysis=='azh' and ('H_vis' in var or 'Mass' in var) ) :
if ops['mssm'] :
targetMass = 170
targetMassUp = 9999
elif analysis == 'htt' and 'm_sv' in var :
targetMassLow = 101
#if '1jet' in ops['targetDir'] : targetMassLow = 90
#elif 'vbf' in ops['targetDir'] : targetMassLow = 100
#elif 'vbf_low' in ops['targetDir'] : targetMassLow = 100
#elif 'vbf_high' in ops['targetDir'] : targetMassLow = 80
#else : targetMassLow = 80
targetMassUp = 149
elif analysis == 'htt' and 'm_visCor' in var :
targetMassLow = 81
targetMassUp = 149
else :
targetMassLow = 81
targetMassUp = 149
nBins = stack.GetStack().Last().GetXaxis().GetNbins()
for k in range( 1, nBins+1 ) :
binHigh = sampHistos['obs'].GetXaxis().GetBinLowEdge(k) + \
sampHistos['obs'].GetXaxis().GetBinWidth(k)
binLow = sampHistos['obs'].GetXaxis().GetBinLowEdge(k)
if binHigh>targetMassLow and binLow<=targetMassUp :
sampHistos['obs'].SetBinContent(k, 0.)
sampHistos['obs'].SetBinError(k, 0.)
if ops['ratio'] and not ":" in var :
ratioHist.SetBinContent(k, 0.)
ratioHist.SetBinError(k, 0.)
if ops['ratio'] and not ":" in var :
ratioPad.cd()
ratioHist.Draw('esamex0')
pad1.cd()
sampHistos['obs'].Draw('esamex0')
if ops['qcdMakeDM'] == 'x' or 'plotMe' in ops['qcdMakeDM'] :
plotDir = '/afs/cern.ch/user/t/truggles/www/%sPlots/%s%s/' % (analysis, channel, ops['targetDir'] )
c1.SaveAs(plotDir+'%s.png' % var )
c1.SaveAs(plotDir+'%s.pdf' % var )
c1.SaveAs(plotDir+'%s.root' % var )
c1.SaveAs(plotDir+'%s.C' % var )
# To speed up, just copy the new png/pdfs to other dir
# this will help with 2D plots
newPlotDir = plotDir.replace('Plots/','PlotsList/')
subprocess.call(['cp',plotDir+'%s.png' % var, newPlotDir+'%s.png' % var])
subprocess.call(['cp',plotDir+'%s.pdf' % var, newPlotDir+'%s.pdf' % var])
subprocess.call(['cp',plotDir+'%s.root' % var, newPlotDir+'%s.root' % var])
subprocess.call(['cp',plotDir+'%s.C' % var, newPlotDir+'%s.C' % var])
""" Additional views for Visible Mass """
#if 'm_visCor' in var :
# pad1.SetLogy()
# stack.SetMaximum( stack.GetMaximum() * 10 )
# stack.SetMinimum( higgs.GetMaximum() * .1 )
# if var == 'm_visCor_mssm' :
# pad1.SetLogx()
# if ratio : ratioPad.SetLogx()
# pad1.Update()
# c1.SaveAs('/afs/cern.ch/user/t/truggles/www/%sPlots/%s/%s_LogY.png' % (analysis, channel, var ) )
# c1.SaveAs('/afs/cern.ch/user/t/truggles/www/%sPlotsList/%s/%s_LogY.png' % (analysis, channel, var ) )
# htmlFile.write( '<img src="%s_LogY.png">\n' % var )
c1.Close()
htmlFile.write( '<img src="%s.png">\n' % var )
#htmlFile.write( '<br>\n' )
htmlFile.write( '</body></html>' )
htmlFile.close()
if qcdMake :
print "\n\n Final QCD and Data Info:\n -- QCD Name: %s\n -- Data Yield = %f\n -- QCD Yield = %f" % (ops['qcdMakeDM'], finalDataYield, finalQCDYield)
dumpFile = open('plotsOut.txt', 'a')
dumpFile.write("\nFinal QCD and Data Info:\n -- QCD Name: %s\n -- Data Yield = %f\n -- QCD Yield = %f" % (ops['qcdMakeDM'], finalDataYield, finalQCDYield))
dumpFile.close()
return finalQCDYield
CRISPY_TEMPLATE_PACK = 'bootstrap3'
RECAPTCHA_PUBLIC_KEY = os.getenv("RECAPTCHA_PUBLIC_KEY")
RECAPTCHA_PRIVATE_KEY = os.getenv("RECAPTCHA_PRIVATE_KEY")
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
EMAIL_USE_TLS = True
EMAIL_HOST = 'smtp.udag.de'
EMAIL_HOST_USER = os.getenv("EMAIL_HOST_USER")
EMAIL_HOST_PASSWORD = os.getenv("EMAIL_HOST_PASSWORD")
EMAIL_PORT = 587
TAGGIT_CASE_INSENSITIVE = True
TAGGIT_TAGS_FROM_STRING = 'home.utils.tags_splitter'
from smart_getenv import getenv
DEFAULT_DAILY_RATE = getenv('DEFAULT_DAILY_RATE', default=100, type=Decimal)
PHONENUMBER_DEFAULT_REGION = "DE"
PHONENUMBER_DB_FORMAT = "NATIONAL"
VAT_RATE = Decimal('0.19')
INCOME_TAX_RATE = Decimal('0.41')
AUTHENTICATION_BACKENDS = (
'social_core.backends.google.GoogleOAuth2',
'django.contrib.auth.backends.ModelBackend',
)
SOCIAL_AUTH_URL_NAMESPACE = 'social'
SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = getenv("SOCIAL_AUTH_GOOGLE_OAUTH2_KEY")
SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = getenv("SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET")
SOCIAL_AUTH_LOGIN_URL = "/admin/account/"
# Collect static files here
STATIC_ROOT = os.path.join(RUN_DIR, 'static')
# Collect media files here
MEDIA_ROOT = os.path.join(RUN_DIR, 'media')
# Look for static assets here
STATICFILES_DIRS = [
os.path.join(BASE_DIR, 'static'),
]
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = getenv('DJANGO_SECRET_KEY', type=str, default='secret_key')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = getenv('DJANGO_DEBUG', type=bool, default=True)
INTERNAL_IPS = getenv('DJANGO_INTERNAL_IPS',
type=list,
separator=',',
default=[])
ALLOWED_HOSTS = getenv('DJANGO_ALLOWED_HOSTS',
type=list,
separator=',',
default=[])
# Application definition
def drawHistos(analysis, samples, **fargs ) :
print "\n%s\n" % fargs['mid3']
mergeMap = getMergeMap(analysis)
skipSSQCDDetails = False
if 'skipSSQCDDetails' in fargs.keys() :
if fargs['skipSSQCDDetails'] :
skipSSQCDDetails = True
genMap = {
# sample : em , tt
'ZTT' : {
'tt' : '*(gen_match_1 == 5 && gen_match_2 == 5)'},
'ZL' : {
'tt' : '*(gen_match_1 < 6 && gen_match_2 < 6 && !(gen_match_1 == 5 && gen_match_2 == 5))'},
'ZJ' : {
'tt' : '*(gen_match_2 == 6 || gen_match_1 == 6)'},
'QCD' : {
'tt' : '*(FFWeightQCD)'},
'TTT' : {
'tt' : '*(gen_match_1 == 5 && gen_match_2 == 5)'},
'TTJ' : {
'tt' : '*(gen_match_1 != 5 || gen_match_2 != 5)'},
'VVT' : {
'tt' : '*(gen_match_1 == 5 && gen_match_2 == 5)'},
'VVJ' : {
'tt' : '*(gen_match_1 != 5 || gen_match_2 != 5)'},
'RedBkgYield' : {'xxxx' : '*(1.)'},
'RedBkgShape' : {'xxxx' : '*(1.)'},
}
channels = fargs['channels']
''' Start PROOF multiprocessing Draw '''
#ROOT.TProof.Open('workers=%s' % str( int(fargs['numCores']) ) )
gROOT.SetBatch(True)
''' Start multiprocessing tests '''
#output = multiprocessing.Queue()
pool = multiprocessing.Pool(processes= fargs[ 'numCores' ] )
multiprocessingOutputs = []
for sample in samples :
numFilesPerCycle = fargs['numFilesPerCycle']
if sample in mergeMap.keys() and fargs[ 'skimmed' ] != 'true' :
numFilesPerCycle = mergeMap[sample]
# the gen matching samples are: based off of the DYJets samples
loopList = []
doFF = getenv('doFF', type=bool)
if 'DYJets' in sample and analysis == 'htt' :
genList = ['ZTT', 'ZL', 'ZJ']
loopList = genList
#loopList.append( sample ) # don't keep full original
elif sample == 'TT' and analysis == 'htt' :
genList = ['TTT', 'TTJ']
loopList = genList
#loopList.append( sample ) # don't keep full original
elif sample in ['T-tW', 'T-tchan', 'Tbar-tW', 'Tbar-tchan',
'WW1l1nu2q', 'WW2l2nu', 'WZ1l1nu2q', 'WZ1l3nu',
'WZ2l2q', 'WZ3l1nu', 'ZZ2l2nu', 'ZZ2l2q', 'ZZ4l',
'VV', 'WWW', 'ZZZ'] and analysis == 'htt' :
genList = ['VVT', 'VVJ']
loopList = genList
#loopList.append( sample ) # don't keep full original
elif 'data' in sample and doFF :
loopList.append( sample )
loopList.append( 'QCD-'+sample.split('-')[1] )
elif 'data' in sample and analysis == 'azh' :
loopList.append( sample )
loopList.append( 'RedBkgYield-'+sample.split('-')[1] )
loopList.append( 'RedBkgShape-'+sample.split('-')[1] )
else : loopList.append( sample )
sampF = ''
if fargs['svFitPost'] == 'true' : sampF = 'sv/'
if fargs['skimmed'] == 'true' : sampF = 'skimmed/'
for subName in loopList :
#print "SubName:",subName
if subName == 'QCD' and 'data' in sample : saveName = 'QCD'
elif 'RedBkg' in subName and 'data' in sample : saveName = subName
elif 'QCD' in subName and 'data' in sample and doFF : saveName = subName
elif subName != sample : saveName = "%s-%s" % (sample.split('_')[0], subName)
else : saveName = sample.split('_')[0]
for channel in channels :
# Check if we should skip running over data set
if skipChanDataCombo( channel, sample, analysis ) : continue
if fargs['svFitPost'] == 'true' or fargs['skimmed'] == 'true' :
fileLen = file_len( 'meta/NtupleInputs_%s/%s%s_%s.txt' % (analysis, sampF, sample, channel) )
else :
fileLen = file_len( 'meta/NtupleInputs_%s/%s.txt' % (analysis, sample) )
print "File len:",fileLen
#print "Num files / cycle:",numFilesPerCycle
numIters = int( math.ceil( 1. * fileLen / numFilesPerCycle ) )
#print "Num Iters: %i" % numIters
print " ====> Starting Plots For %s_%s_%s <==== " % (analysis, saveName, channel)
chain = ROOT.TChain('Ntuple')
for i in range( numIters ) :
#print "%s_%i" % ( sample, i)
#print " --- Adding to chain: %s%s/%s_%i_%s.root" % (analysis, fargs['mid2'], sample.split('_')[0], i, channel)
chain.Add('%s%s/%s_%i_%s.root' % (analysis, fargs['mid2'], sample.split('_')[0], i, channel) )
print "ENTRIES: %s %i" % (sample, chain.GetEntries() )
if 'data' in sample : isData = True
else : isData = False
additionalCut = fargs['additionalCut']
if subName != sample and 'RedBkg' not in subName and 'QCD-' not in subName :
if genMap[subName][channel] == '' : continue
if additionalCut == '' : additionalCut = genMap[subName][channel]
else : additionalCut += genMap[subName][channel]
#print "AdditionalCuts",additionalCut
blind = False
outFile = ROOT.TFile('%s%s/%s_%s.root' % (analysis, fargs['mid3'], saveName , channel), 'RECREATE')
if not fargs['debug'] == 'true' :
multiprocessingOutputs.append( pool.apply_async(analysisPlots.plotHistosProof,
args=(analysis,
outFile,
chain,
sample,
channel,
isData,
additionalCut,
blind,
skipSSQCDDetails,
sample+'-'+subName)) )
""" for debugging without multiprocessing """
if fargs['debug'] == 'true' :
analysisPlots.plotHistosProof(
analysis,
outFile,
chain,
sample,
channel,
isData,
additionalCut,
blind,
skipSSQCDDetails,
sample+'-'+subName)
#analysisPlots.plotHistosProof( analysis, outFile, chain, sample, channel, isData, additionalCut, blind, skipSSQCDDetails )
#outFile.Close()
if fargs['debug'] != 'true' :
mpResults = [p.get() for p in multiprocessingOutputs]
print "#################################################################"
print "### Finished plotting all samples ###"
print "#################################################################"
def test_getenv_type_str(self):
"""
Ensure getenv returns string if environment variable exists and desired type is string.
"""
os.environ[self.test_var_name] = 'abc'
self.assertEqual(getenv(self.test_var_name, type=str), 'abc')
ManagedIdentityCredential,
EnvironmentCredential,
)
from azure.storage.queue import QueueServiceClient
from azure.ai.formrecognizer import FormRecognizerClient
from azure.ai.textanalytics import TextAnalyticsClient
from azure.cosmos import CosmosClient
from azure.keyvault.secrets import SecretClient
from smart_getenv import getenv
from dotmap import DotMap
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv())
sleep = getenv("AZURE_STORAGE_QUEUE_RECEIVE_SLEEP", type=bool, default=1)
credential = ChainedTokenCredential(
AzureCliCredential(), EnvironmentCredential(), ManagedIdentityCredential()
)
queue_service_client = QueueServiceClient(
account_url=getenv("AZURE_STORAGE_QUEUE_ENDPOINT"), credential=credential
)
queue_client = queue_service_client.get_queue_client(
queue=getenv("AZURE_STORAGE_QUEUE_NAME", default="messages")
)
fr_client = FormRecognizerClient(
endpoint=getenv("AZURE_FORM_RECOGNIZER_ENDPOINT"), credential=credential
def plotHistosProof( analysis, outFile, chain, sample, channel, isData, additionalCut, blind=False, skipSSQCDDetails=False, genCode='x' ) :
if genCode == 'x' : genCode = sample
''' Make a channel specific selection of desired histos and fill them '''
newVarMapUnsorted = getHistoDict( analysis, channel )
newVarMap = returnSortedDict( newVarMapUnsorted )
#print outFile, channel
histosDir = outFile.mkdir( "%s_Histos" % channel )
histosDir.cd()
# Set additionalCut to reflect ZH reducible background estimation
# process
# Add in the ability to do Reducible Background estimations for
# AZH / ZH analysis
# Add channel specific cuts
if 'ADD_CHANNEL_SPECIFIC_ISO_CUTS' in additionalCut :
prodMap = getProdMap()
if analysis == 'azh' and 'RedBkgYield' in outFile.GetName() :
additionalCut = getRedBkgCutsAndWeights(
analysis, channel, additionalCut, prodMap )
elif analysis == 'azh' and 'RedBkgShape' in outFile.GetName() :
additionalCut = getRedBkgShape(
analysis, channel, additionalCut, prodMap )
else : # No reducible bkg
additionalCut = getChannelSpecificFinalCuts(
analysis, channel, additionalCut, prodMap )
''' Combine Gen and Chan specific into one fill section '''
histos = {}
''' Get Energy Scale Map which is now confusing with
decay mode specific shifts '''
esMap = getESMap()
### Check if we intend to do Fake Factor based MC cuts
### These differ because of requiring a random choice
### of l1 and l2, then seeing if l1 is gen matched
### to anything besides a fake/jet
### This is only applied for DYJets, WJets, TT, and QCD MC
### Check if doFF is turned on and, if so, double
### our output histograms to account for the
### required subtractions
doFF = getenv('doFF', type=bool)
if doFF :
tmpDict = {}
for var, info in newVarMap.iteritems() :
tmpDict[var] = info
tmpDict[var+'_ffSub'] = info
newVarMap = returnSortedDict( tmpDict )
# Check for per-var adjustments later
# Full JES shapes if doFullJES
doFullJES = getenv('doFullJES', type=bool)
if doFullJES : jesUncerts = getUncerts()
for var, info in newVarMap.iteritems() :
if skipSSQCDDetails and not (var == 'eta_1' or var == 'm_visCor') : continue
''' Skip plotting 2D vars for 0jet and inclusive selections '''
if 'ZTTinclusive' in outFile.GetName() or 'ZTT0jet' in outFile.GetName() :
if ":" in var : continue
#print var
''' Skip plotting unused shape systematics '''
if skipSystShapeVar( var, sample, channel, genCode ) : continue
''' Define syst shape weights if applicable '''
shapeSyst = ''
# High Pt tau reweighting,
# not applied to data or FF jetToTauFakes
if '_tauPt' in var :
shapeSyst = HighPtTauWeight( var )
# top pt reweighting only applied to ttbar
elif '_topPt' in var :
if '_topPtUp' in var : shapeSyst = '*(topWeight)'
elif '_topPtDown' in var : shapeSyst = '*(1./topWeight)'
# z pt reweight only applied to LO DYJets samples, DYJetsLow in amc@nlo
elif '_zPt' in var : # Shifts has been changed to 10% of correction
if '_zPtUp' in var : shapeSyst = '*(1. + (-1. + zPtWeight) * 0.1 )'
elif '_zPtDown' in var : shapeSyst = '*(1. - (-1. + zPtWeight) * 0.1 )'
# ggH scale to ggHtoTauTau signal
elif '_ggH' in var :
# Different scale depending on final category
if 'ZTT0jet2D' in outFile.GetName() :
if '_ggHUp' in var : shapeSyst = '*(ggHWeight0Jet)'
elif '_ggHDown' in var : shapeSyst = '*(1./ggHWeight0Jet)'
elif 'ZTTboosted' in outFile.GetName() :
if '_ggHUp' in var : shapeSyst = '*(ggHWeightBoost)'
elif '_ggHDown' in var : shapeSyst = '*(1./ggHWeightBoost)'
elif 'ZTTvbf' in outFile.GetName() :
if '_ggHUp' in var : shapeSyst = '*(ggHWeightVBF)'
elif '_ggHDown' in var : shapeSyst = '*(1./ggHWeightVBF)'
else : # Probably inclusive
if '_ggHUp' in var : shapeSyst = '*(ggHWeight0Jet)'
elif '_ggHDown' in var : shapeSyst = '*(1./ggHWeight0Jet)'
# topQuarkggH scale to ggHtoTauTau signal
elif '_topQuarkggH' in var :
if '_topQuarkggHUp' in var : shapeSyst = '*(ggHtopQuarkWeight)'
elif '_topQuarkggHDown' in var : shapeSyst = '*(2.-ggHtopQuarkWeight)'
# Jet to Tau Fake
# These look to be applied in reverse
# the shift convention was choosen to match MSSM 2016 ICHEP
elif '_JetToTau' in var :
if '_JetToTauUp' in var :
shapeSyst = '*(2 - jetToTauFakeWeight)' # = to 1 - SF
elif '_JetToTauDown' in var :
shapeSyst = '*(jetToTauFakeWeight)'
# This is the Zmumu CR shape uncertainty from data/MC correction
# below. It is currently only applied to VBF
# This shape will be skipped if not DYJets
# Update - no shape uncert on boosted
elif '_Zmumu' in var :
if '_ZmumuUp' in var and 'ZTTvbf' in outFile.GetName() :
shapeSyst = '*(1. + zmumuVBFWeight)'
elif '_ZmumuDown' in var and 'ZTTvbf' in outFile.GetName() :
shapeSyst = '*(1./(1. + zmumuVBFWeight))'
# Add the Zmumu CR normalizations from Cecile's studies
# from Nov 18, 2016 SM-HTT
# Update 2 Mar, 2017:
# - VBF has specific shape correction too
# Removed 2% global normalization adjustment, April 4, 2017
if 'DYJets' in sample or 'EWKZ' in sample :
if 'ZTTvbf' in outFile.GetName() :
shapeSyst += '*(1.00 + zmumuVBFWeight)'
else : # Applied to ALL categories
shapeSyst += '*(1.00)'
# Energy Scale reweighting applied to all Real Hadronic Taus
# gen_match == 5
# this is not an "if" style shape b/c we need to apply
# normal pt cuts if the shape syst is not called
# so instead we appeand it to what ever else we have
shapeSyst += ESCuts( esMap, sample, channel, var )
# Additionally, if we have Energy Scale, we also need
# to change any Higgs_PtCor vars to Higgs_PtCor_UP/DOWN
additionalCutToUse = additionalCut
if 'energyScale' in var and 'data' not in sample :
dm = ''
if 'energyScaleAll' in var : dm = '_'
elif 'energyScaleDM0' in var : dm = '_DM0_'
elif 'energyScaleDM1' in var : dm = '_DM1_'
elif 'energyScaleDM10' in var : dm = '_DM10_'
if 'Up' in var[-2:] : shiftDir = 'UP'
if 'Down' in var[-4:] : shiftDir = 'DOWN'
additionalCutToUse = additionalCutToUse.replace('pt_sv','pt_sv%s%s' % (dm, shiftDir) )
additionalCutToUse = additionalCutToUse.replace('Higgs_PtCor','Higgs_PtCor%s%s' % (dm, shiftDir) )
# Jet Energy Scale:
# similar as TES above, edit the additionalCut
# so that njets is replaced by njets_JESUP
# make because we unroll in mjj, edit that
# as well when it's in the plotVar for 2D
# plotVar is changed below with this happens for other shifts
jesUnc = 'En'
if 'JES' in var and 'data' not in sample :
# with ~30 shifts, get the name of the shift
if doFullJES :
for unc in jesUncerts :
if unc in var : jesUnc = unc
if 'Up' in var[-2:] :
if doFullJES :
# jDeta is minimally impacted by JES shifts, so the minor adjustments are not saved atm
# and is negligible for mjj > 100
additionalCutToUse = additionalCutToUse.replace('jetVeto30','jetVeto30_Jet%sUp' % jesUnc)
# We changed naming to simple mjj, njetingap20 and jdeta for sync for nominal
additionalCutToUse = additionalCutToUse.replace('mjj','vbfMass_Jet%sUp' % jesUnc)
else :
#additionalCutToUse = additionalCutToUse.replace('njetingap','vbfJetVeto30_JetEnUp') # Cut no long used
additionalCutToUse = additionalCutToUse.replace('jdeta','vbfDeta_JetEnUp')
#print additionalCutToUse+"\n"
if 'Down' in var[-4:] :
if doFullJES :
additionalCutToUse = additionalCutToUse.replace('jetVeto30','jetVeto30_Jet%sDown' % jesUnc)
additionalCutToUse = additionalCutToUse.replace('mjj','vbfMass_Jet%sDown' % jesUnc)
else :
#additionalCutToUse = additionalCutToUse.replace('njetingap','vbfJetVeto30_JetEnDown')
additionalCutToUse = additionalCutToUse.replace('jdeta','vbfDeta_JetEnDown')
#print additionalCutToUse+"\n"
# Met Systematics propagated to svFit
# make sure to get proper Higgs_Pt (pt_sv)
if '_metClustered' in var and 'data' not in sample :
if 'Up' in var[-2:] :
additionalCutToUse = additionalCutToUse.replace('pt_sv','pt_sv_ClusteredMet_UP')
additionalCutToUse = additionalCutToUse.replace('Higgs_PtCor','Higgs_PtCor_ClusteredMet_UP')
if 'Down' in var[-4:] :
additionalCutToUse = additionalCutToUse.replace('pt_sv','pt_sv_ClusteredMet_DOWN')
additionalCutToUse = additionalCutToUse.replace('Higgs_PtCor','Higgs_PtCor_ClusteredMet_DOWN')
if '_metUnclustered' in var and 'data' not in sample :
if 'Up' in var[-2:] :
additionalCutToUse = additionalCutToUse.replace('pt_sv','pt_sv_UncMet_UP')
additionalCutToUse = additionalCutToUse.replace('Higgs_PtCor','Higgs_PtCor_UncMet_UP')
if 'Down' in var[-4:] :
additionalCutToUse = additionalCutToUse.replace('pt_sv','pt_sv_UncMet_DOWN')
additionalCutToUse = additionalCutToUse.replace('Higgs_PtCor','Higgs_PtCor_UncMet_DOWN')
# This addes the Fake Factor shape systematics weights
# And add the variable specific Fake Factor cut
# (isolation and gen match change per variable def)
ffShapeSyst = ''
if doFF :
ffRegion = 'anti-iso' if '_ffSub' in var else 'signal'
ffShapeSyst += getFFShapeSystApp( ffRegion, isData, outFile, var )
ffShapeSyst += getFFCutsAndWeights( ffRegion, isData, outFile )
if ":" in var :
histos[ var ] = make2DHisto( var )
else :
histos[ var ] = makeHisto( var, info[0], info[1], info[2])
# Adding Trigger, ID and Iso, & Efficiency Scale Factors
# and, top pt reweighting
# weight is a composition of all applied MC/Data corrections
sfs = '*(1.)'
if analysis == 'htt' :
sfs = '*(weight)'
if channel == 'tt' :
# Not currently included in weight for sync ntuple
sfs += '*(tauIDweight_1 * tauIDweight_2)'
if analysis == 'azh' :
sfs = '*(puweight*azhWeight)'
xsec = '*(XSecLumiWeight)'
#print "%s High Pt Tau Weight: %s" % (var, tauW)
#print var,shapeSyst
totalCutAndWeightMC = '(GenWeight/abs( GenWeight ))%s%s%s%s%s' % (additionalCutToUse, sfs, xsec, shapeSyst, ffShapeSyst)
#print totalCutAndWeightMC
# Check if the variable to plot is in the chain, if not, skip it
# don't crash on systematics based variables
varBase = var.replace('_ffSub','')
plotVar = var.replace('_ffSub','') # remove the histo naming off the back of the plotting var
if 'Up' in var or 'Down' in var :
tmp = varBase.split('_')
shapeName = tmp.pop()
varBase = '_'.join(tmp)
if 'energyScale' in shapeName :
shiftDir = 'UP' if 'Up' in var else 'DOWN'
if 'pt_sv:m_sv' in var :
if 'All' in var : plotVar = 'pt_sv_%s:m_sv_%s' % (shiftDir, shiftDir)
if 'DM0' in var : plotVar = 'pt_sv_DM0_%s:m_sv_DM0_%s' % (shiftDir, shiftDir)
if 'DM1' in var : plotVar = 'pt_sv_DM1_%s:m_sv_DM1_%s' % (shiftDir, shiftDir)
if 'DM10' in var : plotVar = 'pt_sv_DM10_%s:m_sv_DM10_%s' % (shiftDir, shiftDir)
elif 'Higgs_PtCor:m_sv' in var :
if 'All' in var : plotVar = 'Higgs_PtCor_%s:m_sv_%s' % (shiftDir, shiftDir)
if 'DM0' in var : plotVar = 'Higgs_PtCor_DM0_%s:m_sv_DM0_%s' % (shiftDir, shiftDir)
if 'DM1' in var : plotVar = 'Higgs_PtCor_DM1_%s:m_sv_DM1_%s' % (shiftDir, shiftDir)
if 'DM10' in var : plotVar = 'Higgs_PtCor_DM10_%s:m_sv_DM10_%s' % (shiftDir, shiftDir)
elif 'mjj:m_sv' in var :
if 'All' in var : plotVar = 'mjj:m_sv_%s' % shiftDir
if 'DM0' in var : plotVar = 'mjj:m_sv_DM0_%s' % shiftDir
if 'DM1' in var : plotVar = 'mjj:m_sv_DM1_%s' % shiftDir
if 'DM10' in var : plotVar = 'mjj:m_sv_DM10_%s' % shiftDir
elif 'm_sv' in var :
if 'All' in var : plotVar = 'm_sv_%s' % shiftDir
if 'DM0' in var : plotVar = 'm_sv_DM0_%s' % shiftDir
if 'DM1' in var : plotVar = 'm_sv_DM1_%s' % shiftDir
if 'DM10' in var : plotVar = 'm_sv_DM10_%s' % shiftDir
elif 'Higgs_PtCor:m_visCor' in var :
if 'All' in var : plotVar = 'Higgs_PtCor_%s:m_visCor_%s' % (shiftDir, shiftDir)
if 'DM0' in var : plotVar = 'Higgs_PtCor_DM0_%s:m_visCor_DM0_%s' % (shiftDir, shiftDir)
if 'DM1' in var : plotVar = 'Higgs_PtCor_DM1_%s:m_visCor_DM1_%s' % (shiftDir, shiftDir)
if 'DM10' in var : plotVar = 'Higgs_PtCor_DM10_%s:m_visCor_DM10_%s' % (shiftDir, shiftDir)
elif 'mjj:m_visCor' in var :
if 'All' in var : plotVar = 'mjj:m_visCor_%s' % shiftDir
if 'DM0' in var : plotVar = 'mjj:m_visCor_DM0_%s' % shiftDir
if 'DM1' in var : plotVar = 'mjj:m_visCor_DM1_%s' % shiftDir
if 'DM10' in var : plotVar = 'mjj:m_visCor_DM10_%s' % shiftDir
elif 'm_visCor' in var :
if 'All' in var : plotVar = 'm_visCor_%s' % shiftDir
if 'DM0' in var : plotVar = 'm_visCor_DM0_%s' % shiftDir
if 'DM1' in var : plotVar = 'm_visCor_DM1_%s' % shiftDir
if 'DM10' in var : plotVar = 'm_visCor_DM10_%s' % shiftDir
elif 'Up' in var :
plotVar = varBase + '_UP'
elif 'Down' in var :
plotVar = varBase + '_DOWN'
elif 'metClustered' in shapeName :
if 'm_sv' in var :
if 'Up' in var[-2:] :
plotVar = plotVar.replace('_metClusteredUp','')
plotVar = plotVar.replace('m_sv','m_sv_ClusteredMet_UP')
plotVar = plotVar.replace('pt_sv','pt_sv_ClusteredMet_UP')
plotVar = plotVar.replace('Higgs_PtCor','Higgs_PtCor_ClusteredMet_UP')
if 'Down' in var[-4:] :
plotVar = plotVar.replace('_metClusteredDown','')
plotVar = plotVar.replace('m_sv','m_sv_ClusteredMet_DOWN')
plotVar = plotVar.replace('pt_sv','pt_sv_ClusteredMet_DOWN')
plotVar = plotVar.replace('Higgs_PtCor','Higgs_PtCor_ClusteredMet_DOWN')
elif 'metUnclustered' in shapeName :
if 'm_sv' in var :
if 'Up' in var[-2:] :
plotVar = plotVar.replace('_metUnclusteredUp','')
plotVar = plotVar.replace('m_sv','m_sv_UncMet_UP')
plotVar = plotVar.replace('pt_sv','pt_sv_UncMet_UP')
plotVar = plotVar.replace('Higgs_PtCor','Higgs_PtCor_UncMet_UP')
if 'Down' in var[-4:] :
plotVar = plotVar.replace('_metUnclusteredDown','')
plotVar = plotVar.replace('m_sv','m_sv_UncMet_DOWN')
plotVar = plotVar.replace('pt_sv','pt_sv_UncMet_DOWN')
plotVar = plotVar.replace('Higgs_PtCor','Higgs_PtCor_UncMet_DOWN')
elif 'JES' in shapeName :
if 'data' in sample :
plotVar = varBase
#if 'mjj:m_sv' in var :
if 'mjj:m_sv' in var or 'mjj:m_visCor' in var :
# Strip _ffSub off for a comparison with the actual shifts
if doFF : compVar = var.replace('_ffSub','')
else : compVar = var
mass = 'm_sv' if 'm_sv' in var else 'm_visCor'
if 'Up' in compVar[-2:] : # Make sure we check the last 2 chars
#plotVar = 'vbfMass_Jet%sUp:m_sv' % jesUnc
plotVar = 'vbfMass_Jet%sUp:%s' % (jesUnc, mass)
if 'Down' in compVar[-4:] : # Make sure we check the last 4 chars
#plotVar = 'vbfMass_Jet%sDown:m_sv' % jesUnc
plotVar = 'vbfMass_Jet%sDown:%s' % (jesUnc, mass)
else : # For this one, we adjust the additionalCuts to
# provide different yields
plotVar = varBase
# Else includes zPt and topPt
else :
plotVar = varBase
#print "Var: %s VarBase: %s" % (var, varBase)
### Make sure that if we have no events
### we still save a blank histo for use later
if chain.GetEntries() == 0 :
print " #### ENTRIES = 0 #### "
if ":" in var :
histos[ var ] = make2DHisto( var )
else :
histos[ var ] = makeHisto( var, info[0], info[1], info[2])
### Check that the target var is in the TTrees
elif hasattr( chain, plotVar ) or ":" in varBase :
#print "trying"
#if sample == 'DYJets' : print sample," Var:",var," VarBase:",varBase, " VarPlot:",plotVar
print "%20s Var: %40s VarBase: %30s VarPlot: %s" % (sample, var, varBase, plotVar)
if isData : # Data has no GenWeight and by def has puweight = 1
dataES = ESCuts( esMap, 'data', channel, var )
#print 'dataES',dataES
chain.Draw( '%s>>%s' % (plotVar, var), '1%s%s%s' % (additionalCutToUse, dataES, ffShapeSyst) )
histos[ var ] = gPad.GetPrimitive( var )
if var == 'm_visCor' :
print 'm_visCor'
#print "Data Count:", histos[ var ].Integral()
print "Cut: %s%s" % (additionalCutToUse, dataES)
else :
chain.Draw( '%s>>%s' % (plotVar, var), '%s' % totalCutAndWeightMC )
''' No reweighting at the moment! '''
histos[ var ] = gPad.GetPrimitive( var )
integralPost = histos[ var ].Integral()
if var == 'm_visCor' :
#print 'm_visCor'
print "tmpIntPost: %f" % integralPost
print "Cut: %s" % totalCutAndWeightMC
# didn't have var in chain
else :
del histos[ var ]
continue
histos[ var ].Write()
#outFile.Write()
#return outFile
outFile.Close()
# -*- coding: utf-8 -*-
import os
import pathlib
import shutil
import redis
from datetime import timedelta
from smart_getenv import getenv
from kombu import Queue
DEBUG = getenv('DEBUG', default=False, type=bool)
FAKE_USER = {
'id': 12345,
'username': '******',
'nickname': 'Sheldon Lee Cooper',
'email': '*****@*****.**',
'privileged': 1,
}
REPO_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PROJECT_NAME = LOGGER_NAME = 'console'
CONFIG_ROOT_DIR = '/etc/kae-console'
CONSOLE_CONFIG_PATHS = [
os.path.join(CONFIG_ROOT_DIR, "config.py"),
os.path.join(REPO_DIR, "local_config.py"),
]
INGRESS_ANNOTATIONS_PREFIX = "nginx.ingress.kubernetes.io"
APP_BUILD_TIMEOUT = 1800 # timeout for build image(30 minutes)
# -*- coding: utf-8 -*-
import os
import redis
from celery.schedules import crontab
from datetime import timedelta
from kombu import Queue
from smart_getenv import getenv
DEBUG = getenv('DEBUG', default=False, type=bool)
FAKE_USER = {
'id': 12345,
'name': 'timfeirg',
'email': '*****@*****.**',
'access_token': 'faketoken',
'privileged': 1,
}
PROJECT_NAME = LOGGER_NAME = 'citadel'
ERU_CONFIG_PATH = getenv(
'ERU_CONFIG_PATH',
default=['citadel/local_config.py', '/etc/eru/citadel.py'])
SERVER_NAME = getenv('SERVER_NAME', default='citadel.test.ricebook.net')
SENTRY_DSN = getenv('SENTRY_DSN', default='')
SECRET_KEY = getenv('SECRET_KEY', default='testsecretkey')
REDIS_URL = getenv('REDIS_URL', default='redis://127.0.0.1:6379/0')
CORE_DEPLOY_INFO_PATH = '/eru-core/deploy'
DEFAULT_ZONE = getenv('DEFAULT_ZONE', default='test-zone')
BUILD_ZONE = getenv('BUILD_ZONE', default='test-zone')
if __name__ == '__main__' :
isoPairs = [
# QCD Syst Mthd Closure ('VLoose','Loose'), # qcd Syst check
# QCD Syst Mthd Closure ('Loose','Tight'), # qcd Syst check
# QCD Syst Mthd Closure ('','Tight'), # qcd Syst check
# QCD Syst Mthd Closure ('','Loose'), # qcd Syst check
('Loose',isoVal), # Normal running
('',isoVal), # Normal running
]
### Check if we intend to do Fake Factor based MC cuts
### These differ because of requiring a random choice
### of l1 and l2, then seeing if l1 is gen matched
### to anything besides a fake/jet
### This is only applied for DYJets, WJets, TT, and QCD MC
doFF = getenv('doFF', type=bool)
if doFF :
# The '' in the following line gives us the signal region
testQCDCuts( folder, samples, isoVal, '', isoVal, 'OS', doFF )
else :
for pair in isoPairs :
for sign in ['OS', 'SS']:
testQCDCuts( folder, samples, isoVal, pair[0], pair[1], sign )
#XXX testQCDCuts( folder, '', 'VTight', 'OS' )
from smart_getenv import getenv
from fabric.api import env
env.disable_known_hosts = True
env.warn_only = True
RUN_DB = {
"db": getenv("LOKTAR_RUN_DB_DB", type=str, default="loktar_ci"),
"table": getenv("LOKTAR_RUN_DB_TABLE", type=str, default="run"),
"host": getenv("LOKTAR_RUN_DB_HOST", type=str, default="elasticsearch"),
"port": getenv("LOKTAR_RUN_DB_PORT", type=int, default=9200)
}
# DEBIAN PACKAGE INFORMATION
CODE_NAME = getenv("LOKTAR_CODE_NAME", type=str, default="rc")
COMPENANT = getenv("LOKTAR_COMPENANT", type=str, default="main")
DEBIAN_REPOSITORY = getenv("LOKTAR_DEBIAN_REPOSITORY", type=str, default=None)
# About retry on command or http requests ...
MAX_RETRY_PYTHON_UPLOAD = getenv("LOKTAR_MAX_RETRY_PYTHON_UPLOAD", type=int, default=3)
MAX_RETRY_DEBIAN_UPLOAD = getenv("LOKTAR_MAX_RETRY_DEBIAN_UPLOAD", type=int, default=3)
MAX_RETRY_DOCKER_BUILD = getenv("LOKTAR_MAX_RETRY_DOCKER_BUILD", type=int, default=3)
MAX_RETRY_DOCKER_PUSH = getenv("LOKTAR_MAX_RETRY_DOCKER_PUSH", type=int, default=3)
MAX_RETRY_GITHUB = getenv("LOKTAR_MAX_RETRY_GITHUB", type=int, default=3)
CONSUL = getenv("CONSUL_HOST", type=str, default=None)
ARTIFACTORY_INFO = {
"login": {
"user": getenv("LOKTAR_ARTIFACTORY_INFO_LOGIN_USER", type=str, default=None),
from smart_getenv import getenv
BOT_NAME = 'steam'
SPIDER_MODULES = ['steam.spiders']
NEWSPIDER_MODULE = 'steam.spiders'
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
ROBOTSTXT_OBEY = True
DOWNLOADER_MIDDLEWARES = {
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': None,
'steam.middlewares.CircumventAgeCheckMiddleware': 600,
}
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_TARGET_CONCURRENCY = 4.0
DUPEFILTER_CLASS = 'steam.middlewares.SteamDupeFilter'
HTTPCACHE_ENABLED = True
HTTPCACHE_EXPIRATION_SECS = 0 # Never expire.
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_IGNORE_HTTP_CODES = [301, 302, 303, 306, 307, 308]
HTTPCACHE_STORAGE = 'steam.middlewares.SteamCacheStorage'
AWS_ACCESS_KEY_ID = getenv('AWS_ACCESS_KEY_ID', type=str, default=None)
AWS_SECRET_ACCESS_KEY = getenv('AWS_SECRET_ACCESS_KEY', type=str, default=None)
# -*- coding: utf-8 -*-
from smart_getenv import getenv
# Turn this off on production
DEBUG = getenv('DEBUG', type=bool, default=False)
# This is mandatory. Please define a secret key - random sequence of characters
SECRET_KEY = getenv('SECRET_KEY', default='')
SQLALCHEMY_DATABASE_URI = '{schema}://{user}:{pwd}@{host}/{dbname}'.format(
schema=getenv('DB_SCHEMA', default='sqlite'),
user=getenv('DB_USER', default=''),
pwd=getenv('DB_PASS', default=''),
host=getenv('DB_HOST', default=''),
dbname=getenv('DB_NAME', default=''))
SQLALCHEMY_BINDS = {
'factsheet': '{schema}://{user}:{pwd}@{host}/{bindname}'.format(
schema=getenv('DB_SCHEMA', default='sqlite'),
user=getenv('DB_USER', default=''),
pwd=getenv('DB_PASS', default=''),
host=getenv('DB_HOST', default=''),
bindname=getenv('BIND_NAME', default='')
)
}
ASSETS_DEBUG = getenv('ASSETS_DEBUG', type=bool, default=False)
AUTH_DEBUG = getenv('AUTH_DEBUG', type=bool, default=False)
EU_ASSESSMENT_MODE = getenv('EU_ASSESSMENT_MODE', type=bool, default=False)
def __getattr__(self, item):
signature = get_signature().get(item, {})
return getenv(item, **signature)
def getHistoDict( analysis, channel ) :
if analysis == 'htt' :
genVarMap = {
#'Z_SS' : [20, -1, 1, 1, 'Z Same Sign', ''],
#XXX 'mjj' : [20, 0, 1000, 1, 'M_{jj} [GeV]', ' GeV'],
#FIXME 'Z_Pt' : [100, 0, 500, 5, 'Z p_{T} [GeV]', ' GeV'],
# 'Higgs_Pt' : [10, 0, 500, 1, 'Higgs p_{T} Uncor [GeV]', ' GeV'],
#XXX 'Higgs_PtCor' : [10, 0, 500, 1, 'Higgs p_{T} [GeV]', ' GeV'],
#XXX 'pt_sv' : [10, 0, 500, 1, 'Higgs svFit p_{T} [GeV]', ' GeV'],
#FIXME 'jdeta' : [20, 0, 10, 1, 'VBF Jets dEta', ' dEta'],
#FIXME# 'Z_DR' : [500, 0, 5, 20, 'Z dR', ' dR'],
#FIXME# 'Z_DPhi' : [800, -4, 4, 40, 'Z dPhi', ' dPhi'],
#FIXME# 'Z_DEta' : [1000, -5, 5, 40, 'Z dEta', ' dEta'],
#FIXME# 'LT' : [600, 0, 300, 20, 'Total LT [GeV]', ' GeV'],
#FIXME# 'Mt' : [600, 0, 400, 40, 'Total m_{T} [GeV]', ' GeV'],
#FIXME 'met' : [250, 0, 250, 20, 'pfMet [GeV]', ' GeV'],
#FIXME# 't1_t2_MvaMet' : [250, 0, 250, 20, 't1 t2 MvaMet [GeV]', ' GeV'],
#FIXME 'metphi' : [80, -4, 4, 10, 'pfMetPhi', ''],
#FIXME# 'mvamet' : [100, 0, 400, 2, 'mvaMetEt [GeV]', ' GeV'],
#FIXME# 'mvametphi' : [100, -5, 5, 2, 'mvaMetPhi', ''],
#FIXME# 'bjetCISVVeto20Medium' : [60, 0, 6, 5, 'nBTag_20Medium', ''],
#FIXME# 'bjetCISVVeto30Medium' : [60, 0, 6, 5, 'nBTag_30Medium', ''],
#FIXME# 'njetspt20' : [100, 0, 10, 10, 'nJetPt20', ''],
#XXX 'jetVeto30' : [100, 0, 10, 10, 'nJetPt30', ''],
#FIXME 'njetingap20' : [100, 0, 10, 10, 'njetingap20', ''],
#FIXME# #'jetVeto40' : [100, 0, 10, 10, 'nJetPt40', ''],
#FIXME# #'nbtag' : [6, 0, 6, 1, 'nBTag', ''],
#FIXME# 'bjetCISVVeto30Tight' : [60, 0, 6, 5, 'nBTag_30Tight', ''],
#FIXME# #'extraelec_veto' : [20, 0, 2, 1, 'Extra Electron Veto', ''],
#FIXME# #'extramuon_veto' : [20, 0, 2, 1, 'Extra Muon Veto', ''],
#FIXME 'jpt_1' : [400, 0, 200, 20, 'Leading Jet Pt', ' GeV'],
#FIXME# 'jeta_1' : [100, -5, 5, 10, 'Leading Jet Eta', ' Eta'],
#FIXME 'jpt_2' : [400, 0, 200, 20, 'Second Jet Pt', ' GeV'],
#FIXME# 'jeta_2' : [100, -5, 5, 10, 'Second Jet Eta', ' Eta'],
#FIXME# #'weight' : [60, -30, 30, 1, 'Gen Weight', ''],
#FIXME# 'npv' : [40, 0, 40, 2, 'Number of Vertices', ''],
#FIXME #'npu' : [50, 1, 40, 2, 'Number of True PU Vertices', ''],
# 'm_vis' : [30, 0, 300, 1, 'M_{vis} Uncor [GeV]', ' GeV'],
'm_visCor' : [30, 0, 300, 1, 'M_{vis} [GeV]', ' GeV'],
#XXX 'mjj:m_visCor' : [300, 0, 300, 10, 'M_{#tau#tau} [GeV]', ' GeV'],
#XXX 'Higgs_PtCor:m_visCor' : [300, 0, 300, 10, 'M_{#tau#tau} [GeV]', ' GeV'],
'Higgs_PtCor:m_sv' : [300, 0, 300, 10, 'M_{#tau#tau} [GeV]', ' GeV'],
'm_sv' : [300, 0, 300, 10, 'M_{#tau#tau} [GeV]', ' GeV'],
#XXX 'pt_sv:m_sv' : [300, 0, 300, 10, 'M_{#tau#tau} [GeV]', ' GeV'],
'mjj:m_sv' : [300, 0, 300, 10, 'M_{#tau#tau} [GeV]', ' GeV'],
# 'mt_sv' : [350, 0, 350, 10, 'Total Transverse Mass [svFit] [GeV]', ' GeV'],
# 'mt_tot' : [3900, 0, 3900, 10, 'Total Transverse Mass [GeV]', ' GeV'],
#'pzetavis' : [300, 0, 300, 20, 'pZetaVis', ' GeV'],
#'pfpzetamis' : [300, 0, 300, 20, 'pfpZetaMis', ' GeV'],
#'pzetamiss' : [500, -200, 300, 20, 'pZetaMis', ' GeV'],
}
''' added shape systematics '''
#toAdd = ['pt_sv:m_sv', 'mjj:m_sv', 'm_visCor', 'm_sv'] # No extra shapes
toAdd = ['pt_sv:m_sv', 'mjj:m_sv', 'm_sv', 'Higgs_PtCor:m_sv'] # No extra shapes
#toAdd = ['Higgs_PtCor:m_visCor', 'mjj:m_visCor', 'm_visCor'] # No extra shapes
#toAdd = ['m_sv', ] # No extra shapes
varsForShapeSyst = []
for item in toAdd :
varsForShapeSyst.append( item )
#shapesToAdd = ['energyScale', 'tauPt', 'topPt', 'zPt']
shapesToAdd = {
'energyScaleAll':'TES All',
'energyScaleDM0':'TES DM0',
'energyScaleDM1':'TES DM1',
'energyScaleDM10':'TES DM10',
'zPt':'Z p_{T}/Mass Reweight',
#'metResponse':'Met Response',
#'metResolution':'Met Resolution',
'tauPt':'High P_{T} Tau',
'topPt':'Top P_{T} Reweight',
'JES' : 'Jet Energy Scale',
'JetToTau' : 'Jet to Tau Fake',
'ggH' : 'ggH Scale',
'topQuarkggH' : 'Top Quark Scale for ggH',
'Zmumu' : 'Z mumu DY Reweight',
'metClustered':'Clustered MET',
'metUnclustered':'Unclustered MET',
}
# Add FF shape systs if doFF
doFF = getenv('doFF', type=bool)
if doFF :
shapesToAdd['qcdffSyst'] = 'FF QCD Syst'
shapesToAdd['ttbarffSyst'] = 'FF ttbar Syst'
shapesToAdd['wjetsffSyst'] = 'FF WJets Syst'
shapesToAdd['0jet1prongffStat'] = 'FF 0Jet 1Prong Stat'
shapesToAdd['0jet3prongffStat'] = 'FF 0Jet 3Prong Stat'
shapesToAdd['1jet1prongffStat'] = 'FF 1Jet 1Prong Stat'
shapesToAdd['1jet3prongffStat'] = 'FF 1Jet 3Prong Stat'
# Add Full JES shapes if doFullJES
doFullJES = getenv('doFullJES', type=bool)
if doFullJES :
# Remove standard JES
if 'JES' in shapesToAdd.keys() : del shapesToAdd['JES']
from util.jetEnergyScale import getUncerts
uncerts = getUncerts()
for uncert in uncerts :
shapesToAdd['JES'+uncert] = 'JES '+uncert
shapesToAdd['JES'+uncert] = 'JES '+uncert
for var in genVarMap.keys() :
if var in varsForShapeSyst :
for shape, app in shapesToAdd.iteritems() :
genVarMap[ var+'_'+shape+'Up' ] = list(genVarMap[ var ])
genVarMap[ var+'_'+shape+'Up' ][4] = genVarMap[ var+'_'+shape+'Up' ][4]+' '+app+' UP'
genVarMap[ var+'_'+shape+'Down' ] = list(genVarMap[ var ])
genVarMap[ var+'_'+shape+'Down' ][4] = genVarMap[ var+'_'+shape+'Down' ][4]+' '+app+' Down'
# Provides a list of histos to create for 'TT' channel
if channel == 'tt' :
chanVarMapTT = {
# 'pt_1' : [200, 0, 200, 5, '#tau_{1} p_{T} Uncor [GeV]', ' GeV'],
# 'pt_2' : [200, 0, 200, 5, '#tau_{2} p_{T} Uncor [GeV]', ' GeV'],
# 'ptCor_1' : [200, 0, 200, 5, '#tau_{1} p_{T} [GeV]', ' GeV'],
# 'ptCor_2' : [200, 0, 200, 5, '#tau_{2} p_{T} [GeV]', ' GeV'],
##FIXME 'gen_match_1' : [14, 0, 7, 1, '#tau_{1} Gen Match', ''],
# 'eta_1' : [60, -3, 3, 4, '#tau_{1} Eta', ' Eta'],
# 'eta_2' : [60, -3, 3, 4, '#tau_{2} Eta', ' Eta'],
# 'decayMode_1' : [15, 0, 15, 1, 't1 Decay Mode', ''],
# 'decayMode_2' : [15, 0, 15, 1, 't2 Decay Mode', ''],
#FIXME 'iso_1' : [100, -1, 1, 1, '#tau_{1} MVArun2v1DBoldDMwLTraw', ''],
#FIXME# 'chargedIsoPtSum_1' : [100, 0, 5, 1, '#tau_{1} charge iso pt sum', ' GeV'],
#FIXME# 'chargedIsoPtSum_2' : [100, 0, 5, 1, '#tau_{2} charge iso pt sum', ' GeV'],
#FIXME# 'chargedIsoPtSumdR03_1' : [100, 0, 5, 1, '#tau_{1} charge iso pt sum dR03', ' GeV'],
#FIXME# 'chargedIsoPtSumdR03_2' : [100, 0, 5, 1, '#tau_{2} charge iso pt sum dR03', ' GeV'],
#FIXME 'gen_match_2' : [14, 0, 7, 1, '#tau_{2} Gen Match', ''],
#FIXME 'iso_2' : [100, -1, 1, 1, '#tau_{2} MVArun2v1DBoldDMwLTraw', ''],
#FIXME# #'t1JetPt' : [400, 0, 400, 20, 't1 Overlapping Jet Pt', ' GeV'],
#FIXME# 'm_1' : [60, 0, 3, 4, 't1 Mass', ' GeV'],
# #'t2JetPt' : [400, 0, 400, 20, 't2 Overlapping Jet Pt', ' GeV'],
# 'm_2' : [60, 0, 3, 4, 't2 Mass', ' GeV'],
#'t1ChargedIsoPtSum' : [0, 10, 8, 't1 ChargedIsoPtSum', ' GeV'],
#'t1NeutralIsoPtSum' : [0, 10, 8, 't1 NeutralIsoPtSum', ' GeV'],
#'t1PuCorrPtSum' : [0, 40, 4, 't1 PuCorrPtSum', ' GeV'],
#'t2ChargedIsoPtSum' : [0, 10, 8, 't2 ChargedIsoPtSum', ' GeV'],
#'t2NeutralIsoPtSum' : [0, 10, 8, 't2 NeutralIsoPtSum', ' GeV'],
#'t2PuCorrPtSum' : [0, 40, 4, 't2 PuCorrPtSum', ' GeV'],
}
for key in chanVarMapTT.keys() :
genVarMap[ key ] = chanVarMapTT[ key ]
return genVarMap
if analysis == 'azh' :
genVarMap = {
# 'Z_Pt' : [400, 0, 400, 40, 'Z p_{T} [GeV]', ' GeV'],
# 'Z_DR' : [500, 0, 5, 50, 'Z dR', ' dR'],
# 'Z_DPhi' : [800, -4, 4, 80, 'Z dPhi', ' dPhi'],
# 'Z_DEta' : [100, -5, 5, 10, 'Z dEta', ' dEta'],
# 'mjj' : [40, 0, 800, 1, 'M_{jj}', ' [GeV]'],
# 'jdeta' : [100, -5, 5, 10, 'VBF dEta', ' dEta'],
'm_vis' : [80, 50, 130, 10, 'Z Mass [GeV]', ' GeV'],
'H_vis' : [400, 0, 400, 40, 'H Visible Mass [GeV]', ' GeV'],
'Mass' : [600, 0, 600, 60, 'M_{ll#tau#tau} [GeV]', ' GeV'],
'LT' : [600, 0, 600, 40, 'Total LT [GeV]', ' GeV'],
'Mt' : [600, 0, 600, 40, 'Total m_{T} [GeV]', ' GeV'],
'LT_higgs' : [150, 0, 150, 10, 'LT_{higgs} [GeV]', ' GeV'],
# 'met' : [250, 0, 250, 20, 'pfMet [GeV]', ' GeV'],
'zhFR0' : [50, 0, 0.5, 2, 'ZH FakeRate Weight 0', ''],
'zhFR1' : [50, 0, 0.5, 2, 'ZH FakeRate Weight 1', ''],
'zhFR2' : [50, 0, 0.5, 2, 'ZH FakeRate Weight 2', ''],
'pt_1' : [200, 0, 200, 10, 'Leg1 p_{T} [GeV]', ' GeV'],
'pt_2' : [200, 0, 200, 10, 'Leg2 p_{T} [GeV]', ' GeV'],
'pt_3' : [200, 0, 200, 10, 'Leg3 p_{T} [GeV]', ' GeV'],
'pt_4' : [200, 0, 200, 10, 'Leg4 p_{T} [GeV]', ' GeV'],
# 'eta_1' : [60, -3, 3, 10, 'Leg1 Eta', ' Eta'],
# 'eta_2' : [60, -3, 3, 10, 'Leg2 Eta', ' Eta'],
# 'eta_3' : [60, -3, 3, 10, 'Leg3 Eta', ' Eta'],
# 'eta_4' : [60, -3, 3, 10, 'Leg4 Eta', ' Eta'],
# 'iso_1' : [20, 0, 0.5, 1, 'Leg1 RelIsoDB03', ''],
# 'iso_2' : [20, 0, 0.5, 1, 'Leg2 RelIsoDB03', ''],
'iso_3' : [20, 0, 1, 1, 'Leg3 Iso', ''],
'iso_4' : [20, 0, 1, 1, 'Leg4 Iso', ''],
#'jpt_1' : [400, 0, 200, 20, 'Leading Jet Pt', ' GeV'],
#'jeta_1' : [100, -5, 5, 10, 'Leading Jet Eta', ' Eta'],
#'jpt_2' : [400, 0, 200, 20, 'Second Jet Pt', ' GeV'],
#'jeta_2' : [100, -5, 5, 10, 'Second Jet Eta', ' Eta'],
#'weight' : [60, -30, 30, 1, 'Gen Weight', ''],
# 'npv' : [40, 0, 40, 4, 'Number of Vertices', ''],
## 'njetspt20' : [100, 0, 10, 10, 'nJetPt20', ''],
# 'jetVeto30' : [100, 0, 10, 10, 'nJetPt30', ''],
## 'azhWeight' : [50, 0, 2, 1, 'Muon + Electron Weights', ''],
# 'muVetoZTTp001dxyz' : [6, -1, 5, 1, 'muVetoZTTp001dxyz', ''],
# 'eVetoZTTp001dxyz' : [6, -1, 5, 1, 'eVetoZTTp001dxyz', ''],
# 'muVetoZTTp001dxyzR0' : [6, -1, 5, 1, 'muVetoZTTp001dxyzR0', ''],
# 'eVetoZTTp001dxyzR0' : [6, -1, 5, 1, 'eVetoZTTp001dxyzR0', ''],
# 'bjetCISVVeto20Medium' : [60, 0, 6, 5, 'nBTag_20Medium', ''],
# 'bjetCISVVeto30Medium' : [60, 0, 6, 5, 'nBTag_30Medium', ''],
# 'bjetCISVVeto30Tight' : [60, 0, 6, 5, 'nBTag_30Tight', ''],
}
llltMap = {
# 'againstElectronVLooseMVA6_4' : [9, -1, 2, 1, 'Against E VL MVA6 Leg 4', ''],
# 'againstElectronLooseMVA6_4' : [9, -1, 2, 1, 'Against E L MVA6 Leg 4', ''],
# 'againstMuonLoose3_4' : [9, -1, 2, 1, 'Against M Loose 3 Leg 4', ''],
# 'againstMuonTight3_4' : [9, -1, 2, 1, 'Against M Tight 3 Leg 4', ''],
}
llttMap = {
# 'againstElectronVLooseMVA6_3' : [9, -1, 2, 1, 'Against E VL MVA6 Leg 3', ''],
# 'againstElectronLooseMVA6_3' : [9, -1, 2, 1, 'Against E L MVA6 Leg 3', ''],
# 'againstMuonLoose3_3' : [9, -1, 2, 1, 'Against M Loose 3 Leg 3', ''],
# 'againstMuonTight3_3' : [9, -1, 2, 1, 'Against M Tight 3 Leg 3', ''],
}
if channel == 'xxxx' :
return genVarMap
if channel in ['eeet', 'eemt', 'eett', 'emmt', 'mmmt', 'mmtt'] :
for var in llltMap.keys() :
genVarMap[var] = llltMap[ var ]
if channel in ['eett', 'mmtt'] :
for var in llttMap.keys() :
genVarMap[var] = llttMap[ var ]
return genVarMap
ChainedTokenCredential,
ManagedIdentityCredential,
EnvironmentCredential,
)
from azure.storage.queue import QueueServiceClient
from azure.ai.formrecognizer import FormRecognizerClient
from azure.ai.textanalytics import TextAnalyticsClient
from azure.cosmos import CosmosClient
from smart_getenv import getenv
from dotmap import DotMap
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv())
sleep = getenv("AZURE_STORAGE_QUEUE_RECEIVE_SLEEP", type=bool, default=1)
credential = ChainedTokenCredential(AzureCliCredential(),
EnvironmentCredential(),
ManagedIdentityCredential())
queue_service_client = QueueServiceClient(
account_url=getenv("AZURE_STORAGE_QUEUE_ENDPOINT"), credential=credential)
queue_client = queue_service_client.get_queue_client(
queue=getenv("AZURE_STORAGE_QUEUE_NAME", default="messages"))
fr_client = FormRecognizerClient(
endpoint=getenv("AZURE_FORM_RECOGNIZER_ENDPOINT"), credential=credential)
ta_client = TextAnalyticsClient(
import click
from kirby.demo import create_demo_db
from kirby.models import db
from kirby.models.security import user_datastore
from kirby.supervisor import run_supervisor
from kirby.web import app_maker
from smart_getenv import getenv
import logging
DEFAULT_LOG_FORMAT = "[%(asctime)s] %(levelname)s:%(name)s:%(message)s"
logging.basicConfig(
level=logging.DEBUG,
format=getenv("LOG_FORMAT", default=DEFAULT_LOG_FORMAT),
)
logging.getLogger("kafka").setLevel(logging.CRITICAL)
logger = logging.getLogger(__name__)
def read_topic(name):
"""
debug function to consume the messages in a kafka topic
:param name: name of the topic
"""
from kafka import KafkaConsumer
import msgpack
import json
from pprint import pformat
from slack import RTMClient
from smart_getenv import getenv
from snark_bot.bot import SlackBot
name = getenv("SLACK_BOT_NAME", type=str)
token = getenv("SLACK_BOT_TOKEN", type=str)
pattern = getenv("SLACK_BOT_RESPONSE_PATTERN", type=str)
remarks_location = getenv("SNARKY_REMARKS_LOCATION", type=str)
bot = SlackBot(name, pattern, remarks_location, token=token)
@RTMClient.run_on(event="message")
def _message_trigger(**kwargs):
bot.process_message(**kwargs)
if __name__ == "__main__":
bot.start()
# -*- coding: utf-8 -*-
from smart_getenv import getenv
# Turn this off on production
DEBUG = getenv('DEBUG', type=bool, default=False)
# This is mandatory. Please define a secret key - random sequence of characters
SECRET_KEY = getenv('SECRET_KEY', default='')
SQLALCHEMY_DATABASE_URI = '{schema}://{user}:{pwd}@{host}/{dbname}'.format(
schema=getenv('DB_SCHEMA', default='sqlite'),
user=getenv('DB_USER', default=''),
pwd=getenv('DB_PASS', default=''),
host=getenv('DB_HOST', default=''),
dbname=getenv('DB_NAME', default=''))
SQLALCHEMY_BINDS = {
'factsheet': '{schema}://{user}:{pwd}@{host}/{bindname}'.format(
schema=getenv('DB_SCHEMA', default='sqlite'),
user=getenv('DB_USER', default=''),
pwd=getenv('DB_PASS', default=''),
host=getenv('DB_HOST', default=''),
bindname=getenv('BIND_NAME', default='')
)
}
ASSETS_DEBUG = getenv('ASSETS_DEBUG', type=bool, default=False)
AUTH_DEBUG = getenv('AUTH_DEBUG', type=bool, default=False)
AUTH_LOG_FILE = getenv('AUTH_LOG_FILE', default='/var/local/art12/logs/flask-auth.log')
