def get_snt_samples(entity, selectors, short=True):
from db import DBInterface
db = DBInterface(fname="allsamples.db")
match_dict = {"dataset_name": entity}
# if we didn't specify a sample_type, then assume we only want CMS3 and not BABY
if not selectors or not any(map(lambda x: "sample_type" in x, selectors)):
# but if user specified an analysis, then don't restrict to CMS3
if not any(map(lambda x: "analysis" in x, selectors)) \
and not any(map(lambda x: "baby_tag" in x, selectors)):
selectors.append("sample_type=CMS3")
if selectors:
for more in selectors:
key = more.split("=")[0].strip()
val = more.split("=")[1].strip()
match_dict[key] = val
samples = db.fetch_samples_matching(match_dict)
samples = sorted(
samples,
key=lambda x: x.get("cms3tag", "") + str(x.get("nevents_out", "0")),
reverse=True)
if short:
# import datetime
new_samples = []
for sample in samples:
if sample["sample_type"] == "CMS3":
for key in ["sample_id","filter_type","assigned_to", \
"comments","twiki_name", "analysis", "baby_tag"]:
del sample[key]
elif sample["sample_type"] == "BABY":
for key in ["sample_id","filter_type","assigned_to", \
"comments","twiki_name","xsec","gtag","nevents_in","nevents_out","kfactor","filter_eff"]:
del sample[key]
# if "timestamp" in sample:
# sample["timestamp"] = str(datetime.datetime.fromtimestamp(sample["timestamp"]))
new_samples.append(sample)
samples = new_samples
return samples
def __init__(self):
"""constructor of TiffanyBot. make an instance of Skype4Py
to register itself"""
self._skype = SkypeEventHandler(DBInterface(DB_NAME))
return
# "Run2_Data",
"Run2_Data2016",
"Run2Samples25ns80X",
"Run2Samples25ns80XminiAODv2",
"Run2Samples25ns80XPrivate",
# "Run2SamplesPrivateSMSFastSim25ns",
# "Run2Samples25ns76XminiAODv2",
# "Run2SamplesSMSFastSim_25ns",
# "Run2SamplesReMINIAOD_25ns",
# "Run2Samples_25ns",
"Run2Samples25ns80XFS",
"Run2Samples25nsMoriond17",
"testTwiki", # used for testing various things
]
db = DBInterface(fname="allsamples.db")
db.drop_table()
db.make_table()
pool = ThreadPool(4)
site_samples = pool.map(get_samples, sites)
pool.close()
pool.join()
isample = 0
for site,samples in zip(sites,site_samples):
for sample in samples:
s = {}
if "dataset" not in sample: continue
#!/usr/bin/python3
import sys
import Adafruit_DHT
from db import DBInterface
sensor = Adafruit_DHT.DHT22
pin =27
db=DBInterface("config.ini")
humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)
db.storeData(temperature,humidity)
db=None
# print 'Temp: {0:0.1f} C Humidity: {1:0.1f} %'.format(temperature, humidity)
def do_test():
sample = {
"dataset_name":
"/DYJetsToLL_M-10to50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIISpring15MiniAODv2-74X_mcRun2_asymptotic_v2-v1/MINIAODSIM",
"twiki_name": "Run2SamplesReMINIAOD_25ns",
"sample_type": "CMS3",
"filter_type": "NoFilter",
"nevents_in": 30301601,
"nevents_out": 30301601,
"xsec": 18610,
"kfactor": 1.11,
"filter_eff": 1,
"baby_tag": "",
"analysis": "",
"timestamp": int(time.time()),
"gtag": "74X_mcRun2_asymptotic_v2",
"cms3tag": "CMS3_V07-04-11",
"location":
"/hadoop/cms/store/group/snt/run2_25ns_MiniAODv2/DYJetsToLL_M-10to50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8_RunIISpring15MiniAODv2-74X_mcRun2_asymptotic_v2-v1/V07-04-11/",
"assigned_to": "Rafael",
"comments": "xsec from MCM"
}
# :memory: uses RAM instead of saving a file
# db = DBInterface(fname=":memory:")
db = DBInterface(fname=":memory:")
# empty DB and re-initialize table
db.drop_table()
db.make_table()
# make some more fake samples
sample1 = sample.copy()
sample2 = sample.copy()
sample2["cms3tag"] = "the_tag_2"
sample3 = sample.copy()
sample3["cms3tag"] = "the_tag_3"
sample3["nevents_out"] = 12345
# insert samples
db.update_sample(sample1)
db.update_sample(sample2)
db.update_sample(sample3)
# next line shouldn't add a new row since there's duplicate checking
db.update_sample(sample3)
assert (len(db.fetch_samples_matching({"sample_type": "CMS3"})) == 3
) # should match all three samples
assert (len(db.fetch_samples_matching({"cms3tag": "the_tag_2"})) == 1
) # should match only sample2
assert (len(db.fetch_samples_matching({"nevents_out": 12345})) == 1
) # should match only sample3
assert (len(
db.fetch_samples_matching({
"cms3tag": "the_tag_3",
"nevents_out": 12345
})) == 1) # should match only sample3
# get sample2, change the evt count (leaving the rest), update it
sample = db.fetch_samples_matching({"cms3tag": "the_tag_2"})[0]
sample["nevents_out"] = 666
db.update_sample(sample)
assert (len(
db.fetch_samples_matching({
"cms3tag": "the_tag_2",
"nevents_out": 666
})) == 1)
assert (len(db.fetch_samples_matching({"sample_type": "CMS3"})) == 3
) # still only have 3 samples
assert (len(db.read_to_dict_list("select * from sample")) == 3
) # check that this method returns all samples
assert (len(db.fetch_samples_matching({"cms3tag": "the_tag_4"})) == 0
) # shouldn't match anything
# if we change the cms3tag and try to update the sample, it's actually a new sample, so another row gets added
sample = db.fetch_samples_matching({"cms3tag": "the_tag_2"})[0]
sample["cms3tag"] = "the_tag_4"
db.update_sample(sample)
assert (len(db.fetch_samples_matching({"cms3tag": "the_tag_4"})) == 1
) # should now match one sample
assert (len(db.fetch_samples_matching({"sample_type": "CMS3"})) == 4
) # and we have 4 rows now
assert (db.fetch_samples_matching({"duck": 1}) == []
) # unrecognized key returns nothing
assert (db.fetch_samples_matching({}) == []
) # empty selection dict returns nothing
assert (db.update_sample({}) == False) # don't do anything if empty
assert (db.update_sample({"duck": 1}) == False
) # don't do anything if unrecognized key
assert (len(db.fetch_samples_matching({"cms3tag": "*_tag_*"})) == 3
) # check wildcard support
db.close()
# print "Calculations correct"
return True
def do_test():
sample = {
"dataset_name": "/DYJetsToLL_M-10to50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIISpring15MiniAODv2-74X_mcRun2_asymptotic_v2-v1/MINIAODSIM",
"twiki_name": "Run2SamplesReMINIAOD_25ns",
"sample_type": "CMS3",
"filter_type": "NoFilter",
"nevents_in": 30301601,
"nevents_out": 30301601,
"xsec": 18610,
"kfactor": 1.11,
"filter_eff": 1,
"gtag": "74X_mcRun2_asymptotic_v2",
"cms3tag": "CMS3_V07-04-11",
"location": "/hadoop/cms/store/group/snt/run2_25ns_MiniAODv2/DYJetsToLL_M-10to50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8_RunIISpring15MiniAODv2-74X_mcRun2_asymptotic_v2-v1/V07-04-11/",
"assigned_to": "Rafael",
"comments": "xsec from MCM"
}
# :memory: uses RAM instead of saving a file
# db = DBInterface(fname=":memory:")
db = DBInterface(fname=":memory:")
# empty DB and re-initialize table
db.drop_table()
db.make_table()
# make some more fake samples
sample1 = sample.copy()
sample2 = sample.copy()
sample2["cms3tag"] = "the_tag_2"
sample3 = sample.copy()
sample3["cms3tag"] = "the_tag_3"
sample3["nevents_out"] = 12345
# insert samples
db.update_sample(sample1)
db.update_sample(sample2)
db.update_sample(sample3)
# next line shouldn't add a new row since there's duplicate checking
db.update_sample(sample3)
assert( len(db.fetch_samples_matching({"sample_type":"CMS3"})) == 3 ) # should match all three samples
assert( len(db.fetch_samples_matching({"cms3tag":"the_tag_2"})) == 1 ) # should match only sample2
assert( len(db.fetch_samples_matching({"nevents_out":12345})) == 1 ) # should match only sample3
assert( len(db.fetch_samples_matching({"cms3tag":"the_tag_3","nevents_out":12345})) == 1 ) # should match only sample3
# get sample2, change the evt count (leaving the rest), update it
sample = db.fetch_samples_matching({"cms3tag":"the_tag_2"})[0]
sample["nevents_out"] = 666
db.update_sample(sample)
assert( len(db.fetch_samples_matching({"cms3tag":"the_tag_2","nevents_out":666})) == 1 )
assert( len(db.fetch_samples_matching({"sample_type":"CMS3"})) == 3 ) # still only have 3 samples
assert( len(db.read_to_dict_list("select * from sample")) == 3 ) # check that this method returns all samples
assert( len(db.fetch_samples_matching({"cms3tag":"the_tag_4"})) == 0 ) # shouldn't match anything
# if we change the cms3tag and try to update the sample, it's actually a new sample, so another row gets added
sample = db.fetch_samples_matching({"cms3tag":"the_tag_2"})[0]
sample["cms3tag"] = "the_tag_4"
db.update_sample(sample)
assert( len(db.fetch_samples_matching({"cms3tag":"the_tag_4"})) == 1 ) # should now match one sample
assert( len(db.fetch_samples_matching({"sample_type":"CMS3"})) == 4 ) # and we have 4 rows now
assert( db.fetch_samples_matching({"duck":1}) == [] ) # unrecognized key returns nothing
assert( db.fetch_samples_matching({}) == [] ) # empty selection dict returns nothing
assert( db.update_sample({}) == False ) # don't do anything if empty
assert( db.update_sample({"duck":1}) == False ) # don't do anything if unrecognized key
assert( len(db.fetch_samples_matching({"cms3tag":"*_tag_*"})) == 3 ) # check wildcard support
db.close()
# print "Calculations correct"
return True
def handle_query(arg_dict):
if not arg_dict: return
query_type = arg_dict.get("type", "basic")
query = arg_dict.get("query", None).strip()
short = arg_dict.get("short", False)
# parse extra information in query if it's not just the dataset
# /Gjet*/*/*, cms3tag=*07*06* | grep location,dataset_name
# ^^dataset^^ ^^^selectors^^^ ^^^^^^^^^^^pipes^^^^^^^^^^
selectors = []
pipes = []
if "|" in query:
first = query.split("|")[0].strip()
pipes = query.split("|")[1:]
if "," in first:
entity = first.split(",")[0].strip()
selectors = first.split(",")[1:]
else:
entity = first
elif "," in query:
entity = query.split(",")[0].strip()
selectors = query.split(",")[1:]
else:
entity = query.strip()
failed = False
fail_reason = ""
warning = ""
payload = {}
if "*" in entity and query_type in ["basic", "files"]:
query_type = "listdatasets"
# # HUGE SLOWDONW (4 seconds :( )
# if query_type in ["basic", "files", "listdatasets", "mcm", "driver", "lhe", "parents", "dbs"]:
# if proxy_hours_left() < 5: proxy_renew()
if not entity:
failed = True
fail_reason = "Dataset not specified"
try:
if query_type == "snt":
payload = get_snt_samples(entity, selectors, short)
elif query_type == "basic":
info = dataset_event_count(entity)
if not info:
failed = True
fail_reason = "Dataset not found"
payload = info
elif query_type == "listdatasets":
datasets = list_of_datasets(entity, short, selectors=selectors)
if not datasets:
failed = True
fail_reason = "No datasets found"
payload = datasets
elif query_type == "sites":
if "/store/" in entity:
payload = get_file_replicas(entity)
else:
if short:
datasets = list_of_datasets(entity, short=True)
if not datasets:
failed = True
fail_reason = "No datasets found"
payload = get_replica_fractions(datasets)
else:
payload = get_file_replicas(entity, typ="dataset")
elif query_type == "files":
files = get_dataset_files(entity, selectors=selectors)
payload = filelist_to_dict(files, short, num=10)
elif query_type == "runs":
payload = get_dataset_runs(entity)
elif query_type == "pick":
payload, warning = get_pick_events(entity, selectors)
elif query_type == "pick_cms4":
payload = get_pick_cms4(entity, selectors)
elif query_type == "config":
config_info = get_dataset_config(entity)
payload = config_info
elif query_type == "mcm":
if selectors:
if "this" == selectors[0].lower():
gen_sim = entity
else:
gen_sim = get_specified_parent(entity,
typ="GEN-SIM",
fallback="AODSIM")
if short:
info = get_slim_mcm_json(gen_sim)
else:
info = get_mcm_json(gen_sim)["results"]
info["sample"] = gen_sim
payload = info
elif query_type == "parents":
lineage = []
dataset = entity
while dataset:
try:
dataset = get_dataset_parent(dataset)
except:
break
if dataset: lineage.append(dataset)
info = {}
info["parents"] = lineage
payload = info
elif query_type == "driver":
if selectors:
if "this" == selectors[0].lower():
gen_sim = entity
else:
gen_sim = get_specified_parent(entity,
typ="GEN-SIM",
fallback="AODSIM")
info = get_mcm_json(gen_sim)["results"]
dataset_base = info["dataset_name"]
campaign = info["prepid"]
driver = get_mcm_setup(campaign)
payload = {"dataset": dataset_base, "cmsDriver": driver}
elif query_type == "lhe":
lhe = get_specified_parent(entity, typ="LHE")
# lhe = get_dataset_parent(gen_sim)
files = get_dataset_files(lhe)
payload["files"] = filelist_to_dict(files, short)
elif query_type == "update_snt":
from db import DBInterface
db = DBInterface(fname="allsamples.db")
s = {}
for keyval in query.split(","):
key, val = keyval.strip().split("=")
s[key] = val
payload = s
did_update = db.update_sample(s)
db.close()
payload["updated"] = did_update
elif query_type == "delete_snt":
from db import DBInterface
db = DBInterface(fname="allsamples.db")
s = {}
for keyval in query.split(","):
key, val = keyval.strip().split("=")
s[key] = val
payload = s
did_delete = db.delete_sample(s)
db.close()
payload["deleted"] = did_delete
elif query_type == "dbs":
if "raw" in selectors:
import base64
payload = get_url_with_cert(query.rsplit(",", 1)[0],
return_raw=True)
payload = base64.b64encode(payload)
else:
payload = get_url_with_cert(query)
# payload = "/DYJetsToLL_M-50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIISpring16MiniAODv1-PUSpring16_80X_mcRun2_asymptotic_2016_v3-v1/MINIAODSIM"
# payload = "/DYJetsToLL_M-50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIISpring16MiniAODv1-PUSpring16_80X_mcRun2_asymptotic_2016_v3-v1/MINIAODSIM"
else:
failed = True
fail_reason = "Query type not found"
except:
failed = True
fail_reason = traceback.format_exc()
if pipes:
for pipe in pipes:
pipe = pipe.strip()
parts = pipe.split(" ", 1)
if len(parts) == 1:
verb = parts[0].strip()
elif len(parts) == 2:
verb, keys = parts
keys = map(lambda x: x.strip(), keys.split(","))
else:
verb = parts[0]
if verb == "grep":
if type(payload) == list:
for ipay in range(len(payload)):
if type(payload[ipay]) is not dict: continue
if len(keys) > 1:
d = {}
for key in keys:
if key in payload[ipay]:
d[key] = payload[ipay][key]
else:
d[key] = get_derived_quantity(
payload[ipay], key)
payload[ipay] = d
else:
payload[ipay] = payload[ipay].get(keys[0], None)
elif type(payload) == dict:
if len(keys) > 0:
d = {}
for key in keys:
d[key] = payload.get(key, None)
payload = d
elif verb == "stats":
if type(payload) == list:
nums = []
for elem in payload:
try:
nums.append(float(elem))
except:
pass
if len(nums) > 0:
payload = {
"N": len(nums),
"total": sum(nums),
"minimum": min(nums),
"maximum": max(nums),
}
else:
payload = {"N": len(payload)}
elif verb == "sort":
if type(payload) == list:
payload = sorted(payload)
return make_response(arg_dict, payload, failed, fail_reason, warning)
def handle_query(arg_dict):
query_type = arg_dict.get("type","basic")
query = arg_dict.get("query", None).strip()
short = arg_dict.get("short", False)
# parse extra information in query if it's not just the dataset
# /Gjet*/*/*, cms3tag=*07*06* | grep location,dataset_name
# ^^dataset^^ ^^^selectors^^^ ^^^^^^^^^^^pipes^^^^^^^^^^
selectors = None
pipes = None
if "|" in query:
first = query.split("|")[0].strip()
pipes = query.split("|")[1:]
if "," in first:
entity = first.split(",")[0].strip()
selectors = first.split(",")[1:]
else:
entity = first
elif "," in query:
entity = query.split(",")[0].strip()
selectors = query.split(",")[1:]
else:
entity = query.strip()
failed = False
fail_reason = ""
warning = ""
payload = {}
if "*" in entity and query_type in ["basic","files"]:
query_type = "listdatasets"
if query_type in ["basic", "files", "listdatasets", "mcm", "driver", "lhe", "parents", "dbs"]:
if proxy_hours_left() < 5: proxy_renew()
if not entity:
failed = True
fail_reason = "Dataset not specified"
try:
if query_type == "basic":
info = dataset_event_count(entity)
if not info:
failed = True
fail_reason = "Dataset not found"
payload = info
elif query_type == "listdatasets":
datasets = list_of_datasets(entity, short)
if not datasets:
failed = True
fail_reason = "No datasets found"
payload = datasets
elif query_type == "files":
files = get_dataset_files(entity)
payload = filelist_to_dict(files, short)
elif query_type == "runs":
payload = get_dataset_runs(entity)
elif query_type == "pick":
payload, warning = get_pick_events(entity, selectors)
elif query_type == "pick_cms3":
payload, warning = get_pick_cms3(entity, selectors)
elif query_type == "config":
config_info = get_dataset_config(entity)
payload = config_info
elif query_type == "mcm":
if selectors:
if "this" == selectors[0].lower():
gen_sim = entity
else:
gen_sim = get_specified_parent(entity, typ="GEN-SIM", fallback="AODSIM")
if short:
info = get_slim_mcm_json(gen_sim)
else:
info = get_mcm_json(gen_sim)["results"]
info["sample"] = gen_sim
payload = info
elif query_type == "parents":
lineage = []
dataset = entity
while dataset:
try:
dataset = get_dataset_parent(dataset)
except:
break
if dataset: lineage.append(dataset)
info = {}
info["parents"] = lineage
payload = info
elif query_type == "driver":
if selectors:
if "this" == selectors[0].lower():
gen_sim = entity
else:
gen_sim = get_specified_parent(entity, typ="GEN-SIM", fallback="AODSIM")
info = get_mcm_json(gen_sim)["results"]
dataset_base = info["dataset_name"]
campaign = info["prepid"]
driver = get_mcm_setup(campaign)
payload = { "dataset": dataset_base, "cmsDriver": driver }
elif query_type == "lhe":
lhe = get_specified_parent(entity, typ="LHE")
# lhe = get_dataset_parent(gen_sim)
files = get_dataset_files(lhe)
payload["files"] = filelist_to_dict(files, short)
elif query_type == "snt":
from db import DBInterface
db = DBInterface(fname="allsamples.db")
match_dict = {"dataset_name": entity}
if selectors:
for more in selectors:
key = more.split("=")[0].strip()
val = more.split("=")[1].strip()
match_dict[key] = val
samples = db.fetch_samples_matching(match_dict)
if short:
new_samples = []
for sample in samples:
for key in ["sample_id","filter_type","assigned_to", \
"comments","twiki_name","sample_type"]:
del sample[key]
new_samples.append(sample)
samples = new_samples
payload = samples
elif query_type == "update_snt":
from db import DBInterface
db = DBInterface(fname="allsamples.db")
s = {}
for keyval in query.split(","):
key,val = keyval.strip().split("=")
s[key] = val
payload = s
did_update = db.update_sample(s)
db.close()
payload["updated"] = did_update
elif query_type == "dbs":
payload = get_url_with_cert(query)
# payload = "/DYJetsToLL_M-50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIISpring16MiniAODv1-PUSpring16_80X_mcRun2_asymptotic_2016_v3-v1/MINIAODSIM"
else:
failed = True
fail_reason = "Query type not found"
except:
failed = True
fail_reason = traceback.format_exc()
if pipes:
for pipe in pipes:
pipe = pipe.strip()
parts = pipe.split(" ",1)
if len(parts) == 1:
verb = parts[0].strip()
elif len(parts) == 2:
verb, keys = parts
keys = map(lambda x: x.strip(), keys.split(","))
else:
verb = parts[0]
if verb == "grep":
if type(payload) == list:
for ipay in range(len(payload)):
if type(payload[ipay]) is not dict: continue
if len(keys) > 1:
d = {}
for key in keys: d[key] = payload[ipay].get(key,None)
payload[ipay] = d
else:
payload[ipay] = payload[ipay].get(keys[0],None)
elif type(payload) == dict:
if len(keys) > 0:
d = {}
for key in keys: d[key] = payload.get(key,None)
payload = d
elif verb == "stats":
if type(payload) == list:
nums = []
for elem in payload:
try: nums.append(float(elem))
except: pass
if len(nums) > 0:
payload = {
"N": len(nums),
"total": sum(nums),
"minimum": min(nums),
"maximum": max(nums),
}
else:
payload = {"N": len(payload)}
return make_response(arg_dict, payload, failed, fail_reason, warning)
"Run2Samples25ns80X",
"Run2Samples25ns80XminiAODv2",
"Run2Samples25ns80XPrivate",
"Run2SamplesPrivateSMSFastSim25ns",
"Run2Samples25ns76XminiAODv2",
"Run2SamplesSMSFastSim_25ns",
"Run2SamplesReMINIAOD_25ns",
"Run2Samples_25ns",
"Run2Samples25ns80XFS",
"testTwiki", # used for testing various things
# "SMS_T5ttttDM175_74X",
# "SMS_T5qqqqWW_74X",
# "SMS_T5ttcc_74X",
]
db = DBInterface(fname="allsamples.db")
db.drop_table()
db.make_table()
pool = ThreadPool(4)
site_samples = pool.map(get_samples, sites)
pool.close()
pool.join()
isample = 0
for site,samples in zip(sites,site_samples):
for sample in samples:
s = {}
if "dataset" not in sample: continue
