def get_snt_samples(entity, selectors, short=True): from db import DBInterface db = DBInterface(fname="allsamples.db") match_dict = {"dataset_name": entity} # if we didn't specify a sample_type, then assume we only want CMS3 and not BABY if not selectors or not any(map(lambda x: "sample_type" in x, selectors)): # but if user specified an analysis, then don't restrict to CMS3 if not any(map(lambda x: "analysis" in x, selectors)) \ and not any(map(lambda x: "baby_tag" in x, selectors)): selectors.append("sample_type=CMS3") if selectors: for more in selectors: key = more.split("=")[0].strip() val = more.split("=")[1].strip() match_dict[key] = val samples = db.fetch_samples_matching(match_dict) samples = sorted( samples, key=lambda x: x.get("cms3tag", "") + str(x.get("nevents_out", "0")), reverse=True) if short: # import datetime new_samples = [] for sample in samples: if sample["sample_type"] == "CMS3": for key in ["sample_id","filter_type","assigned_to", \ "comments","twiki_name", "analysis", "baby_tag"]: del sample[key] elif sample["sample_type"] == "BABY": for key in ["sample_id","filter_type","assigned_to", \ "comments","twiki_name","xsec","gtag","nevents_in","nevents_out","kfactor","filter_eff"]: del sample[key] # if "timestamp" in sample: # sample["timestamp"] = str(datetime.datetime.fromtimestamp(sample["timestamp"])) new_samples.append(sample) samples = new_samples return samples
def __init__(self): """constructor of TiffanyBot. make an instance of Skype4Py to register itself""" self._skype = SkypeEventHandler(DBInterface(DB_NAME)) return
# "Run2_Data", "Run2_Data2016", "Run2Samples25ns80X", "Run2Samples25ns80XminiAODv2", "Run2Samples25ns80XPrivate", # "Run2SamplesPrivateSMSFastSim25ns", # "Run2Samples25ns76XminiAODv2", # "Run2SamplesSMSFastSim_25ns", # "Run2SamplesReMINIAOD_25ns", # "Run2Samples_25ns", "Run2Samples25ns80XFS", "Run2Samples25nsMoriond17", "testTwiki", # used for testing various things ] db = DBInterface(fname="allsamples.db") db.drop_table() db.make_table() pool = ThreadPool(4) site_samples = pool.map(get_samples, sites) pool.close() pool.join() isample = 0 for site,samples in zip(sites,site_samples): for sample in samples: s = {} if "dataset" not in sample: continue
#!/usr/bin/python3 import sys import Adafruit_DHT from db import DBInterface sensor = Adafruit_DHT.DHT22 pin =27 db=DBInterface("config.ini") humidity, temperature = Adafruit_DHT.read_retry(sensor, pin) db.storeData(temperature,humidity) db=None # print 'Temp: {0:0.1f} C Humidity: {1:0.1f} %'.format(temperature, humidity)
def do_test(): sample = { "dataset_name": "/DYJetsToLL_M-10to50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIISpring15MiniAODv2-74X_mcRun2_asymptotic_v2-v1/MINIAODSIM", "twiki_name": "Run2SamplesReMINIAOD_25ns", "sample_type": "CMS3", "filter_type": "NoFilter", "nevents_in": 30301601, "nevents_out": 30301601, "xsec": 18610, "kfactor": 1.11, "filter_eff": 1, "baby_tag": "", "analysis": "", "timestamp": int(time.time()), "gtag": "74X_mcRun2_asymptotic_v2", "cms3tag": "CMS3_V07-04-11", "location": "/hadoop/cms/store/group/snt/run2_25ns_MiniAODv2/DYJetsToLL_M-10to50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8_RunIISpring15MiniAODv2-74X_mcRun2_asymptotic_v2-v1/V07-04-11/", "assigned_to": "Rafael", "comments": "xsec from MCM" } # :memory: uses RAM instead of saving a file # db = DBInterface(fname=":memory:") db = DBInterface(fname=":memory:") # empty DB and re-initialize table db.drop_table() db.make_table() # make some more fake samples sample1 = sample.copy() sample2 = sample.copy() sample2["cms3tag"] = "the_tag_2" sample3 = sample.copy() sample3["cms3tag"] = "the_tag_3" sample3["nevents_out"] = 12345 # insert samples db.update_sample(sample1) db.update_sample(sample2) db.update_sample(sample3) # next line shouldn't add a new row since there's duplicate checking db.update_sample(sample3) assert (len(db.fetch_samples_matching({"sample_type": "CMS3"})) == 3 ) # should match all three samples assert (len(db.fetch_samples_matching({"cms3tag": "the_tag_2"})) == 1 ) # should match only sample2 assert (len(db.fetch_samples_matching({"nevents_out": 12345})) == 1 ) # should match only sample3 assert (len( db.fetch_samples_matching({ "cms3tag": "the_tag_3", "nevents_out": 12345 })) == 1) # should match only sample3 # get sample2, change the evt count (leaving the rest), update it sample = db.fetch_samples_matching({"cms3tag": "the_tag_2"})[0] sample["nevents_out"] = 666 db.update_sample(sample) assert (len( db.fetch_samples_matching({ "cms3tag": "the_tag_2", "nevents_out": 666 })) == 1) assert (len(db.fetch_samples_matching({"sample_type": "CMS3"})) == 3 ) # still only have 3 samples assert (len(db.read_to_dict_list("select * from sample")) == 3 ) # check that this method returns all samples assert (len(db.fetch_samples_matching({"cms3tag": "the_tag_4"})) == 0 ) # shouldn't match anything # if we change the cms3tag and try to update the sample, it's actually a new sample, so another row gets added sample = db.fetch_samples_matching({"cms3tag": "the_tag_2"})[0] sample["cms3tag"] = "the_tag_4" db.update_sample(sample) assert (len(db.fetch_samples_matching({"cms3tag": "the_tag_4"})) == 1 ) # should now match one sample assert (len(db.fetch_samples_matching({"sample_type": "CMS3"})) == 4 ) # and we have 4 rows now assert (db.fetch_samples_matching({"duck": 1}) == [] ) # unrecognized key returns nothing assert (db.fetch_samples_matching({}) == [] ) # empty selection dict returns nothing assert (db.update_sample({}) == False) # don't do anything if empty assert (db.update_sample({"duck": 1}) == False ) # don't do anything if unrecognized key assert (len(db.fetch_samples_matching({"cms3tag": "*_tag_*"})) == 3 ) # check wildcard support db.close() # print "Calculations correct" return True
def do_test(): sample = { "dataset_name": "/DYJetsToLL_M-10to50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIISpring15MiniAODv2-74X_mcRun2_asymptotic_v2-v1/MINIAODSIM", "twiki_name": "Run2SamplesReMINIAOD_25ns", "sample_type": "CMS3", "filter_type": "NoFilter", "nevents_in": 30301601, "nevents_out": 30301601, "xsec": 18610, "kfactor": 1.11, "filter_eff": 1, "gtag": "74X_mcRun2_asymptotic_v2", "cms3tag": "CMS3_V07-04-11", "location": "/hadoop/cms/store/group/snt/run2_25ns_MiniAODv2/DYJetsToLL_M-10to50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8_RunIISpring15MiniAODv2-74X_mcRun2_asymptotic_v2-v1/V07-04-11/", "assigned_to": "Rafael", "comments": "xsec from MCM" } # :memory: uses RAM instead of saving a file # db = DBInterface(fname=":memory:") db = DBInterface(fname=":memory:") # empty DB and re-initialize table db.drop_table() db.make_table() # make some more fake samples sample1 = sample.copy() sample2 = sample.copy() sample2["cms3tag"] = "the_tag_2" sample3 = sample.copy() sample3["cms3tag"] = "the_tag_3" sample3["nevents_out"] = 12345 # insert samples db.update_sample(sample1) db.update_sample(sample2) db.update_sample(sample3) # next line shouldn't add a new row since there's duplicate checking db.update_sample(sample3) assert( len(db.fetch_samples_matching({"sample_type":"CMS3"})) == 3 ) # should match all three samples assert( len(db.fetch_samples_matching({"cms3tag":"the_tag_2"})) == 1 ) # should match only sample2 assert( len(db.fetch_samples_matching({"nevents_out":12345})) == 1 ) # should match only sample3 assert( len(db.fetch_samples_matching({"cms3tag":"the_tag_3","nevents_out":12345})) == 1 ) # should match only sample3 # get sample2, change the evt count (leaving the rest), update it sample = db.fetch_samples_matching({"cms3tag":"the_tag_2"})[0] sample["nevents_out"] = 666 db.update_sample(sample) assert( len(db.fetch_samples_matching({"cms3tag":"the_tag_2","nevents_out":666})) == 1 ) assert( len(db.fetch_samples_matching({"sample_type":"CMS3"})) == 3 ) # still only have 3 samples assert( len(db.read_to_dict_list("select * from sample")) == 3 ) # check that this method returns all samples assert( len(db.fetch_samples_matching({"cms3tag":"the_tag_4"})) == 0 ) # shouldn't match anything # if we change the cms3tag and try to update the sample, it's actually a new sample, so another row gets added sample = db.fetch_samples_matching({"cms3tag":"the_tag_2"})[0] sample["cms3tag"] = "the_tag_4" db.update_sample(sample) assert( len(db.fetch_samples_matching({"cms3tag":"the_tag_4"})) == 1 ) # should now match one sample assert( len(db.fetch_samples_matching({"sample_type":"CMS3"})) == 4 ) # and we have 4 rows now assert( db.fetch_samples_matching({"duck":1}) == [] ) # unrecognized key returns nothing assert( db.fetch_samples_matching({}) == [] ) # empty selection dict returns nothing assert( db.update_sample({}) == False ) # don't do anything if empty assert( db.update_sample({"duck":1}) == False ) # don't do anything if unrecognized key assert( len(db.fetch_samples_matching({"cms3tag":"*_tag_*"})) == 3 ) # check wildcard support db.close() # print "Calculations correct" return True
def handle_query(arg_dict): if not arg_dict: return query_type = arg_dict.get("type", "basic") query = arg_dict.get("query", None).strip() short = arg_dict.get("short", False) # parse extra information in query if it's not just the dataset # /Gjet*/*/*, cms3tag=*07*06* | grep location,dataset_name # ^^dataset^^ ^^^selectors^^^ ^^^^^^^^^^^pipes^^^^^^^^^^ selectors = [] pipes = [] if "|" in query: first = query.split("|")[0].strip() pipes = query.split("|")[1:] if "," in first: entity = first.split(",")[0].strip() selectors = first.split(",")[1:] else: entity = first elif "," in query: entity = query.split(",")[0].strip() selectors = query.split(",")[1:] else: entity = query.strip() failed = False fail_reason = "" warning = "" payload = {} if "*" in entity and query_type in ["basic", "files"]: query_type = "listdatasets" # # HUGE SLOWDONW (4 seconds :( ) # if query_type in ["basic", "files", "listdatasets", "mcm", "driver", "lhe", "parents", "dbs"]: # if proxy_hours_left() < 5: proxy_renew() if not entity: failed = True fail_reason = "Dataset not specified" try: if query_type == "snt": payload = get_snt_samples(entity, selectors, short) elif query_type == "basic": info = dataset_event_count(entity) if not info: failed = True fail_reason = "Dataset not found" payload = info elif query_type == "listdatasets": datasets = list_of_datasets(entity, short, selectors=selectors) if not datasets: failed = True fail_reason = "No datasets found" payload = datasets elif query_type == "sites": if "/store/" in entity: payload = get_file_replicas(entity) else: if short: datasets = list_of_datasets(entity, short=True) if not datasets: failed = True fail_reason = "No datasets found" payload = get_replica_fractions(datasets) else: payload = get_file_replicas(entity, typ="dataset") elif query_type == "files": files = get_dataset_files(entity, selectors=selectors) payload = filelist_to_dict(files, short, num=10) elif query_type == "runs": payload = get_dataset_runs(entity) elif query_type == "pick": payload, warning = get_pick_events(entity, selectors) elif query_type == "pick_cms4": payload = get_pick_cms4(entity, selectors) elif query_type == "config": config_info = get_dataset_config(entity) payload = config_info elif query_type == "mcm": if selectors: if "this" == selectors[0].lower(): gen_sim = entity else: gen_sim = get_specified_parent(entity, typ="GEN-SIM", fallback="AODSIM") if short: info = get_slim_mcm_json(gen_sim) else: info = get_mcm_json(gen_sim)["results"] info["sample"] = gen_sim payload = info elif query_type == "parents": lineage = [] dataset = entity while dataset: try: dataset = get_dataset_parent(dataset) except: break if dataset: lineage.append(dataset) info = {} info["parents"] = lineage payload = info elif query_type == "driver": if selectors: if "this" == selectors[0].lower(): gen_sim = entity else: gen_sim = get_specified_parent(entity, typ="GEN-SIM", fallback="AODSIM") info = get_mcm_json(gen_sim)["results"] dataset_base = info["dataset_name"] campaign = info["prepid"] driver = get_mcm_setup(campaign) payload = {"dataset": dataset_base, "cmsDriver": driver} elif query_type == "lhe": lhe = get_specified_parent(entity, typ="LHE") # lhe = get_dataset_parent(gen_sim) files = get_dataset_files(lhe) payload["files"] = filelist_to_dict(files, short) elif query_type == "update_snt": from db import DBInterface db = DBInterface(fname="allsamples.db") s = {} for keyval in query.split(","): key, val = keyval.strip().split("=") s[key] = val payload = s did_update = db.update_sample(s) db.close() payload["updated"] = did_update elif query_type == "delete_snt": from db import DBInterface db = DBInterface(fname="allsamples.db") s = {} for keyval in query.split(","): key, val = keyval.strip().split("=") s[key] = val payload = s did_delete = db.delete_sample(s) db.close() payload["deleted"] = did_delete elif query_type == "dbs": if "raw" in selectors: import base64 payload = get_url_with_cert(query.rsplit(",", 1)[0], return_raw=True) payload = base64.b64encode(payload) else: payload = get_url_with_cert(query) # payload = "/DYJetsToLL_M-50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIISpring16MiniAODv1-PUSpring16_80X_mcRun2_asymptotic_2016_v3-v1/MINIAODSIM" # payload = "/DYJetsToLL_M-50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIISpring16MiniAODv1-PUSpring16_80X_mcRun2_asymptotic_2016_v3-v1/MINIAODSIM" else: failed = True fail_reason = "Query type not found" except: failed = True fail_reason = traceback.format_exc() if pipes: for pipe in pipes: pipe = pipe.strip() parts = pipe.split(" ", 1) if len(parts) == 1: verb = parts[0].strip() elif len(parts) == 2: verb, keys = parts keys = map(lambda x: x.strip(), keys.split(",")) else: verb = parts[0] if verb == "grep": if type(payload) == list: for ipay in range(len(payload)): if type(payload[ipay]) is not dict: continue if len(keys) > 1: d = {} for key in keys: if key in payload[ipay]: d[key] = payload[ipay][key] else: d[key] = get_derived_quantity( payload[ipay], key) payload[ipay] = d else: payload[ipay] = payload[ipay].get(keys[0], None) elif type(payload) == dict: if len(keys) > 0: d = {} for key in keys: d[key] = payload.get(key, None) payload = d elif verb == "stats": if type(payload) == list: nums = [] for elem in payload: try: nums.append(float(elem)) except: pass if len(nums) > 0: payload = { "N": len(nums), "total": sum(nums), "minimum": min(nums), "maximum": max(nums), } else: payload = {"N": len(payload)} elif verb == "sort": if type(payload) == list: payload = sorted(payload) return make_response(arg_dict, payload, failed, fail_reason, warning)
def handle_query(arg_dict): query_type = arg_dict.get("type","basic") query = arg_dict.get("query", None).strip() short = arg_dict.get("short", False) # parse extra information in query if it's not just the dataset # /Gjet*/*/*, cms3tag=*07*06* | grep location,dataset_name # ^^dataset^^ ^^^selectors^^^ ^^^^^^^^^^^pipes^^^^^^^^^^ selectors = None pipes = None if "|" in query: first = query.split("|")[0].strip() pipes = query.split("|")[1:] if "," in first: entity = first.split(",")[0].strip() selectors = first.split(",")[1:] else: entity = first elif "," in query: entity = query.split(",")[0].strip() selectors = query.split(",")[1:] else: entity = query.strip() failed = False fail_reason = "" warning = "" payload = {} if "*" in entity and query_type in ["basic","files"]: query_type = "listdatasets" if query_type in ["basic", "files", "listdatasets", "mcm", "driver", "lhe", "parents", "dbs"]: if proxy_hours_left() < 5: proxy_renew() if not entity: failed = True fail_reason = "Dataset not specified" try: if query_type == "basic": info = dataset_event_count(entity) if not info: failed = True fail_reason = "Dataset not found" payload = info elif query_type == "listdatasets": datasets = list_of_datasets(entity, short) if not datasets: failed = True fail_reason = "No datasets found" payload = datasets elif query_type == "files": files = get_dataset_files(entity) payload = filelist_to_dict(files, short) elif query_type == "runs": payload = get_dataset_runs(entity) elif query_type == "pick": payload, warning = get_pick_events(entity, selectors) elif query_type == "pick_cms3": payload, warning = get_pick_cms3(entity, selectors) elif query_type == "config": config_info = get_dataset_config(entity) payload = config_info elif query_type == "mcm": if selectors: if "this" == selectors[0].lower(): gen_sim = entity else: gen_sim = get_specified_parent(entity, typ="GEN-SIM", fallback="AODSIM") if short: info = get_slim_mcm_json(gen_sim) else: info = get_mcm_json(gen_sim)["results"] info["sample"] = gen_sim payload = info elif query_type == "parents": lineage = [] dataset = entity while dataset: try: dataset = get_dataset_parent(dataset) except: break if dataset: lineage.append(dataset) info = {} info["parents"] = lineage payload = info elif query_type == "driver": if selectors: if "this" == selectors[0].lower(): gen_sim = entity else: gen_sim = get_specified_parent(entity, typ="GEN-SIM", fallback="AODSIM") info = get_mcm_json(gen_sim)["results"] dataset_base = info["dataset_name"] campaign = info["prepid"] driver = get_mcm_setup(campaign) payload = { "dataset": dataset_base, "cmsDriver": driver } elif query_type == "lhe": lhe = get_specified_parent(entity, typ="LHE") # lhe = get_dataset_parent(gen_sim) files = get_dataset_files(lhe) payload["files"] = filelist_to_dict(files, short) elif query_type == "snt": from db import DBInterface db = DBInterface(fname="allsamples.db") match_dict = {"dataset_name": entity} if selectors: for more in selectors: key = more.split("=")[0].strip() val = more.split("=")[1].strip() match_dict[key] = val samples = db.fetch_samples_matching(match_dict) if short: new_samples = [] for sample in samples: for key in ["sample_id","filter_type","assigned_to", \ "comments","twiki_name","sample_type"]: del sample[key] new_samples.append(sample) samples = new_samples payload = samples elif query_type == "update_snt": from db import DBInterface db = DBInterface(fname="allsamples.db") s = {} for keyval in query.split(","): key,val = keyval.strip().split("=") s[key] = val payload = s did_update = db.update_sample(s) db.close() payload["updated"] = did_update elif query_type == "dbs": payload = get_url_with_cert(query) # payload = "/DYJetsToLL_M-50_TuneCUETP8M1_13TeV-amcatnloFXFX-pythia8/RunIISpring16MiniAODv1-PUSpring16_80X_mcRun2_asymptotic_2016_v3-v1/MINIAODSIM" else: failed = True fail_reason = "Query type not found" except: failed = True fail_reason = traceback.format_exc() if pipes: for pipe in pipes: pipe = pipe.strip() parts = pipe.split(" ",1) if len(parts) == 1: verb = parts[0].strip() elif len(parts) == 2: verb, keys = parts keys = map(lambda x: x.strip(), keys.split(",")) else: verb = parts[0] if verb == "grep": if type(payload) == list: for ipay in range(len(payload)): if type(payload[ipay]) is not dict: continue if len(keys) > 1: d = {} for key in keys: d[key] = payload[ipay].get(key,None) payload[ipay] = d else: payload[ipay] = payload[ipay].get(keys[0],None) elif type(payload) == dict: if len(keys) > 0: d = {} for key in keys: d[key] = payload.get(key,None) payload = d elif verb == "stats": if type(payload) == list: nums = [] for elem in payload: try: nums.append(float(elem)) except: pass if len(nums) > 0: payload = { "N": len(nums), "total": sum(nums), "minimum": min(nums), "maximum": max(nums), } else: payload = {"N": len(payload)} return make_response(arg_dict, payload, failed, fail_reason, warning)
"Run2Samples25ns80X", "Run2Samples25ns80XminiAODv2", "Run2Samples25ns80XPrivate", "Run2SamplesPrivateSMSFastSim25ns", "Run2Samples25ns76XminiAODv2", "Run2SamplesSMSFastSim_25ns", "Run2SamplesReMINIAOD_25ns", "Run2Samples_25ns", "Run2Samples25ns80XFS", "testTwiki", # used for testing various things # "SMS_T5ttttDM175_74X", # "SMS_T5qqqqWW_74X", # "SMS_T5ttcc_74X", ] db = DBInterface(fname="allsamples.db") db.drop_table() db.make_table() pool = ThreadPool(4) site_samples = pool.map(get_samples, sites) pool.close() pool.join() isample = 0 for site,samples in zip(sites,site_samples): for sample in samples: s = {} if "dataset" not in sample: continue