def sample(name, version=None, n_events = -1, format = ("ICF", 2), path="/vols/cms02/as1604/PhysicsUtils/SampleDB2/store", node = "IC_DCACHE" ): store = openStore(path) if not store.Exists(name+"/_sample_.json"): print "Sample '%s' not found" % name sys.exit(1) samp = store.Get(name+"/_sample_.json") vers = None if version is not None: if not store.Exists("%s/%s.json" % (name, version)): print "No version '%s' for sample '%s'" % (version,name) sys.exit(1) vers = store.Get("%s/%s.json" % (name, version)) else: if not store.Exists("%s/%s.json" % (name, samp.latest)): print "Oops the sample specified latest version '%s' cannot be found" % samp.latest vers = store.Get("%s/%s.json" % (name, sampl.latest)) ps = PSet(Format = format) if samp.cross_section == -1: ps._quiet_set("Weight", 1.0) else: ps._quiet_set("CrossSection", samp.cross_section) total_events = 0 files = [] for b in vers.blocks: if b["node"] != node: continue try: for f in b["files"]: if f["events"] == -1: continue files.append(str(se_path_to_url(f["path"], "dcap"))) total_events += f["events"] if n_events != -1 and total_events >= n_events: raise BreakMeOut except BreakMeOut: break ps._quiet_set("File", files) name = samp.name[1:].replace("/", "_") if version is not None: name += "_"+vers.name ps._quiet_set("Name", str(name)) ps._quiet_set("Events", total_events) return ps
def add_files(): filter_dbs = raw_input("Filter by DBS: ") sql = "select job.rowid, user, state, dset.dataset, tag.susycaf, dset.rowid, tag.rowid, job.rpath from job, dset, tag where dset.rowid = job.dsetid and tag.rowid = job.tagid" if filter_dbs: sql += " and dset.dataset LIKE ?" job = prompt_pager("Please choose a job", sql, "{rowid:<6} {user:<10} {state:<10} {dataset:<20} {susycaf:<10} {rpath:<30}", ["rowid", "rpath", "dataset"], filter_dbs+"%") else: job = prompt_pager("Please choose a job", sql, "{rowid:<6} {user:<10} {state:<10} {dataset:<20} {susycaf:<10} {rpath:<30}", ["rowid", "rpath", "dataset"]) ds_paths = [] if len(job[2].split(",")) == 1: ds_paths = [(0, job[2],job[1])] else: for i, d in enumerate(job[2].split(",")): ds_paths.append((i, d, job[1]+"/"+d[1:].replace("/", "."))) print "You have chosen job: %d" % job[0] print "{0:<40} {1:<40}".format("Dataset", "Path") for i, d, p in ds_paths: print "{0:<40} {1:<40}".format(d,p) print "Scan for files?" files = [] if prompt_yes(): for i, d, p in ds_paths: print "="*70 print "Scanning %s for files..." % p print "="*70 url = utils.se_path_to_url(p) flist = utils.se_lcg_ls(url) flist = ui_edit(flist) print "This subjob has %d files" % len(flist) print "Are you sure you want to add them to the DB?" if prompt_yes(): for f in flist: db.add_file(job[0], i, utils.se_path_to_node(f), f, -1) print "Done" else: print "Subjob %s not added" % d lines = [] for d, f in files: lines.extend(f) ui_edit(lines) print
store = openStore(conf["path"], lock=True) sample = prompt_sample(store) extend = prompt_version(store, sample) blocks = [] while True: j = prompt_job(conf["db_path"]) path = j[3] sub = 0 if len(j[1].split(",")) > 1: sub = prompt_subjob(conf["db_path"], j[1].split(",")) path = j[3] + '/' + string.replace(j[1][1:], '/', '.') print "Scan path '%s for files?" % path files = [] file_counts = {} if raw_input() == "y": files.extend(utils.se_lcg_ls(utils.se_path_to_url(path))) print "Files found: %d" % len(files) print "Scan for event counts?" if raw_input() == "y": import ROOT as r for idx, f in enumerate(files): print "*"*60 print "Scanning file (%d of %d): %s" % (idx, len(files), utils.se_path_to_url(f, "dcap")) f = r.TDCacheFile(utils.se_path_to_url(f, "dcap")) if f is None: continue d = f.Get("susyTree") if d is None: print "Directory not found"