from striped.job import SinglePointStripedSession as Session
import numpy as np
import fitsio
job_server_address = ("dbwebdev.fnal.gov", 8765) #development
#job_server_address = ("ifdb01.fnal.gov", 8765) #production
session = Session(job_server_address)
sample = fitsio.read("/data/ivm/DES/sample.fits")
np.sort(sample, order="HPIX")
sample_data = np.array(
zip(sample["RA"], sample["DEC"],
np.asarray(sample["HPIX"], dtype=np.float64)))
def callback(type, nevents, data):
if type == "update_streams":
if "matches" in data:
matches = data["matches"].reshape((-1, 2))
nerrors = 0
for idx, oid in matches:
if oid != sample["COADD_OBJECT_ID"][idx]:
print "error"
nerrors += 1
print nerrors, "errors total"
job = session.createJob("Y3A2",
user_callback=callback,
worker_class_file="match_worker.py",
from striped.common import Tracer
T = Tracer()
with T["run"]:
with T["imports"]:
from striped.job import SinglePointStripedSession as Session
import numpy as np
from numpy.lib.recfunctions import append_fields
import fitsio, healpy as hp
import sys, time
#job_server_address = ("dbwebdev.fnal.gov", 8765) #development
job_server_address = ("ifdb01.fnal.gov", 8765) #production
session = Session(job_server_address)
input_file = sys.argv[1]
input_filename = input_file.rsplit("/",1)[-1].rsplit(".",1)[-1]
with T["fits/read"]:
input_data = fitsio.read(input_file, ext=2, columns=["ALPHAWIN_J2000","DELTAWIN_J2000"])
with T["hpix"]:
hpix = hp.ang2pix(nside=16384,theta=input_data['ALPHAWIN_J2000'],phi=input_data['DELTAWIN_J2000'],
lonlat=True, nest=True)
hpix = np.asarray(hpix, np.float64)
input_data = append_fields(input_data, "HPIX", hpix)
np.sort(input_data, order="HPIX")
input_data = np.array(zip(input_data['ALPHAWIN_J2000'], input_data['DELTAWIN_J2000'], input_data['HPIX']))
def __init__(self, params, bulk, job, db):
self.NEvents = 0
self.NFrames = 0
def accumulate(self, nevents, data):
self.NEvents += data.get("NEvents", 0)
self.NFrames += data.get("NFrames", 0)
return {"events_processed":self.NEvents}
def accumulated(self, nevents):
return {"NEvents":self.NEvents, "NFrames":self.NFrames}
"""
session = Session("striped_dev.yaml")
bulk = np.random.random((10,10))
class callback:
def on_data(self, wid, n, data):
print n, data
dataset_name = "MNIST"
job = session.createJob(dataset_name,
worker_class_source = worker_class,
#bulk_data = {"x":bulk },
callbacks = [callback()],
user_params = {
"dataset":dataset_name
worker_class = """
import time, socket
class Worker(object):
Columns = ["A.p", "n"]
def run(self, events, job, database):
job.fill(n = events.n)
job.fill(p = events.A.p)
"""
dataset = "Sample"
session = Session("striped_dev.yaml")
h_n = Hist(hbin("n", 20, 0, 20))
h_p = Hist(hbin("p", 20, 0, 20))
job = session.createJob(dataset,
user_params={"param": {
"hello": "world"
}},
worker_class_source=worker_class,
histograms=[h_n, h_p])
job.run()
runtime = job.TFinish - job.TStart
nevents = job.EventsProcessed
print "%s: %.6fM events, %.6fM events/second" % (
dataset, float(nevents) / 1000000, nevents / runtime / 1000000)
T = Tracer()
Usage = """
python all_hpix.py
"""
opts, args = getopt.getopt(sys.argv[1:], "?h")
opts = dict(opts)
if "-?" in opts or "-h" in opts:
print Usage
sys.exit(1)
#job_server_address = ("dbwebdev.fnal.gov", 8765)
job_server_address = ("ifdb01.fnal.gov", 8765)
session = Session(job_server_address)
class Collector:
def __init__(self):
self.Collected = []
def on_streams_update(self, nevents, data):
if "message" in data:
for msg in data["message"]:
print msg
if "hpix" in data:
self.Collected.extend(data["hpix"])
def hpix(self):
hpix_set = set()
T = Tracer()
Usage = """
python verify.py
"""
opts, args = getopt.getopt(sys.argv[1:], "?h")
opts = dict(opts)
if "-?" in opts or "-h" in opts:
print Usage
sys.exit(1)
#job_server_address = ("dbwebdev.fnal.gov", 8765)
job_server_address = ("ifdb01.fnal.gov", 8765)
session = Session(job_server_address)
class Callback:
def on_streams_update(self, nevents, data):
if "message" in data:
for msg in data["message"]:
print msg
job = session.createJob("Bliss",
use_data_cache=False,
user_callback=Callback(),
worker_class_file="verify_worker.py")
job.run()
import numpy as np
import pandas as pd
worker_class = """
import cloudpickle
class Worker(object):
Columns = ["NJets"]
def run(self, events, job):
job.message("%d events" % (len(events),))
x = 5/0
"""
session = Session(("ifdb01.fnal.gov", 8765))
h_by_dataset = Hist(hbin("NJets", 20, 0, 20), groupby("dataset"))
datasets = [
"Summer16.TTHH_TuneCUETP8M2T4_13TeV-madgraph-pythia8" # 100000 events
]
class Callback:
def on_message(self, wid, nevents, message):
print "Message received from worker %d after seeing %d events: <%s>" % (wid, nevents, message)
def on_exception(self, wid, info):
print "Worker %d failed with exception:\n%s" % (wid, info)
callback = Callback()
import pandas as pd
worker_class = """
class Worker(object):
Columns = ["nJet","nMuon","nElectron","Jet.pt", "Muon.pt"]
def run(self, events, job):
job.fill(nJet=events.nJet)
job.fill(nElectron=events.nElectron)
job.fill(nMuon=events.nMuon)
job.fill(JetPt = events.Jet.pt)
job.fill(MuonPt = events.Muon.pt)
"""
session = Session()
njets = Hist(hbin("nJet", 20, 0, 20))
nmuon = Hist(hbin("nMuon", 20, 0, 20))
nelectron = Hist(hbin("nElectron", 20, 0, 20))
muon_pt = Hist(hbin("MuonPt", 70, 0., 700.))
jet_pt = Hist(hbin("JetPt", 70, 0., 700.))
dataset = "QCD_HT200to300_PUMoriond17_05Feb2018_94X_mcRun2_asymptotic_v2-v1_NANOAODSIM"
#dataset = "JetHT_Run2016H_05Feb2018_ver2-v1_NANOAOD"
class Callback:
def on_exception(self, wid, info):
print "Exception:", info
worker_class = """
import numpy as np
class Worker(object):
Columns = ["NJets"]
def run(self, events, job, db):
data = np.frombuffer(db["calib200"], "<f4")
job.fill(x = data)
job.message("average=%f" % (np.mean(data),))
"""
job_server = ("ifdb02.fnal.gov", 8765)
session = Session(job_server)
h = Hist(hbin("x", 20, 0, 1))
dataset = "Summer16.TTHH_TuneCUETP8M2T4_13TeV-madgraph-pythia8"
job = session.createJob(dataset,
worker_class_source=worker_class,
histograms=[h])
job.run()
runtime = job.TFinish - job.TStart
nevents = job.EventsProcessed
print "%s: %.6fM events, %.6fM events/second" % (
dataset, float(nevents) / 1000000, nevents / runtime / 1000000)
data_frame = h.pandas()