def test_parsl_start_stop():
parsl = pytest.importorskip("parsl", minversion="0.7.2")
from coffea.processor.parsl.detail import (_parsl_initialize, _parsl_stop,
_default_cfg)
_parsl_initialize(config=_default_cfg)
_parsl_stop()
def test_parsl_executor():
parsl = pytest.importorskip("parsl", minversion="0.7.2")
from coffea.processor import run_parsl_job
from coffea.processor.parsl.detail import (_parsl_initialize, _parsl_stop)
from parsl.providers import LocalProvider
from parsl.channels import LocalChannel
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_hostname
from parsl.config import Config
parsl_config = Config(
executors=[
HighThroughputExecutor(
label="coffea_parsl_default",
address=address_by_hostname(),
cores_per_worker=max(multiprocessing.cpu_count() // 2, 1),
max_workers=1,
provider=LocalProvider(channel=LocalChannel(),
init_blocks=1,
max_blocks=1,
nodes_per_block=1),
)
],
strategy=None,
)
import os
import os.path as osp
filelist = {
'ZJets': [osp.join(os.getcwd(), 'tests/samples/nano_dy.root')],
'Data': [osp.join(os.getcwd(), 'tests/samples/nano_dimuon.root')]
}
treename = 'Events'
from coffea.processor.test_items import NanoTestProcessor
from coffea.processor.parsl.parsl_executor import parsl_executor
dfk = _parsl_initialize(parsl_config)
proc = NanoTestProcessor()
hists = run_parsl_job(filelist,
treename,
processor_instance=proc,
executor=parsl_executor,
data_flow=dfk)
_parsl_stop(dfk)
assert (hists['cutflow']['ZJets_pt'] == 4)
assert (hists['cutflow']['ZJets_mass'] == 1)
assert (hists['cutflow']['Data_pt'] == 15)
assert (hists['cutflow']['Data_mass'] == 5)
def do_parsl_job(parsl_config, filelist):
from coffea.processor.parsl.detail import (_parsl_initialize, _parsl_stop)
from coffea.processor import run_parsl_job
import os
import os.path as osp
treename = 'Events'
from coffea.processor.test_items import NanoTestProcessor
from coffea.processor.parsl.parsl_executor import parsl_executor
_parsl_initialize(parsl_config)
proc = NanoTestProcessor()
hists = run_parsl_job(filelist,
treename,
processor_instance=proc,
executor=parsl_executor)
hists2 = run_parsl_job(filelist,
treename,
processor_instance=proc,
executor=parsl_executor,
executor_args={'flatten': False})
_parsl_stop()
assert (hists['cutflow']['ZJets_pt'] == 4)
assert (hists['cutflow']['ZJets_mass'] == 1)
assert (hists['cutflow']['Data_pt'] == 15)
assert (hists['cutflow']['Data_mass'] == 5)
assert (hists2['cutflow']['ZJets_pt'] == 4)
assert (hists2['cutflow']['ZJets_mass'] == 1)
assert (hists2['cutflow']['Data_pt'] == 15)
assert (hists2['cutflow']['Data_mass'] == 5)
def do_parsl_job(parsl_config):
from coffea.processor.parsl.detail import (_parsl_initialize, _parsl_stop)
from coffea.processor import run_parsl_job
import os
import os.path as osp
filelist = {
'ZJets': [osp.join(os.getcwd(), 'tests/samples/nano_dy.root')],
'Data': [osp.join(os.getcwd(), 'tests/samples/nano_dimuon.root')]
}
treename = 'Events'
from coffea.processor.test_items import NanoTestProcessor
from coffea.processor.parsl.parsl_executor import parsl_executor
dfk = _parsl_initialize(parsl_config)
proc = NanoTestProcessor()
hists = run_parsl_job(filelist,
treename,
processor_instance=proc,
executor=parsl_executor,
data_flow=dfk)
hists2 = run_parsl_job(filelist,
treename,
processor_instance=proc,
executor=parsl_executor,
data_flow=dfk,
executor_args={'flatten': False})
_parsl_stop(dfk)
assert (hists['cutflow']['ZJets_pt'] == 4)
assert (hists['cutflow']['ZJets_mass'] == 1)
assert (hists['cutflow']['Data_pt'] == 15)
assert (hists['cutflow']['Data_mass'] == 5)
assert (hists2['cutflow']['ZJets_pt'] == 4)
assert (hists2['cutflow']['ZJets_mass'] == 1)
assert (hists2['cutflow']['Data_pt'] == 15)
assert (hists2['cutflow']['Data_mass'] == 5)
