import datetime
from lobster import cmssw
from lobster.core import AdvancedOptions, Category, Config, StorageConfiguration, Workflow
from lobster.core import ParentDataset, ProductionDataset
# version = datetime.datetime.now().strftime('%Y%m%d')
version = "v5"
storage = StorageConfiguration(output=[
"hdfs://eddie.crc.nd.edu:19000/store/user/khurtado/lobster_mc_" + version,
# "file:///hadoop/store/user/khurtado/lobster_mc_" + version,
# "root://deepthought.crc.nd.edu//store/user/khurtado/lobster_mc_" + version,
# "chirp://eddie.crc.nd.edu:9094/store/user/khurtado/lobster_test_" + version,
"gsiftp://T3_US_NotreDame/store/user/khurtado/lobster_mc_" + version,
# "srm://T3_US_NotreDame/store/user/khurtado/lobster_mc_" + version,
])
workflows = []
lhe = Workflow(label='lhe_step',
pset='mc_gen/HIG-RunIIWinter15wmLHE-00196_1_cfg.py',
sandbox=cmssw.Sandbox(release='mc_gen/CMSSW_7_1_16_patch1'),
merge_size='10M',
dataset=ProductionDataset(events_per_task=50,
events_per_lumi=5,
number_of_tasks=10),
category=Category(name='lhe', cores=1, memory=1000))
gs = Workflow(label='gs_step',
pset='mc_gen/HIG-RunIISummer15GS-00177_1_cfg.py',
import datetime
from lobster import cmssw
from lobster.core import AdvancedOptions, Category, Config, StorageConfiguration, Workflow
version = datetime.datetime.now().strftime('%Y%m%d_%H%M')
storage = StorageConfiguration(
output=[
"hdfs://eddie.crc.nd.edu:19000/store/user/$USER/lobster_test_" + version,
"file:///hadoop/store/user/$USER/lobster_test_" + version,
# ND is not in the XrootD redirector, thus hardcode server.
# Note the double-slash after the hostname!
"root://deepthought.crc.nd.edu//store/user/$USER/lobster_test_" + version,
"chirp://eddie.crc.nd.edu:9094/store/user/$USER/lobster_test_" + version,
"gsiftp://T3_US_NotreDame/store/user/$USER/lobster_test_" + version,
"srm://T3_US_NotreDame/store/user/$USER/lobster_test_" + version
]
)
processing = Category(
name='processing',
cores=1,
runtime=900,
memory=1000
)
workflows = []
ttH = Workflow(
label='ttH',
from lobster import cmssw
from lobster.core import AdvancedOptions, Category, Config, StorageConfiguration, Workflow
from lobster.core import ParentDataset, ProductionDataset
version = 'v7'
storage = StorageConfiguration(
output=[
"hdfs://eddie.crc.nd.edu:19000/store/user/matze/ttH/fastsim_" + version,
"file:///hadoop/store/user/matze/ttH/fastsim_" + version,
"root://deepthought.crc.nd.edu//store/user/matze/ttH/fastsim_" + version,
"gsiftp://T3_US_NotreDame/store/user/matze/ttH/fastsim_" + version,
# "chirp://eddie.crc.nd.edu:9094/store/user/matze/ttH/fastsim_" + version,
# "srm://T3_US_NotreDame/store/user/matze/ttH/fastsim_" + version,
]
)
datasets = ['ttH', 'ttjets_sl', 'ttjets_dl']
tasksizes = [200, 10000, 10000]
events = [20e6, 200e6, 100e6]
workflows = []
for dset, tasksize, events in zip(datasets, tasksizes, events):
tasks = int(events / tasksize)
lhe = Workflow(
label=dset + '_lhe',
pset='configs/' + dset + '_lhe.py',
merge_size='2000M',
output_path = "/store/user/$USER/postLHE_step/tests/{tag}/{ver}".format(tag=grp_tag,ver=out_ver)
workdir_path = "/tmpscratch/users/$USER/postLHE_step/tests/{tag}/{ver}".format(tag=grp_tag,ver=out_ver)
plotdir_path = "~/www/lobster/postLHE_step/tests/{tag}/{ver}".format(tag=grp_tag,ver=out_ver)
else:
print "Unknown run setup, {setup}".format(setup=RUN_SETUP)
raise ValueError
storage = StorageConfiguration(
input=[
"hdfs://eddie.crc.nd.edu:19000" + input_path,
"root://deepthought.crc.nd.edu/" + input_path, # Note the extra slash after the hostname!
"gsiftp://T3_US_NotreDame" + input_path,
"srm://T3_US_NotreDame" + input_path,
],
output=[
"hdfs://eddie.crc.nd.edu:19000" + output_path,
# ND is not in the XrootD redirector, thus hardcode server.
"root://deepthought.crc.nd.edu/" + output_path, # Note the extra slash after the hostname!
"gsiftp://T3_US_NotreDame" + output_path,
"srm://T3_US_NotreDame" + output_path,
"file:///hadoop" + output_path,
],
disable_input_streaming=False,
)
lhe_dirs = []
for path in input_dirs:
for fd in os.listdir(path):
if fd.find('lhe_step_') < 0:
continue
arr = fd.split('_')
master_label = '%s_%s' % (production_tag, timestamp_tag)
input_path = "/store/user/"
output_path = "/store/user/$USER/FullProduction/%s" % (production_tag)
workdir_path = "/tmpscratch/users/$USER/FullProduction/%s" % (production_tag)
plotdir_path = "~/www/lobster/FullProduction/%s" % (production_tag)
storage = StorageConfiguration(
input=[
"file:///hadoop" +
input_path, # Note the extra slash after the hostname!
"root://deepthought.crc.nd.edu/" +
input_path, # Note the extra slash after the hostname!
],
output=[
"hdfs://eddie.crc.nd.edu:19000" + output_path,
"root://deepthought.crc.nd.edu/" +
output_path, # Note the extra slash after the hostname!
"gsiftp://T3_US_NotreDame" + output_path,
"srm://T3_US_NotreDame" + output_path,
"file:///hadoop" + output_path,
],
disable_input_streaming=True,
)
#################################################################
# Worker Res.:
# Cores: 12 | 4
# Memory: 16000 | 8000
# Disk: 13000 | 6500
#################################################################
