import datetime from lobster import cmssw from lobster.core import AdvancedOptions, Category, Config, StorageConfiguration, Workflow from lobster.core import ParentDataset, ProductionDataset # version = datetime.datetime.now().strftime('%Y%m%d') version = "v5" storage = StorageConfiguration(output=[ "hdfs://eddie.crc.nd.edu:19000/store/user/khurtado/lobster_mc_" + version, # "file:///hadoop/store/user/khurtado/lobster_mc_" + version, # "root://deepthought.crc.nd.edu//store/user/khurtado/lobster_mc_" + version, # "chirp://eddie.crc.nd.edu:9094/store/user/khurtado/lobster_test_" + version, "gsiftp://T3_US_NotreDame/store/user/khurtado/lobster_mc_" + version, # "srm://T3_US_NotreDame/store/user/khurtado/lobster_mc_" + version, ]) workflows = [] lhe = Workflow(label='lhe_step', pset='mc_gen/HIG-RunIIWinter15wmLHE-00196_1_cfg.py', sandbox=cmssw.Sandbox(release='mc_gen/CMSSW_7_1_16_patch1'), merge_size='10M', dataset=ProductionDataset(events_per_task=50, events_per_lumi=5, number_of_tasks=10), category=Category(name='lhe', cores=1, memory=1000)) gs = Workflow(label='gs_step', pset='mc_gen/HIG-RunIISummer15GS-00177_1_cfg.py',
import datetime from lobster import cmssw from lobster.core import AdvancedOptions, Category, Config, StorageConfiguration, Workflow version = datetime.datetime.now().strftime('%Y%m%d_%H%M') storage = StorageConfiguration( output=[ "hdfs://eddie.crc.nd.edu:19000/store/user/$USER/lobster_test_" + version, "file:///hadoop/store/user/$USER/lobster_test_" + version, # ND is not in the XrootD redirector, thus hardcode server. # Note the double-slash after the hostname! "root://deepthought.crc.nd.edu//store/user/$USER/lobster_test_" + version, "chirp://eddie.crc.nd.edu:9094/store/user/$USER/lobster_test_" + version, "gsiftp://T3_US_NotreDame/store/user/$USER/lobster_test_" + version, "srm://T3_US_NotreDame/store/user/$USER/lobster_test_" + version ] ) processing = Category( name='processing', cores=1, runtime=900, memory=1000 ) workflows = [] ttH = Workflow( label='ttH',
from lobster import cmssw from lobster.core import AdvancedOptions, Category, Config, StorageConfiguration, Workflow from lobster.core import ParentDataset, ProductionDataset version = 'v7' storage = StorageConfiguration( output=[ "hdfs://eddie.crc.nd.edu:19000/store/user/matze/ttH/fastsim_" + version, "file:///hadoop/store/user/matze/ttH/fastsim_" + version, "root://deepthought.crc.nd.edu//store/user/matze/ttH/fastsim_" + version, "gsiftp://T3_US_NotreDame/store/user/matze/ttH/fastsim_" + version, # "chirp://eddie.crc.nd.edu:9094/store/user/matze/ttH/fastsim_" + version, # "srm://T3_US_NotreDame/store/user/matze/ttH/fastsim_" + version, ] ) datasets = ['ttH', 'ttjets_sl', 'ttjets_dl'] tasksizes = [200, 10000, 10000] events = [20e6, 200e6, 100e6] workflows = [] for dset, tasksize, events in zip(datasets, tasksizes, events): tasks = int(events / tasksize) lhe = Workflow( label=dset + '_lhe', pset='configs/' + dset + '_lhe.py', merge_size='2000M',
output_path = "/store/user/$USER/postLHE_step/tests/{tag}/{ver}".format(tag=grp_tag,ver=out_ver) workdir_path = "/tmpscratch/users/$USER/postLHE_step/tests/{tag}/{ver}".format(tag=grp_tag,ver=out_ver) plotdir_path = "~/www/lobster/postLHE_step/tests/{tag}/{ver}".format(tag=grp_tag,ver=out_ver) else: print "Unknown run setup, {setup}".format(setup=RUN_SETUP) raise ValueError storage = StorageConfiguration( input=[ "hdfs://eddie.crc.nd.edu:19000" + input_path, "root://deepthought.crc.nd.edu/" + input_path, # Note the extra slash after the hostname! "gsiftp://T3_US_NotreDame" + input_path, "srm://T3_US_NotreDame" + input_path, ], output=[ "hdfs://eddie.crc.nd.edu:19000" + output_path, # ND is not in the XrootD redirector, thus hardcode server. "root://deepthought.crc.nd.edu/" + output_path, # Note the extra slash after the hostname! "gsiftp://T3_US_NotreDame" + output_path, "srm://T3_US_NotreDame" + output_path, "file:///hadoop" + output_path, ], disable_input_streaming=False, ) lhe_dirs = [] for path in input_dirs: for fd in os.listdir(path): if fd.find('lhe_step_') < 0: continue arr = fd.split('_')
master_label = '%s_%s' % (production_tag, timestamp_tag) input_path = "/store/user/" output_path = "/store/user/$USER/FullProduction/%s" % (production_tag) workdir_path = "/tmpscratch/users/$USER/FullProduction/%s" % (production_tag) plotdir_path = "~/www/lobster/FullProduction/%s" % (production_tag) storage = StorageConfiguration( input=[ "file:///hadoop" + input_path, # Note the extra slash after the hostname! "root://deepthought.crc.nd.edu/" + input_path, # Note the extra slash after the hostname! ], output=[ "hdfs://eddie.crc.nd.edu:19000" + output_path, "root://deepthought.crc.nd.edu/" + output_path, # Note the extra slash after the hostname! "gsiftp://T3_US_NotreDame" + output_path, "srm://T3_US_NotreDame" + output_path, "file:///hadoop" + output_path, ], disable_input_streaming=True, ) ################################################################# # Worker Res.: # Cores: 12 | 4 # Memory: 16000 | 8000 # Disk: 13000 | 6500 #################################################################