def merge_batch(args):
from gcSettings import Settings
cfg = Settings()
cfg.workflow.task = 'UserTask'
cfg.workflow.backend = 'local'
cfg.workflow.duration = '-1'
cfg.jobs.wall_time = '3:00:00'
cfg.jobs.memory = "6000"
cfg.jobs.max_retry = 1
cfg.usertask.executable = 'Artus/Utility/scripts/artus_userjob_epilog.sh'
cmssw_base = os.getenv("CMSSW_BASE") + "/src/"
executable = 'artusMergeOutputs.py '
cfg.usertask.input_files= [cmssw_base + "Artus/Configuration/scripts/artusMergeOutputs.py"]
project_dirs = "-i " + " ".join(args.project_dir)
outputs_per_nick = folders_to_merge(args)
# extract nicks that should be ran on
cfg.parameters.parameters = ["NICK"]
nicks_to_process = outputs_per_nick.keys() if(args.project_subdir == None) else [args.project_subdir] # keep only single path
input_dirs = []
for project_dir in args.project_dir:
for nick in nicks_to_process:
input_dir = os.path.join(project_dir, nick)
if os.path.exists(input_dir):
input_dirs.append(input_dir)
input_dir = os.path.join(project_dir, "output", nick)
if os.path.exists(input_dir):
input_dirs.append(input_dir)
required_scratch_space = max(map(get_folder_size, input_dirs)) * 2 + 100 * 1024 * 1024
cfg.backend.submit_options = "-l h_fsize=" + str(required_scratch_space / 1024 / 1024 / 1024)+"G"
cfg.parameters.NICK = nicks_to_process
cfg.jobs.jobs = len(nicks_to_process)
arguments = cmssw_base
arguments = arguments + " " + executable
arguments = arguments + " " .join(args.project_dir)
arguments = arguments + " --project-subdir @NICK@ "
if(args.output_dir != None):
arguments = arguments + " --output-dir " + args.output_dir
cfg.usertask.arguments = "%s"%arguments
merged_directory = os.path.join(args.project_dir[0] if(args.output_dir == None) else args.output_dir, "merged")
cfg.storage.se_path = merged_directory
cfg.storage.scratch_space_used = required_scratch_space / 1024 / 1024
cfg.storage.se_output_files = "merged.root"
cfg.storage.se_output_pattern = "@NICK@/@[email protected]"
cfg.GLOBAL.workdir = os.path.join(args.project_dir[0] if(args.output_dir == None) else args.output_dir, "workdir_merge")
from grid_control.utils.activity import Activity
Activity.root = Activity('Running grid-control', name = 'root')
from gcTool import gc_create_workflow, gc_create_config
config = gc_create_config( configDict = Settings.getConfigDict())
workflow = gc_create_workflow(config)
#activate for large verbosity
#logging.getLogger('process').setLevel(logging.DEBUG1)
workflow.run()
def merge_batch(args):
from gcSettings import Settings
cfg = Settings()
cfg.workflow.task = 'UserTask'
cfg.workflow.backend = 'local'
cfg.workflow.duration = '-1'
cfg.jobs.wall_time = '3:00:00'
cfg.jobs.memory = "6000"
cfg.jobs.max_retry = 1
cfg.usertask.executable = 'Artus/Utility/scripts/artus_userjob_epilog.sh'
cmssw_base = os.getenv("CMSSW_BASE") + "/src/"
executable = 'artusMergeOutputs.py '
cfg.usertask.input_files= [cmssw_base + "Artus/Configuration/scripts/artusMergeOutputs.py"]
project_dirs = "-i " + " ".join(args.project_dir)
outputs_per_nick = folders_to_merge(args)
# extract nicks that should be ran on
cfg.parameters.parameters = ["NICK"]
nicks_to_process = outputs_per_nick.keys() if(args.project_subdir == None) else [args.project_subdir] # keep only single path
input_dirs = []
for project_dir in args.project_dir:
for nick in nicks_to_process:
input_dir = os.path.join(project_dir, nick)
if os.path.exists(input_dir):
input_dirs.append(input_dir)
input_dir = os.path.join(project_dir, "output", nick)
if os.path.exists(input_dir):
input_dirs.append(input_dir)
required_scratch_space = max(map(get_folder_size, input_dirs)) * 2 + 100 * 1024 * 1024
cfg.backend.submit_options = "-l h_fsize=" + str(required_scratch_space / 1024 / 1024 / 1024)+"G"
cfg.parameters.NICK = nicks_to_process
cfg.jobs.jobs = len(nicks_to_process)
arguments = cmssw_base
arguments = arguments + " " + executable
arguments = arguments + " " .join(args.project_dir)
arguments = arguments + " --project-subdir @NICK@ "
if(args.output_dir != None):
arguments = arguments + " --output-dir " + args.output_dir
cfg.usertask.arguments = "%s"%arguments
merged_directory = os.path.join(args.project_dir[0] if(args.output_dir == None) else args.output_dir, "merged")
cfg.storage.se_path = merged_directory
cfg.storage.scratch_space_used = required_scratch_space / 1024 / 1024
cfg.storage.se_output_files = "merged.root"
cfg.storage.se_output_pattern = "@NICK@/@[email protected]"
cfg.GLOBAL.workdir = os.path.join(args.project_dir[0] if(args.output_dir == None) else args.output_dir, "workdir_merge")
from grid_control.utils.activity import Activity
Activity.root = Activity('Running grid-control', name = 'root')
from gcTool import gc_create_workflow, gc_create_config
config = gc_create_config( configDict = Settings.getConfigDict())
workflow = gc_create_workflow(config)
#activate for large verbosity
#logging.getLogger('process').setLevel(logging.DEBUG1)
workflow.run()
'This file is supposed to be run directly by python - not by go.py!')
try:
from gcTool import gc_create_config, gc_create_workflow
except ImportError:
raise Exception(
'grid-control is not correctly installed or the gc package directory is not part of the PYTHONPATH.'
)
# Setup workflow
config = gc_create_config(
configDict={
'interactive': {
'default': False
},
'global': {
'backend': 'Host',
'task': 'UserTask'
},
'task': {
'executable': 'Example02_local.sh'
},
'jobs': {
'wall time': '1:00',
'jobs': 2
},
})
# Create and run workflow with GUI output
workflow = gc_create_workflow(config)
workflow.run()
#!/usr/bin/env python
import logging
# Throw exceptions if the file is executed in the wrong way or grid-control is not correctly installed
if 'Settings' in locals():
raise Exception('This file is supposed to be run directly by python - not by go.py!')
try:
from gcSettings import Settings
from gcTool import gc_create_config, gc_create_workflow
except ImportError:
raise Exception('grid-control is not correctly installed or the gc package directory is not part of the PYTHONPATH.')
setup = Settings()
setup.Global.report = 'null'
setup.Global.backend = 'Host'
setup.Global.task = 'UserTask'
setup.Global.duration = -1
setup.jobs.jobs = 1
setup.jobs.wall_time = 1
setup.task.executable = 'Example02_local.sh'
setup.interactive.default = False
config = gc_create_config(configDict = setup.getConfigDict())
logging.getLogger().setLevel(logging.CRITICAL)
logging.getLogger('jobs').setLevel(logging.INFO)
workflow = gc_create_workflow(config)
workflow.run()