Esempio n. 1
0
def merge_batch(args):

	from gcSettings import Settings
	cfg = Settings()
	cfg.workflow.task = 'UserTask'
	cfg.workflow.backend = 'local'
	cfg.workflow.duration = '-1'

	cfg.jobs.wall_time = '3:00:00'
	cfg.jobs.memory = "6000"
	cfg.jobs.max_retry = 1

	cfg.usertask.executable = 'Artus/Utility/scripts/artus_userjob_epilog.sh'
	cmssw_base = os.getenv("CMSSW_BASE") + "/src/"
	executable = 'artusMergeOutputs.py '
	cfg.usertask.input_files= [cmssw_base + "Artus/Configuration/scripts/artusMergeOutputs.py"] 

	project_dirs = "-i " + " ".join(args.project_dir)
	outputs_per_nick = folders_to_merge(args)
	# extract nicks that should be ran on
	cfg.parameters.parameters = ["NICK"]
	nicks_to_process = outputs_per_nick.keys() if(args.project_subdir == None) else [args.project_subdir] # keep only single path
	input_dirs  = []
	for project_dir in args.project_dir:
		for nick in nicks_to_process:
			input_dir = os.path.join(project_dir, nick)
			if os.path.exists(input_dir):
				input_dirs.append(input_dir)
			input_dir = os.path.join(project_dir, "output", nick)
			if os.path.exists(input_dir):
				input_dirs.append(input_dir)
	
	required_scratch_space = max(map(get_folder_size, input_dirs)) * 2 + 100 * 1024 * 1024
	cfg.backend.submit_options = "-l h_fsize=" + str(required_scratch_space / 1024 / 1024 / 1024)+"G"
	cfg.parameters.NICK = nicks_to_process 
	cfg.jobs.jobs = len(nicks_to_process)

	arguments = cmssw_base
	arguments = arguments + " " + executable
	arguments = arguments + " " .join(args.project_dir)
	arguments = arguments + " --project-subdir @NICK@ "
	if(args.output_dir != None):
		arguments = arguments + " --output-dir " + args.output_dir
		
	cfg.usertask.arguments = "%s"%arguments
	merged_directory = os.path.join(args.project_dir[0] if(args.output_dir == None) else args.output_dir, "merged")
	cfg.storage.se_path = merged_directory 
	cfg.storage.scratch_space_used = required_scratch_space / 1024 / 1024
	cfg.storage.se_output_files = "merged.root"
	cfg.storage.se_output_pattern = "@NICK@/@[email protected]"
	cfg.GLOBAL.workdir = os.path.join(args.project_dir[0] if(args.output_dir == None) else args.output_dir, "workdir_merge")
	from grid_control.utils.activity import Activity
	Activity.root = Activity('Running grid-control', name = 'root')
	from gcTool import gc_create_workflow, gc_create_config
	config = gc_create_config( configDict = Settings.getConfigDict())

	workflow = gc_create_workflow(config)
	#activate for large verbosity
	#logging.getLogger('process').setLevel(logging.DEBUG1) 
	workflow.run()
Esempio n. 2
0
def merge_batch(args):

	from gcSettings import Settings
	cfg = Settings()
	cfg.workflow.task = 'UserTask'
	cfg.workflow.backend = 'local'
	cfg.workflow.duration = '-1'

	cfg.jobs.wall_time = '3:00:00'
	cfg.jobs.memory = "6000"
	cfg.jobs.max_retry = 1

	cfg.usertask.executable = 'Artus/Utility/scripts/artus_userjob_epilog.sh'
	cmssw_base = os.getenv("CMSSW_BASE") + "/src/"
	executable = 'artusMergeOutputs.py '
	cfg.usertask.input_files= [cmssw_base + "Artus/Configuration/scripts/artusMergeOutputs.py"] 

	project_dirs = "-i " + " ".join(args.project_dir)
	outputs_per_nick = folders_to_merge(args)
	# extract nicks that should be ran on
	cfg.parameters.parameters = ["NICK"]
	nicks_to_process = outputs_per_nick.keys() if(args.project_subdir == None) else [args.project_subdir] # keep only single path
	input_dirs  = []
	for project_dir in args.project_dir:
		for nick in nicks_to_process:
			input_dir = os.path.join(project_dir, nick)
			if os.path.exists(input_dir):
				input_dirs.append(input_dir)
			input_dir = os.path.join(project_dir, "output", nick)
			if os.path.exists(input_dir):
				input_dirs.append(input_dir)
	
	required_scratch_space = max(map(get_folder_size, input_dirs)) * 2 + 100 * 1024 * 1024
	cfg.backend.submit_options = "-l h_fsize=" + str(required_scratch_space / 1024 / 1024 / 1024)+"G"
	cfg.parameters.NICK = nicks_to_process 
	cfg.jobs.jobs = len(nicks_to_process)

	arguments = cmssw_base
	arguments = arguments + " " + executable
	arguments = arguments + " " .join(args.project_dir)
	arguments = arguments + " --project-subdir @NICK@ "
	if(args.output_dir != None):
		arguments = arguments + " --output-dir " + args.output_dir
		
	cfg.usertask.arguments = "%s"%arguments
	merged_directory = os.path.join(args.project_dir[0] if(args.output_dir == None) else args.output_dir, "merged")
	cfg.storage.se_path = merged_directory 
	cfg.storage.scratch_space_used = required_scratch_space / 1024 / 1024
	cfg.storage.se_output_files = "merged.root"
	cfg.storage.se_output_pattern = "@NICK@/@[email protected]"
	cfg.GLOBAL.workdir = os.path.join(args.project_dir[0] if(args.output_dir == None) else args.output_dir, "workdir_merge")
	from grid_control.utils.activity import Activity
	Activity.root = Activity('Running grid-control', name = 'root')
	from gcTool import gc_create_workflow, gc_create_config
	config = gc_create_config( configDict = Settings.getConfigDict())

	workflow = gc_create_workflow(config)
	#activate for large verbosity
	#logging.getLogger('process').setLevel(logging.DEBUG1) 
	workflow.run()
Esempio n. 3
0
        'This file is supposed to be run directly by python - not by go.py!')
try:
    from gcTool import gc_create_config, gc_create_workflow
except ImportError:
    raise Exception(
        'grid-control is not correctly installed or the gc package directory is not part of the PYTHONPATH.'
    )

# Setup workflow
config = gc_create_config(
    configDict={
        'interactive': {
            'default': False
        },
        'global': {
            'backend': 'Host',
            'task': 'UserTask'
        },
        'task': {
            'executable': 'Example02_local.sh'
        },
        'jobs': {
            'wall time': '1:00',
            'jobs': 2
        },
    })

# Create and run workflow with GUI output
workflow = gc_create_workflow(config)
workflow.run()
Esempio n. 4
0
#!/usr/bin/env python

import logging

# Throw exceptions if the file is executed in the wrong way or grid-control is not correctly installed
if 'Settings' in locals():
	raise Exception('This file is supposed to be run directly by python - not by go.py!')
try:
	from gcSettings import Settings
	from gcTool import gc_create_config, gc_create_workflow
except ImportError:
	raise Exception('grid-control is not correctly installed or the gc package directory is not part of the PYTHONPATH.')

setup = Settings()
setup.Global.report = 'null'
setup.Global.backend = 'Host'
setup.Global.task = 'UserTask'
setup.Global.duration = -1
setup.jobs.jobs = 1
setup.jobs.wall_time = 1
setup.task.executable = 'Example02_local.sh'
setup.interactive.default = False

config = gc_create_config(configDict = setup.getConfigDict())
logging.getLogger().setLevel(logging.CRITICAL)
logging.getLogger('jobs').setLevel(logging.INFO)
workflow = gc_create_workflow(config)
workflow.run()