def config(package, **kwargs):
"""Show configuration."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
config = kwargs['config']
exit(ArtBase(art_directory).config(package, nightly_release, project, platform, config))
def output(package, test_name, **kwargs):
"""Get the output of a job."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
user = kwargs['user']
exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).output(package, test_name, user))
def run(script_directory, sequence_tag, **kwargs):
"""Run jobs from a package in a local build, needs release and grid setup."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
job_type = 'build' if kwargs['type'] is None else kwargs['type']
exit(ArtBuild(art_directory, nightly_release, project, platform, nightly_tag, script_directory, max_jobs=int(kwargs['max_jobs']), ci=kwargs['ci']).task_list(job_type, sequence_tag))
def included(script_directory, **kwargs):
"""Show list of files which will be included for art submit/art grid."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
job_type = kwargs['type'] # None will list all types
index_type = kwargs['test_type']
exit(ArtBase(art_directory).included(script_directory, job_type, index_type, nightly_release, project, platform))
def list(package, **kwargs):
"""List the jobs of a package."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
job_type = 'grid'
index_type = kwargs['test_type']
json_format = kwargs['json']
user = kwargs['user']
exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).list(package, job_type, index_type, json_format, user))
def compare_grid(package, test_name, **kwargs):
"""Compare the output of a job."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
days = int(kwargs['days'])
user = kwargs['user']
files = kwargs['file']
entries = kwargs['entries']
mode = kwargs['mode']
exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).compare(package, test_name, days, user, files, entries=entries, mode=mode, shell=True))
def grid(script_directory, sequence_tag, **kwargs):
"""Run jobs from a package on the grid, needs release and grid setup."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
job_type = 'grid' if kwargs['type'] is None else kwargs['type']
inform_panda = False
package = None
config = None
no_action = kwargs['no_action']
wait_and_copy = False
exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, script_directory=script_directory, skip_setup=True, max_jobs=int(kwargs['max_jobs'])).task_list(job_type, sequence_tag, inform_panda, package, no_action, wait_and_copy, config))
def copy(indexed_package, **kwargs):
"""Copy outputs to eos area."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
# NOTE: default depends on USER, not set it here but in ArtGrid.copy
dst = kwargs['dst']
user = kwargs['user']
no_unpack = kwargs['no_unpack']
tmp = kwargs['tmp']
seq = int(kwargs['seq'])
keep_tmp = kwargs['keep_tmp']
exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).copy(indexed_package, dst=dst, user=user, no_unpack=no_unpack, tmp=tmp, seq=seq, keep_tmp=keep_tmp))
def submit(sequence_tag, **kwargs):
"""Submit nightly jobs to the grid, NOT for users."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
job_type = 'grid' if kwargs['type'] is None else kwargs['type']
user = os.getenv('USER', 'artprod')
inform_panda = user == 'artprod'
package = kwargs['package']
config = kwargs['config']
no_action = kwargs['no_action']
wait_and_copy = True
exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag, max_jobs=int(kwargs['max_jobs'])).task_list(job_type, sequence_tag, inform_panda, package, no_action, wait_and_copy, config))
def build_job(script_directory, sequence_tag, package, outfile, job_type,
job_index, **kwargs):
"""Build a single job, given a particular index.
Tests are called with the following parameters:
SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME
"""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
exit(
ArtBuild(art_directory, nightly_release, project, platform,
nightly_tag, script_directory).job(sequence_tag, package,
outfile, job_type,
job_index))
def grid_batch(script_directory, sequence_tag, package, outfile, inform_panda,
job_type, job_index, **kwargs):
"""Run a batch job, given a particular index.
Tests are called with the following parameters:
SCRIPT_DIRECTORY, PACKAGE, TYPE, TEST_NAME, STAGE
"""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
skip_setup = kwargs['skip_setup']
exit(
ArtGrid(art_directory, nightly_release, project, platform, nightly_tag,
script_directory, skip_setup).batch(sequence_tag, package,
outfile, inform_panda,
job_type, job_index))
def __init__(self, arguments):
"""Clean when more than 'days' old."""
log = logging.getLogger(MODULE)
self.verbose = arguments['--verbose']
self.eos = arguments['--eos']
default_days = int(arguments['--days'])
self.delete = arguments['--delete']
nightly_release = arguments['<nightly_release>']
project = arguments['<project>']
platform = arguments['<platform>']
base_dir = arguments['--base-dir']
# Check for missing arguments
if nightly_release is None:
(nightly_release, project, platform, dummy) = get_atlas_env()
if platform is None:
platform = 'x86_64-slc6-gcc62-opt'
log.info("Defaulting to platform %s", platform)
if self.eos:
# Test if we have access to kerberos
(code, out, err, command, start_time, end_time) = run_command('klist', verbose=self.verbose)
if code != 0:
log.critical("%s", err)
exit(1)
package = arguments['--package']
if package is not None:
self.clean_release(os.path.join(base_dir, nightly_release, project, platform), default_days, package)
return
if arguments['--release']:
self.clean_release(os.path.join(base_dir, nightly_release, project, platform), default_days)
return
config_file = arguments['--config']
config = ArtConfiguration(config_file)
for package in config.packages():
copy = config.get(nightly_release, project, platform, package, 'copy', False)
if copy:
days = config.get(nightly_release, project, platform, package, 'days', default_days)
self.clean_release(os.path.join(base_dir, nightly_release, project, platform), days, package)
return
def createpoolfile(package, **kwargs):
"""Show configuration."""
set_log(kwargs)
art_directory = os.path.dirname(os.path.realpath(sys.argv[0]))
(nightly_release, project, platform, nightly_tag) = get_atlas_env()
exit(ArtGrid(art_directory, nightly_release, project, platform, nightly_tag).createpoolfile())
