def test_stageout_es_events_pw(self):
"""
Make sure that no exceptions to stage out file.
"""
error = None
try:
from pilot.info import infosys, InfoService
infoservice = InfoService()
infoservice.init('BNL_CLOUD_MCORE', infosys.confinfo,
infosys.extinfo)
output_file = os.path.join('/tmp', str(uuid.uuid4()))
shutil.copy('/bin/hostname', output_file)
file_data = {
'scope': 'transient',
'lfn': os.path.basename(output_file),
#'ddmendpoint': None,
#'type': 'es_events',
#'surl': output_file
#'turl': None,
#'filesize': None,
#'checksum': None
}
file_spec = FileSpec(filetype='output', **file_data)
xdata = [file_spec]
workdir = os.path.dirname(output_file)
client = StageOutESClient(infoservice)
kwargs = dict(workdir=workdir, cwd=workdir, usecontainer=False)
client.transfer(xdata, activity=['es_events', 'pw'], **kwargs)
except exception.PilotException, error:
logger.error("Pilot Exeception: %s, %s" %
(error.get_detail(), traceback.format_exc()))
def singularity_wrapper(cmd, workdir, job=None):
"""
Prepend the given command with the singularity execution command
E.g. cmd = /bin/bash hello_world.sh
-> singularity_command = singularity exec -B <bindmountsfromcatchall> <img> /bin/bash hello_world.sh
singularity exec -B <bindmountsfromcatchall> /cvmfs/atlas.cern.ch/repo/images/singularity/x86_64-slc6.img <script>
Note: if the job object is not set, then it is assumed that the middleware container is to be used.
:param cmd: command to be prepended (string).
:param workdir: explicit work directory where the command should be executed (needs to be set for Singularity) (string).
:param job: job object.
:return: prepended command with singularity execution command (string).
"""
if job:
queuedata = job.infosys.queuedata
else:
infoservice = InfoService()
infoservice.init(os.environ.get('PILOT_SITENAME'), infosys.confinfo, infosys.extinfo)
queuedata = infoservice.queuedata
container_name = queuedata.container_type.get("pilot") # resolve container name for user=pilot
logger.debug("resolved container_name from queuedata.contaner_type: %s" % container_name)
if container_name == 'singularity':
logger.info("singularity has been requested")
# Get the singularity options
singularity_options = queuedata.container_options
if singularity_options != "":
singularity_options += ","
else:
singularity_options = "-B "
singularity_options += "/cvmfs,${workdir},/home"
logger.debug("using singularity_options: %s" % singularity_options)
# Get the image path
if job:
image_path = job.imagename or get_grid_image_for_singularity(job.platform)
else:
image_path = config.Container.middleware_container
# Does the image exist?
if image_path:
# Prepend it to the given command
cmd = "export workdir=" + workdir + "; singularity --verbose exec " + singularity_options + " " + image_path + \
" /bin/bash -c " + pipes.quote("cd $workdir;pwd;%s" % cmd)
# for testing user containers
# singularity_options = "-B $PWD:/data --pwd / "
# singularity_cmd = "singularity exec " + singularity_options + image_path
# cmd = re.sub(r'-p "([A-Za-z0-9.%/]+)"', r'-p "%s\1"' % urllib.pathname2url(singularity_cmd), cmd)
else:
logger.warning("singularity options found but image does not exist")
logger.info("updated command: %s" % cmd)
return cmd
def test_stageout_es_events_non_exist_pw(self):
"""
Make sure that no exceptions to stage out file.
"""
error = None
try:
from pilot.info import infosys, InfoService
infoservice = InfoService()
infoservice.init('BNL_CLOUD_MCORE', infosys.confinfo,
infosys.extinfo)
output_file = os.path.join('/tmp', str(uuid.uuid4()))
shutil.copy('/bin/hostname', output_file)
file_data = {
'scope': 'transient',
'lfn': os.path.basename(output_file),
#'ddmendpoint': None,
#'type': 'es_events',
#'surl': output_file
#'turl': None,
#'filesize': None,
#'checksum': None
}
file_spec = FileSpec(filetype='output', **file_data)
xdata = [file_spec]
workdir = os.path.dirname(output_file)
client = StageOutESClient(infoservice)
kwargs = dict(workdir=workdir, cwd=workdir, usecontainer=False)
client.prepare_destinations(
xdata, activity=['es_events_non_exist', 'pw']
) # allow to write to `es_events_non_exist` and `pw` astorages
client.transfer(xdata,
activity=['es_events_non_exist', 'pw'],
**kwargs)
except exception.PilotException as error: # Python 2/3
logger.error("Pilot Exeception: %s, %s" %
(error.get_detail(), traceback.format_exc()))
except Exception as e: # Python 2/3
logger.error(traceback.format_exc())
error = exception.StageOutFailure("stageOut failed with error=%s" %
e)
logger.info('Summary of transferred files:')
for e in xdata:
logger.info(" -- lfn=%s, status_code=%s, status=%s" %
(e.lfn, e.status_code, e.status))
if error:
logger.error(
'Failed to stage-out eventservice file(%s): error=%s' %
(output_file, error.get_detail()))
raise error
# guids = file_list_dictionary.get('guids')
# generate the trace report
trace_report = TraceReport(pq=os.environ.get('PILOT_SITENAME', ''),
localSite=args.localsite,
remoteSite=args.remotesite,
dataset="",
eventType=args.eventtype)
job = Job(produserid=args.produserid,
jobid=args.jobid,
taskid=args.taskid,
jobdefinitionid=args.jobdefinitionid)
trace_report.init(job)
try:
infoservice = InfoService()
infoservice.init(args.queuename, infosys.confinfo, infosys.extinfo)
infosys.init(
args.queuename
) # is this correct? otherwise infosys.queuedata doesn't get set
except Exception as e:
message(e)
# perform stage-in (single transfers)
err = ""
errcode = 0
if args.eventservicemerge:
client = StageInESClient(infoservice,
logger=logger,
trace_report=trace_report)
activity = 'es_events_read'
# This script shows how to use the Data API stage-in client to download a file from storage
from pilot.api import data
from pilot.info import InfoService, FileSpec, infosys
# Without infosys initialization:
client = data.StageInClient()
files = [{
'scope': 'mc16_13TeV',
'lfn': 'EVNT.11320990._003958.pool.root.1',
'workdir': '.',
'ddmendpoint': 'RRC-KI-T1_DATADISK'
}]
xfiles = [FileSpec(type='input', **f) for f in files]
r = client.transfer(xfiles)
# With infosys initialization:
infoservice = InfoService()
infoservice.init('ANALY_CERN', infosys.confinfo, infosys.extinfo)
client = data.StageInClient(infoservice)
files = [{
'scope': 'mc16_13TeV',
'lfn': 'EVNT.11320990._003958.pool.root.1',
'workdir': '.'
}]
xfiles = [FileSpec(type='input', **f) for f in files]
r = client.transfer(xfiles)