def stageout_es_real(self, output_file):
"""
Stage out event service output file.
:param output_file: output file name.
"""
job = self.get_job()
logger.info('prepare to stage-out eventservice files')
error = None
file_data = {
'scope': 'transient',
'lfn': os.path.basename(output_file),
}
file_spec = FileSpec(filetype='output', **file_data)
xdata = [file_spec]
kwargs = dict(workdir=job.workdir,
cwd=job.workdir,
usecontainer=False,
job=job)
try_failover = False
activity = [
'es_events', 'pw'
] ## FIX ME LATER: replace `pw` with `write_lan` once AGIS is updated (acopytools)
try:
client = StageOutESClient(job.infosys, logger=logger)
try_failover = True
client.prepare_destinations(
xdata, activity
) ## IF ES job should be allowed to write only at `es_events` astorages, then fix activity names here
client.transfer(xdata, activity=activity, **kwargs)
except PilotException as error:
logger.error(error.get_detail())
except Exception as e:
logger.error(traceback.format_exc())
error = StageOutFailure("stageOut failed with error=%s" % e)
logger.info('Summary of transferred files:')
logger.info(" -- lfn=%s, status_code=%s, status=%s" %
(file_spec.lfn, file_spec.status_code, file_spec.status))
if error:
logger.error(
'Failed to stage-out eventservice file(%s): error=%s' %
(output_file, error.get_detail()))
elif file_spec.status != 'transferred':
msg = 'Failed to stage-out ES file(%s): logic corrupted: unknown internal error, fspec=%s' % (
output_file, file_spec)
logger.error(msg)
raise StageOutFailure(msg)
failover_storage_activity = ['es_failover', 'pw']
if try_failover and error and error.get_error_code() not in [
ErrorCodes.MISSINGOUTPUTFILE
]: ## try to failover to other storage
xdata2 = [FileSpec(filetype='output', **file_data)]
try:
client.prepare_destinations(xdata2, failover_storage_activity)
if xdata2[0].ddmendpoint != xdata[
0].ddmendpoint: ## skip transfer to same output storage
msg = 'Will try to failover ES transfer to astorage with activity=%s, rse=%s' % (
failover_storage_activity, xdata2[0].ddmendpoint)
logger.info(msg)
client.transfer(xdata2, activity=activity, **kwargs)
logger.info(
'Summary of transferred files (failover transfer):')
logger.info(" -- lfn=%s, status_code=%s, status=%s" %
(xdata2[0].lfn, xdata2[0].status_code,
xdata2[0].status))
except PilotException as e:
if e.get_error_code() == ErrorCodes.NOSTORAGE:
logger.info(
'Failover ES storage is not defined for activity=%s .. skipped'
% failover_storage_activity)
else:
logger.error(
'Transfer to failover storage=%s failed .. skipped, error=%s'
% (xdata2[0].ddmendpoint, e.get_detail()))
except Exception:
logger.error('Failover ES stageout failed .. skipped')
logger.error(traceback.format_exc())
if xdata2[0].status == 'transferred':
error = None
file_spec = xdata2[0]
if error:
raise error
storage_id = infosys.get_storage_id(file_spec.ddmendpoint)
return file_spec.ddmendpoint, storage_id, file_spec.filesize, file_spec.checksum
log.error(error.get_detail())
except Exception, e:
import traceback
log.error(traceback.format_exc())
error = exception.StageOutFailure("stageOut failed with error=%s" %
e)
log.info('Summary of transferred files:')
log.info(" -- lfn=%s, status_code=%s, status=%s" %
(file_spec.lfn, file_spec.status_code, file_spec.status))
if error or file_spec.status != 'transferred':
log.error('Failed to stage-out eventservice file(%s): error=%s' %
(output_file, error.get_detail()))
raise error
storage_id = infosys.get_storage_id(file_spec.ddmendpoint)
return file_spec.ddmendpoint, storage_id, file_spec.filesize, file_spec.checksum
def stageout_es(self, force=False):
"""
Stage out event service outputs.
"""
job = self.get_job()
log = get_logger(job.jobid, logger)
if len(self.__queued_out_messages):
if force or self.__last_stageout_time is None or (
time.time() > self.__last_stageout_time +
job.infosys.queuedata.es_stageout_gap):
def test_stageout_noexist_activity_stagein(self):
"""
Make sure that no exceptions to stage out file.
"""
error = None
try:
from pilot.info import infosys, InfoService
infoservice = InfoService()
infoservice.init('BNL_CLOUD_MCORE', infosys.confinfo,
infosys.extinfo)
output_file = os.path.join('/tmp', str(uuid.uuid4()))
shutil.copy('/bin/hostname', output_file)
file_data = {
'scope': 'transient',
'lfn': os.path.basename(output_file),
#'ddmendpoint': None,
#'type': 'es_events',
#'surl': output_file
#'turl': None,
#'filesize': None,
#'checksum': None
}
file_spec = FileSpec(filetype='output', **file_data)
xdata = [file_spec]
workdir = os.path.dirname(output_file)
client = StageOutESClient(infoservice)
kwargs = dict(workdir=workdir, cwd=workdir, usecontainer=False)
client.prepare_destinations(
xdata, activity=[
'es_events_no_exist', 'pw'
]) # allow to write to `es_events_no_exist` and `pw` astorages
client.transfer(xdata,
activity=['es_events_no_exist', 'pw'],
**kwargs)
except exception.PilotException as error: # Python 2/3
logger.error("Pilot Exeception: %s, %s" %
(error.get_detail(), traceback.format_exc()))
except Exception as e: # Python 2/3
logger.error(traceback.format_exc())
error = exception.StageOutFailure("stageOut failed with error=%s" %
e)
logger.info('Summary of transferred files:')
for e in xdata:
logger.info(" -- lfn=%s, status_code=%s, status=%s" %
(e.lfn, e.status_code, e.status))
if error:
logger.error(
'Failed to stage-out eventservice file(%s): error=%s' %
(output_file, error.get_detail()))
raise error
storage_id = infosys.get_storage_id(file_spec.ddmendpoint)
logger.info('File %s staged out to %s(id: %s)' %
(file_spec.lfn, file_spec.ddmendpoint, storage_id))
new_file_data = {
'scope': 'test',
'lfn': file_spec.lfn,
'storage_token': '%s/1000' % storage_id
}
try:
new_file_spec = FileSpec(filetype='input', **new_file_data)
xdata = [new_file_spec]
workdir = os.path.dirname(output_file)
client = StageInESClient(infoservice)
kwargs = dict(workdir=workdir, cwd=workdir, usecontainer=False)
client.prepare_sources(xdata)
client.transfer(xdata, activity=['es_events_read'], **kwargs)
except exception.PilotException as error: # Python 2/3
logger.error("Pilot Exeception: %s, %s" %
(error.get_detail(), traceback.format_exc()))
except Exception as e: # Python 2/3
logger.error(traceback.format_exc())
error = exception.StageInFailure("stagein failed with error=%s" %
e)
logger.info('Summary of transferred files:')
for e in xdata:
logger.info(" -- lfn=%s, status_code=%s, status=%s" %
(e.lfn, e.status_code, e.status))
if error:
logger.error('Failed to stage-in eventservice file(%s): error=%s' %
(output_file, error.get_detail()))
raise error