def stageout_es_real(self, output_file):
"""
Stage out event service output file.
:param output_file: output file name.
"""
job = self.get_job()
log = get_logger(job.jobid, logger)
log.info('prepare to stage-out eventservice files')
error = None
try:
file_data = {
'scope': 'transient',
'lfn': os.path.basename(output_file),
}
file_spec = FileSpec(filetype='output', **file_data)
xdata = [file_spec]
client = StageOutESClient(job.infosys, logger=log)
kwargs = dict(workdir=job.workdir,
cwd=job.workdir,
usecontainer=False,
job=job)
client.transfer(xdata, activity=['es_events', 'pw'], **kwargs)
except exception.PilotException, error:
log.error(error.get_detail())
def test_stageout_es_events_pw(self):
"""
Make sure that no exceptions to stage out file.
"""
error = None
try:
from pilot.info import infosys, InfoService
infoservice = InfoService()
infoservice.init('BNL_CLOUD_MCORE', infosys.confinfo,
infosys.extinfo)
output_file = os.path.join('/tmp', str(uuid.uuid4()))
shutil.copy('/bin/hostname', output_file)
file_data = {
'scope': 'transient',
'lfn': os.path.basename(output_file),
#'ddmendpoint': None,
#'type': 'es_events',
#'surl': output_file
#'turl': None,
#'filesize': None,
#'checksum': None
}
file_spec = FileSpec(filetype='output', **file_data)
xdata = [file_spec]
workdir = os.path.dirname(output_file)
client = StageOutESClient(infoservice)
kwargs = dict(workdir=workdir, cwd=workdir, usecontainer=False)
client.transfer(xdata, activity=['es_events', 'pw'], **kwargs)
except exception.PilotException, error:
logger.error("Pilot Exeception: %s, %s" %
(error.get_detail(), traceback.format_exc()))
def create_file_spec(self, pfn):
checksum = calculate_checksum(pfn)
filesize = os.path.getsize(pfn)
file_data = {'scope': 'transient',
'lfn': os.path.basename(pfn),
'checksum': checksum,
'filesize': filesize,
}
file_spec = FileSpec(filetype='output', **file_data)
return file_spec
def test_stageout_es_events_non_exist_pw(self):
"""
Make sure that no exceptions to stage out file.
"""
error = None
try:
from pilot.info import infosys, InfoService
infoservice = InfoService()
infoservice.init('BNL_CLOUD_MCORE', infosys.confinfo,
infosys.extinfo)
output_file = os.path.join('/tmp', str(uuid.uuid4()))
shutil.copy('/bin/hostname', output_file)
file_data = {
'scope': 'transient',
'lfn': os.path.basename(output_file),
#'ddmendpoint': None,
#'type': 'es_events',
#'surl': output_file
#'turl': None,
#'filesize': None,
#'checksum': None
}
file_spec = FileSpec(filetype='output', **file_data)
xdata = [file_spec]
workdir = os.path.dirname(output_file)
client = StageOutESClient(infoservice)
kwargs = dict(workdir=workdir, cwd=workdir, usecontainer=False)
client.prepare_destinations(
xdata, activity=['es_events_non_exist', 'pw']
) # allow to write to `es_events_non_exist` and `pw` astorages
client.transfer(xdata,
activity=['es_events_non_exist', 'pw'],
**kwargs)
except exception.PilotException as error: # Python 2/3
logger.error("Pilot Exeception: %s, %s" %
(error.get_detail(), traceback.format_exc()))
except Exception as e: # Python 2/3
logger.error(traceback.format_exc())
error = exception.StageOutFailure("stageOut failed with error=%s" %
e)
logger.info('Summary of transferred files:')
for e in xdata:
logger.info(" -- lfn=%s, status_code=%s, status=%s" %
(e.lfn, e.status_code, e.status))
if error:
logger.error(
'Failed to stage-out eventservice file(%s): error=%s' %
(output_file, error.get_detail()))
raise error
def setUp(self):
test_file = open('test.txt', 'w')
test_file.write('For test purposes only.')
test_file.close()
fspec_out = FileSpec()
fspec_out.lfn = 'test.txt'
fspec_out.scope = 'user.tjavurek'
fspec_out.checksum = {'adler32': '682c08b9'}
fspec_out.pfn = os.getcwd() + '/' + 'test.txt'
fspec_out.ddmendpoint = 'UNI-FREIBURG_SCRATCHDISK'
self.outdata = [fspec_out]
def stageout_es_real(self, output_file):
"""
Stage out event service output file.
:param output_file: output file name.
"""
job = self.get_job()
logger.info('prepare to stage-out eventservice files')
error = None
file_data = {
'scope': 'transient',
'lfn': os.path.basename(output_file),
}
file_spec = FileSpec(filetype='output', **file_data)
xdata = [file_spec]
kwargs = dict(workdir=job.workdir,
cwd=job.workdir,
usecontainer=False,
job=job)
try_failover = False
activity = [
'es_events', 'pw'
] ## FIX ME LATER: replace `pw` with `write_lan` once AGIS is updated (acopytools)
try:
client = StageOutESClient(job.infosys, logger=logger)
try_failover = True
client.prepare_destinations(
xdata, activity
) ## IF ES job should be allowed to write only at `es_events` astorages, then fix activity names here
client.transfer(xdata, activity=activity, **kwargs)
except PilotException as error:
logger.error(error.get_detail())
except Exception as e:
logger.error(traceback.format_exc())
error = StageOutFailure("stageOut failed with error=%s" % e)
logger.info('Summary of transferred files:')
logger.info(" -- lfn=%s, status_code=%s, status=%s" %
(file_spec.lfn, file_spec.status_code, file_spec.status))
if error:
logger.error(
'Failed to stage-out eventservice file(%s): error=%s' %
(output_file, error.get_detail()))
elif file_spec.status != 'transferred':
msg = 'Failed to stage-out ES file(%s): logic corrupted: unknown internal error, fspec=%s' % (
output_file, file_spec)
logger.error(msg)
raise StageOutFailure(msg)
failover_storage_activity = ['es_failover', 'pw']
if try_failover and error and error.get_error_code() not in [
ErrorCodes.MISSINGOUTPUTFILE
]: ## try to failover to other storage
xdata2 = [FileSpec(filetype='output', **file_data)]
try:
client.prepare_destinations(xdata2, failover_storage_activity)
if xdata2[0].ddmendpoint != xdata[
0].ddmendpoint: ## skip transfer to same output storage
msg = 'Will try to failover ES transfer to astorage with activity=%s, rse=%s' % (
failover_storage_activity, xdata2[0].ddmendpoint)
logger.info(msg)
client.transfer(xdata2, activity=activity, **kwargs)
logger.info(
'Summary of transferred files (failover transfer):')
logger.info(" -- lfn=%s, status_code=%s, status=%s" %
(xdata2[0].lfn, xdata2[0].status_code,
xdata2[0].status))
except PilotException as e:
if e.get_error_code() == ErrorCodes.NOSTORAGE:
logger.info(
'Failover ES storage is not defined for activity=%s .. skipped'
% failover_storage_activity)
else:
logger.error(
'Transfer to failover storage=%s failed .. skipped, error=%s'
% (xdata2[0].ddmendpoint, e.get_detail()))
except Exception:
logger.error('Failover ES stageout failed .. skipped')
logger.error(traceback.format_exc())
if xdata2[0].status == 'transferred':
error = None
file_spec = xdata2[0]
if error:
raise error
storage_id = infosys.get_storage_id(file_spec.ddmendpoint)
return file_spec.ddmendpoint, storage_id, file_spec.filesize, file_spec.checksum
from pandaharvester.harvestercore.job_spec import JobSpec
from pilot.info.filespec import FileSpec
queueName = sys.argv[1]
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(queueName)
jobSpec = JobSpec()
new_file_data = {
'scope': 'test',
'lfn': 'TXT.19772875._044894.tar.gz.1',
'attemptNr': 0
}
new_file_spec = FileSpec(filetype='input', **new_file_data)
new_file_spec.attemptNr = 0
new_file_spec.path = '/home/psvirin/harvester3'
jobSpec.inFiles = {new_file_spec}
jobSpec.outFiles = {}
jobSpec.jobParams = {
'inFiles':
'TXT.19772875._044894.tar.gz.1',
'scopeIn':
'mc15_13TeV',
'fsize':
'658906675',
'GUID':
'7e3776f9bb0af341b03e59d3de895a13',
'checksum':
logger.error(
'Failed to stage-out eventservice file(%s): error=%s' %
(output_file, error.get_detail()))
raise error
storage_id = infosys.get_storage_id(file_spec.ddmendpoint)
logger.info('File %s staged out to %s(id: %s)' %
(file_spec.lfn, file_spec.ddmendpoint, storage_id))
new_file_data = {
'scope': 'test',
'lfn': file_spec.lfn,
'storage_token': '%s/1000' % storage_id
}
try:
new_file_spec = FileSpec(filetype='input', **new_file_data)
xdata = [new_file_spec]
workdir = os.path.dirname(output_file)
client = StageInESClient(infoservice)
kwargs = dict(workdir=workdir, cwd=workdir, usecontainer=False)
client.transfer(xdata, activity=['es_events_read'], **kwargs)
except exception.PilotException, error:
logger.error("Pilot Exeception: %s, %s" %
(error.get_detail(), traceback.format_exc()))
except Exception, e:
logger.error(traceback.format_exc())
error = exception.StageInFailure("stagein failed with error=%s" %
e)
logger.info('Summary of transferred files:')