args.queuename
) # is this correct? otherwise infosys.queuedata doesn't get set
except Exception as e:
message(e)
# perform stage-in (single transfers)
err = ""
errcode = 0
if args.eventservicemerge:
client = StageInESClient(infoservice,
logger=logger,
trace_report=trace_report)
activity = 'es_events_read'
else:
client = StageInClient(infoservice,
logger=logger,
trace_report=trace_report)
activity = 'pr'
kwargs = dict(workdir=args.workdir,
cwd=args.workdir,
usecontainer=False,
use_pcache=args.usepcache,
use_bulk=False,
use_vp=args.usevp,
input_dir=args.inputdir,
catchall=args.catchall)
xfiles = []
for lfn in replica_dictionary:
files = [{
'scope':
replica_dictionary[lfn]['scope'],
def _stage_in(args, job):
"""
:return: True in case of success
"""
log = get_logger(job.jobid)
# tested ok:
#log.info('testing sending SIGUSR1')
#import signal
#os.kill(os.getpid(), signal.SIGUSR1)
# write time stamps to pilot timing file
add_to_pilot_timing(job.jobid, PILOT_PRE_STAGEIN, time.time(), args)
# any DBRelease files should not be staged in
for fspec in job.indata:
if 'DBRelease' in fspec.lfn:
fspec.status = 'no_transfer'
event_type = "get_sm"
#if log_transfer:
# eventType += '_logs'
#if special_log_transfer:
# eventType += '_logs_os'
if job.is_analysis():
event_type += "_a"
rse = get_rse(job.indata)
localsite = remotesite = rse
trace_report = TraceReport(pq='', localSite=localsite, remoteSite=remotesite, dataset="", eventType=event_type)
trace_report.init(job)
# now that the trace report has been created, remove any files that are not to be transferred (DBRelease files) from the indata list
toberemoved = []
for fspec in job.indata:
if fspec.status == 'no_transfer':
toberemoved.append(fspec)
for fspec in toberemoved:
logger.info('removing fspec object (lfn=%s) from list of input files' % fspec.lfn)
job.indata.remove(fspec)
try:
if job.is_eventservicemerge:
client = StageInESClient(job.infosys, logger=log, trace_report=trace_report)
activity = 'es_events_read'
else:
client = StageInClient(job.infosys, logger=log, trace_report=trace_report)
activity = 'pr'
kwargs = dict(workdir=job.workdir, cwd=job.workdir, usecontainer=False, job=job) #, mode='stage-in')
client.transfer(job.indata, activity=activity, **kwargs)
except PilotException as error:
log.error('PilotException caught: %s' % error)
job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(error.get_error_code())
except Exception as error:
log.error('failed to stage-in: error=%s' % error)
log.info('summary of transferred files:')
for e in job.indata:
if not e.status:
status = "(not transferred)"
else:
status = e.status
log.info(" -- lfn=%s, status_code=%s, status=%s" % (e.lfn, e.status_code, status))
# write time stamps to pilot timing file
add_to_pilot_timing(job.jobid, PILOT_POST_STAGEIN, time.time(), args)
remain_files = [e for e in job.indata if e.status not in ['remote_io', 'transferred', 'no_transfer']]
if not remain_files:
log.info("stage-in finished")
else:
log.info("stage-in failed")
return not remain_files
def _stage_in(args, job):
"""
:return: True in case of success
"""
# tested ok:
#logger.info('testing sending SIGUSR1')
#import signal
#os.kill(os.getpid(), signal.SIGUSR1)
# write time stamps to pilot timing file
add_to_pilot_timing(job.jobid, PILOT_PRE_STAGEIN, time.time(), args)
# any DBRelease files should not be staged in
skip_special_files(job)
# now that the trace report has been created, remove any files that are not to be transferred (DBRelease files) from the indata list
update_indata(job)
label = 'stage-in'
# should stage-in be done by a script (for containerisation) or by invoking the API (ie classic mode)?
use_container = pilot.util.middleware.use_middleware_script(
job.infosys.queuedata.container_type.get("middleware"))
if use_container:
logger.info('stage-in will be done by a script')
try:
eventtype, localsite, remotesite = get_trace_report_variables(
job, label=label)
pilot.util.middleware.containerise_middleware(
job,
job.indata,
args.queue,
eventtype,
localsite,
remotesite,
job.infosys.queuedata.container_options,
args.input_dir,
label=label,
container_type=job.infosys.queuedata.container_type.get(
"middleware"))
except PilotException as error:
logger.warning(
'stage-in containerisation threw a pilot exception: %s', error)
except Exception as error:
import traceback
logger.warning('stage-in containerisation threw an exception: %s',
error)
logger.error(traceback.format_exc())
else:
try:
logger.info('stage-in will not be done in a container')
# create the trace report
trace_report = create_trace_report(job, label=label)
if job.is_eventservicemerge:
client = StageInESClient(job.infosys,
logger=logger,
trace_report=trace_report)
activity = 'es_events_read'
else:
client = StageInClient(job.infosys,
logger=logger,
trace_report=trace_report)
activity = 'pr'
use_pcache = job.infosys.queuedata.use_pcache
kwargs = dict(workdir=job.workdir,
cwd=job.workdir,
usecontainer=False,
use_pcache=use_pcache,
use_bulk=False,
input_dir=args.input_dir,
use_vp=job.use_vp,
catchall=job.infosys.queuedata.catchall)
client.prepare_sources(job.indata)
client.transfer(job.indata, activity=activity, **kwargs)
except PilotException as error:
import traceback
error_msg = traceback.format_exc()
logger.error(error_msg)
msg = errors.format_diagnostics(error.get_error_code(), error_msg)
job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(
error.get_error_code(), msg=msg)
except Exception as error:
logger.error('failed to stage-in: error=%s', error)
logger.info('summary of transferred files:')
for infile in job.indata:
status = infile.status if infile.status else "(not transferred)"
logger.info(" -- lfn=%s, status_code=%s, status=%s", infile.lfn,
infile.status_code, status)
# write time stamps to pilot timing file
add_to_pilot_timing(job.jobid, PILOT_POST_STAGEIN, time.time(), args)
remain_files = [
infile for infile in job.indata
if infile.status not in ['remote_io', 'transferred', 'no_transfer']
]
logger.info("stage-in finished") if not remain_files else logger.info(
"stage-in failed")
return not remain_files
def _stage_in(args, job):
"""
:return: True in case of success
"""
log = get_logger(job.jobid)
# tested ok:
#log.info('testing sending SIGUSR1')
#import signal
#os.kill(os.getpid(), signal.SIGUSR1)
# write time stamps to pilot timing file
add_to_pilot_timing(job.jobid, PILOT_PRE_STAGEIN, time.time(), args)
# any DBRelease files should not be staged in
for fspec in job.indata:
if 'DBRelease' in fspec.lfn:
fspec.status = 'no_transfer'
event_type = "get_sm"
if job.is_analysis():
event_type += "_a"
rse = get_rse(job.indata)
localsite = remotesite = rse
trace_report = TraceReport(pq=os.environ.get('PILOT_SITENAME', ''),
localSite=localsite,
remoteSite=remotesite,
dataset="",
eventType=event_type)
trace_report.init(job)
# now that the trace report has been created, remove any files that are not to be transferred (DBRelease files) from the indata list
toberemoved = []
for fspec in job.indata:
if fspec.status == 'no_transfer':
toberemoved.append(fspec)
for fspec in toberemoved:
logger.info('removing fspec object (lfn=%s) from list of input files' %
fspec.lfn)
job.indata.remove(fspec)
########### bulk transfer test
# THE FOLLOWING WORKS BUT THERE IS AN ISSUE WITH TRACES, CHECK STAGEIN SCRIPT IF STORED CORRECTLY
#filename = 'initial_trace_report.json'
#tpath = os.path.join(job.workdir, filename)
#write_json(tpath, trace_report)
#lfns, scopes = get_filedata_strings(job.indata)
#script = 'stagein.py'
#srcdir = os.environ.get('PILOT_SOURCE_DIR')
#scriptpath = os.path.join(os.path.join(srcdir, 'pilot/scripts'), script)
#copy(scriptpath, srcdir)
#cmd = 'python %s --lfns=%s --scopes=%s --tracereportname=%s -w %s -d -q %s' %\
# (os.path.join(srcdir, script), lfns, scopes, tpath, job.workdir, args.queue)
#logger.debug('could have executed: %s' % script)
#exit_code, stdout, stderr = execute(cmd, mode='python')
#logger.debug('exit_code=%d' % exit_code)
#logger.debug('stdout=%s' % stdout)
#logger.debug('stderr=%s' % stderr)
########### bulk transfer test
try:
if job.is_eventservicemerge:
client = StageInESClient(job.infosys,
logger=log,
trace_report=trace_report)
activity = 'es_events_read'
else:
client = StageInClient(job.infosys,
logger=log,
trace_report=trace_report)
activity = 'pr'
kwargs = dict(workdir=job.workdir,
cwd=job.workdir,
usecontainer=False,
job=job,
use_bulk=False)
client.prepare_sources(job.indata)
client.transfer(job.indata, activity=activity, **kwargs)
except PilotException as error:
import traceback
error_msg = traceback.format_exc()
log.error(error_msg)
msg = errors.format_diagnostics(error.get_error_code(), error_msg)
job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(
error.get_error_code(), msg=msg)
except Exception as error:
log.error('failed to stage-in: error=%s' % error)
log.info('summary of transferred files:')
for e in job.indata:
status = e.status if e.status else "(not transferred)"
log.info(" -- lfn=%s, status_code=%s, status=%s" %
(e.lfn, e.status_code, status))
# write time stamps to pilot timing file
add_to_pilot_timing(job.jobid, PILOT_POST_STAGEIN, time.time(), args)
remain_files = [
e for e in job.indata
if e.status not in ['remote_io', 'transferred', 'no_transfer']
]
if not remain_files:
log.info("stage-in finished")
else:
log.info("stage-in failed")
return not remain_files