def main():
"""
Main function of PanDA Pilot 2.
Prepare for and execute the requested workflow.
:return: exit code (int).
"""
# get the logger
logger = logging.getLogger(__name__)
# print the pilot version
pilot_version_banner()
# define threading events
args.graceful_stop = threading.Event()
args.abort_job = threading.Event()
args.job_aborted = threading.Event()
# define useful variables
args.retrieve_next_job = True # go ahead and download a new job
args.signal = None # to store any incoming signals
args.signal_counter = 0 # keep track of number of received kill signal (suicide counter)
args.kill_time = 0 # keep track of when first kill signal arrived
# read and parse config file
config.read(args.config)
# perform https setup
https_setup(args, get_pilot_version())
# initialize InfoService
try:
infosys.init(args.queue)
# check if queue is ACTIVE
if infosys.queuedata.state != 'ACTIVE':
logger.critical('specified queue is NOT ACTIVE: %s -- aborting' %
infosys.queuedata.name)
raise PilotException("Panda Queue is NOT ACTIVE")
except PilotException as error:
logger.fatal(error)
return error.get_error_code()
# set the site name for rucio ## is it really used?
environ['PILOT_RUCIO_SITENAME'] = infosys.queuedata.site
# set requested workflow
logger.info('pilot arguments: %s' % str(args))
logger.info('selected workflow: %s' % args.workflow)
workflow = __import__('pilot.workflow.%s' % args.workflow, globals(),
locals(), [args.workflow], -1)
# execute workflow
try:
exit_code = workflow.run(args)
except Exception as e:
logger.fatal('main pilot function caught exception: %s' % e)
exit_code = None
return exit_code
def setUp(self):
# skip tests if running on a Mac -- Macs don't have /proc
self.mac = False
if os.environ.get('MACOSX') == 'true' or not os.path.exists('/proc/meminfo'):
self.mac = True
from pilot.info import infosys
infosys.init("CERN")
def stage_in(self, tmpLog, jobspec, files):
tmpLog.debug('To stagein files[] {0}'.format(files))
# get infosys
# infoservice = InfoService()
# infoservice.init(jobspec.computingSite, infosys.confinfo, infosys.extinfo)
infosys.init(jobspec.computingSite, infosys.confinfo, infosys.extinfo)
# always disable remote/direct io
infosys.queuedata.direct_access_lan = False
infosys.queuedata.direct_access_wan = False
# set data client, always use rucio
data_client = data.StageInClient(infosys,
acopytools={'default': ['rucio']},
default_copytools='rucio')
allChecked = True
ErrMsg = 'These files failed to download : '
# change directory to basPath for input to pass pilot check_availablespace
os.chdir(self.basePath)
# transfer
if len(files) > 0:
try:
result = data_client.transfer(files)
except Exception as e:
tmpLog.error('error when stage_in: {0} ; {1}'.format(
e.__class__.__name__, e))
raise
else:
tmpLog.debug(
'pilot.api data.StageInClient.transfer(files) result: {0}'.
format(result))
# loop over each file check result all must be true for entire result to be true
if result:
for answer in result:
if answer.status_code != 0:
allChecked = False
ErrMsg = ErrMsg + (" %s " % answer.lfn)
else:
tmpLog.info(
'Looks like all files already inplace: {0}'.format(
files))
# return
tmpLog.debug('stop thread')
if allChecked:
return True, ''
else:
return False, ErrMsg
trace_report = TraceReport(pq=os.environ.get('PILOT_SITENAME', ''),
localSite=args.localsite,
remoteSite=args.remotesite,
dataset="",
eventType=args.eventtype)
job = Job(produserid=args.produserid,
jobid=args.jobid,
taskid=args.taskid,
jobdefinitionid=args.jobdefinitionid)
trace_report.init(job)
try:
infoservice = InfoService()
infoservice.init(args.queuename, infosys.confinfo, infosys.extinfo)
infosys.init(
args.queuename
) # is this correct? otherwise infosys.queuedata doesn't get set
except Exception as e:
message(e)
# perform stage-in (single transfers)
err = ""
errcode = 0
if args.eventservicemerge:
client = StageInESClient(infoservice,
logger=logger,
trace_report=trace_report)
activity = 'es_events_read'
else:
client = StageInClient(infoservice,
logger=logger,