def main():
"""
Main function of PanDA Pilot 2.
Prepare for and execute the requested workflow.
:return: exit code (int).
"""
# get the logger
logger = logging.getLogger(__name__)
# print the pilot version
pilot_version_banner()
# define threading events
args.graceful_stop = threading.Event()
args.abort_job = threading.Event()
args.job_aborted = threading.Event()
# define useful variables
args.retrieve_next_job = True # go ahead and download a new job
args.signal = None # to store any incoming signals
args.signal_counter = 0 # keep track of number of received kill signal (suicide counter)
args.kill_time = 0 # keep track of when first kill signal arrived
# read and parse config file
config.read(args.config)
# perform https setup
https_setup(args, get_pilot_version())
# initialize InfoService
try:
infosys.init(args.queue)
# check if queue is ACTIVE
if infosys.queuedata.state != 'ACTIVE':
logger.critical('specified queue is NOT ACTIVE: %s -- aborting' %
infosys.queuedata.name)
raise PilotException("Panda Queue is NOT ACTIVE")
except PilotException as error:
logger.fatal(error)
return error.get_error_code()
# set the site name for rucio ## is it really used?
environ['PILOT_RUCIO_SITENAME'] = infosys.queuedata.site
# set requested workflow
logger.info('pilot arguments: %s' % str(args))
logger.info('selected workflow: %s' % args.workflow)
workflow = __import__('pilot.workflow.%s' % args.workflow, globals(),
locals(), [args.workflow], -1)
# execute workflow
try:
exit_code = workflow.run(args)
except Exception as e:
logger.fatal('main pilot function caught exception: %s' % e)
exit_code = None
return exit_code
def set_environment_variables(args, mainworkdir):
"""
Set environment variables. To be replaced with singleton implementation.
This function sets PILOT_WORK_DIR, PILOT_HOME, PILOT_SITENAME, PILOT_USER and PILOT_VERSION and others.
:param args: args object.
:param mainworkdir: work directory (string).
:return:
"""
# working directory as set with a pilot option (e.g. ..)
environ['PILOT_WORK_DIR'] = args.workdir # TODO: replace with singleton
# main work directory (e.g. /scratch/PanDA_Pilot2_3908_1537173670)
environ['PILOT_HOME'] = mainworkdir # TODO: replace with singleton
# pilot source directory (e.g. /cluster/home/usatlas1/gram_scratch_hHq4Ns/condorg_oqmHdWxz)
environ[
'PILOT_SOURCE_DIR'] = args.sourcedir # TODO: replace with singleton
# set the pilot user (e.g. ATLAS)
environ['PILOT_USER'] = args.pilot_user # TODO: replace with singleton
# internal pilot state
environ['PILOT_JOB_STATE'] = 'startup' # TODO: replace with singleton
# set the pilot version
environ['PILOT_VERSION'] = get_pilot_version()
# set the default wrap-up/finish instruction
environ['PILOT_WRAP_UP'] = 'NORMAL'
# proxy verifications
environ['PILOT_PROXY_VERIFICATION'] = '%s' % args.verify_proxy
environ[
'PILOT_PAYLOAD_PROXY_VERIFICATION'] = '%s' % args.verify_payload_proxy
# keep track of the server updates, if any
environ['SERVER_UPDATE'] = SERVER_UPDATE_NOT_DONE
# set the (HPC) resource name (if set in options)
environ['PILOT_RESOURCE_NAME'] = args.hpc_resource
# allow for the possibility of turning off rucio traces
environ['PILOT_USE_RUCIO_TRACES'] = str(args.use_rucio_traces)
# event service executor type
environ['PILOT_ES_EXECUTOR_TYPE'] = args.executor_type
if args.output_dir:
environ['PILOT_OUTPUT_DIR'] = args.output_dir
# keep track of the server urls
_port = ":%s" % args.port
url = args.url if _port in args.url else args.url + _port
environ['PANDA_SERVER_URL'] = url
environ['QUEUEDATA_SERVER_URL'] = '%s' % args.queuedata_url
def set_environment_variables(args, mainworkdir):
"""
Set environment variables. To be replaced with singleton implementation.
This function sets PILOT_WORK_DIR, PILOT_HOME, PILOT_SITENAME, PILOT_USER and PILOT_VERSION and others.
:param args: args object.
:param mainworkdir: work directory (string).
:return:
"""
# working directory as set with a pilot option (e.g. ..)
environ['PILOT_WORK_DIR'] = args.workdir # TODO: replace with singleton
# main work directory (e.g. /scratch/PanDA_Pilot2_3908_1537173670)
environ['PILOT_HOME'] = mainworkdir # TODO: replace with singleton
# pilot source directory (e.g. /cluster/home/usatlas1/gram_scratch_hHq4Ns/condorg_oqmHdWxz)
environ['PILOT_SOURCE_DIR'] = args.sourcedir # TODO: replace with singleton
# set the pilot user (e.g. ATLAS)
environ['PILOT_USER'] = args.pilot_user # TODO: replace with singleton
# internal pilot state
environ['PILOT_JOB_STATE'] = 'startup' # TODO: replace with singleton
# set the pilot version
environ['PILOT_VERSION'] = get_pilot_version()
# set the default wrap-up/finish instruction
environ['PILOT_WRAP_UP'] = 'NORMAL'
# proxy verifications
environ['PILOT_PROXY_VERIFICATION'] = '%s' % args.verify_proxy
# environ['PILOT_PAYLOAD_PROXY_VERIFICATION'] = '%s' % args.verify_payload_proxy
# keep track of the server updates, if any
environ['SERVER_UPDATE'] = SERVER_UPDATE_NOT_DONE
# set the (HPC) resource name (if set in options)
environ['PILOT_RESOURCE_NAME'] = args.hpc_resource
# keep track of the PanDA server url
environ['QUEUEDATA_SERVER_URL'] = '%s' % args.queuedata_url
def pilot_version_banner():
"""
Print a pilot version banner.
:return:
"""
logger = logging.getLogger(__name__)
version = '*** PanDA Pilot version %s ***' % get_pilot_version()
logger.info('*' * len(version))
logger.info(version)
logger.info('*' * len(version))
logger.info('')
if is_virtual_machine():
logger.info('pilot is running in a VM')
display_architecture_info()
logger.info('*' * len(version))
def __init__(self, *args, **kwargs):
event_version = "%s+%s" % (get_pilot_version(),
get_rucio_client_version())
defs = { # for reference, see Tracing report document in wiki area of Pilot GitHub repository
'eventType': '',
'eventVersion': event_version, # Pilot+Rucio client version
'protocol': None, # set by specific copy tool
'clientState': 'INIT_REPORT',
'localSite': environ.get('RUCIO_LOCAL_SITE_ID', ''),
'remoteSite': '',
'timeStart': None,
'catStart': None,
'relativeStart': None,
'transferStart': None,
'validateStart': None,
'timeEnd': None,
'dataset': '',
'version': None,
'duid': None,
'filename': None,
'guid': None,
'filesize': None,
'usr': None,
'appid': None,
'hostname': '',
'ip': '',
'suspicious': '0',
'usrdn': '',
'url': None,
'stateReason': None,
'uuid': None,
'taskid': '',
'pq': environ.get('PILOT_SITENAME', '')
}
super(TraceReport, self).__init__(defs)
self.update(dict(*args, **kwargs)) # apply extra input
def __init__(self, *args, **kwargs):
event_version = "%s+%s" % (get_pilot_version(),
get_rucio_client_version())
defs = {
'eventType': '', # sitemover
'eventVersion': event_version, # Pilot+Rucio client version
'protocol': None, # set by specific copy tool
'clientState': 'INIT_REPORT',
'localSite': '', # localsite
'remoteSite': '', # equals remotesite
'timeStart': None, # time to start
'catStart': None,
'relativeStart': None,
'transferStart': None,
'validateStart': None,
'timeEnd': None,
'dataset': '',
'version': None,
'duid': None,
'filename': None,
'guid': None,
'filesize': None,
'usr': None,
'appid': None,
'hostname': '',
'ip': '',
'suspicious': '0',
'usrdn': '',
'url': None,
'stateReason': None,
'uuid': None,
'taskid': ''
}
super(TraceReport, self).__init__(defs)
self.update(dict(*args, **kwargs)) # apply extra input
def set_environment_variables(args, mainworkdir):
"""
Set environment variables. To be replaced with singleton implementation.
This function sets PILOT_WORK_DIR, PILOT_HOME, PILOT_SITENAME, PILOT_USER and PILOT_VERSION and others.
:param args:
:param mainworkdir:
:return:
"""
# working directory as set with a pilot option (e.g. ..)
environ['PILOT_WORK_DIR'] = args.workdir # TODO: replace with singleton
# main work directory (e.g. /scratch/PanDA_Pilot2_3908_1537173670)
environ['PILOT_HOME'] = mainworkdir # TODO: replace with singleton
# pilot source directory (e.g. /cluster/home/usatlas1/gram_scratch_hHq4Ns/condorg_oqmHdWxz)
environ[
'PILOT_SOURCE_DIR'] = args.sourcedir # TODO: replace with singleton
# store the site name as set with a pilot option
environ['PILOT_SITENAME'] = args.site # TODO: replace with singleton
# set the pilot user (e.g. ATLAS)
environ['PILOT_USER'] = args.pilot_user # TODO: replace with singleton
# internal pilot state
environ['PILOT_JOB_STATE'] = 'startup' # TODO: replace with singleton
# set the pilot version
environ['PILOT_VERSION'] = get_pilot_version()
# set the default wrap-up/finish instruction
environ['PILOT_WRAP_UP'] = 'NORMAL'
# keep track of the server updates, if any
environ['SERVER_UPDATE'] = SERVER_UPDATE_NOT_DONE