def main(): """ Main function of PanDA Pilot 2. Prepare for and execute the requested workflow. :return: exit code (int). """ # get the logger logger = logging.getLogger(__name__) # print the pilot version pilot_version_banner() # define threading events args.graceful_stop = threading.Event() args.abort_job = threading.Event() args.job_aborted = threading.Event() # define useful variables args.retrieve_next_job = True # go ahead and download a new job args.signal = None # to store any incoming signals args.signal_counter = 0 # keep track of number of received kill signal (suicide counter) args.kill_time = 0 # keep track of when first kill signal arrived # read and parse config file config.read(args.config) # perform https setup https_setup(args, get_pilot_version()) # initialize InfoService try: infosys.init(args.queue) # check if queue is ACTIVE if infosys.queuedata.state != 'ACTIVE': logger.critical('specified queue is NOT ACTIVE: %s -- aborting' % infosys.queuedata.name) raise PilotException("Panda Queue is NOT ACTIVE") except PilotException as error: logger.fatal(error) return error.get_error_code() # set the site name for rucio ## is it really used? environ['PILOT_RUCIO_SITENAME'] = infosys.queuedata.site # set requested workflow logger.info('pilot arguments: %s' % str(args)) logger.info('selected workflow: %s' % args.workflow) workflow = __import__('pilot.workflow.%s' % args.workflow, globals(), locals(), [args.workflow], -1) # execute workflow try: exit_code = workflow.run(args) except Exception as e: logger.fatal('main pilot function caught exception: %s' % e) exit_code = None return exit_code
def set_environment_variables(args, mainworkdir): """ Set environment variables. To be replaced with singleton implementation. This function sets PILOT_WORK_DIR, PILOT_HOME, PILOT_SITENAME, PILOT_USER and PILOT_VERSION and others. :param args: args object. :param mainworkdir: work directory (string). :return: """ # working directory as set with a pilot option (e.g. ..) environ['PILOT_WORK_DIR'] = args.workdir # TODO: replace with singleton # main work directory (e.g. /scratch/PanDA_Pilot2_3908_1537173670) environ['PILOT_HOME'] = mainworkdir # TODO: replace with singleton # pilot source directory (e.g. /cluster/home/usatlas1/gram_scratch_hHq4Ns/condorg_oqmHdWxz) environ[ 'PILOT_SOURCE_DIR'] = args.sourcedir # TODO: replace with singleton # set the pilot user (e.g. ATLAS) environ['PILOT_USER'] = args.pilot_user # TODO: replace with singleton # internal pilot state environ['PILOT_JOB_STATE'] = 'startup' # TODO: replace with singleton # set the pilot version environ['PILOT_VERSION'] = get_pilot_version() # set the default wrap-up/finish instruction environ['PILOT_WRAP_UP'] = 'NORMAL' # proxy verifications environ['PILOT_PROXY_VERIFICATION'] = '%s' % args.verify_proxy environ[ 'PILOT_PAYLOAD_PROXY_VERIFICATION'] = '%s' % args.verify_payload_proxy # keep track of the server updates, if any environ['SERVER_UPDATE'] = SERVER_UPDATE_NOT_DONE # set the (HPC) resource name (if set in options) environ['PILOT_RESOURCE_NAME'] = args.hpc_resource # allow for the possibility of turning off rucio traces environ['PILOT_USE_RUCIO_TRACES'] = str(args.use_rucio_traces) # event service executor type environ['PILOT_ES_EXECUTOR_TYPE'] = args.executor_type if args.output_dir: environ['PILOT_OUTPUT_DIR'] = args.output_dir # keep track of the server urls _port = ":%s" % args.port url = args.url if _port in args.url else args.url + _port environ['PANDA_SERVER_URL'] = url environ['QUEUEDATA_SERVER_URL'] = '%s' % args.queuedata_url
def set_environment_variables(args, mainworkdir): """ Set environment variables. To be replaced with singleton implementation. This function sets PILOT_WORK_DIR, PILOT_HOME, PILOT_SITENAME, PILOT_USER and PILOT_VERSION and others. :param args: args object. :param mainworkdir: work directory (string). :return: """ # working directory as set with a pilot option (e.g. ..) environ['PILOT_WORK_DIR'] = args.workdir # TODO: replace with singleton # main work directory (e.g. /scratch/PanDA_Pilot2_3908_1537173670) environ['PILOT_HOME'] = mainworkdir # TODO: replace with singleton # pilot source directory (e.g. /cluster/home/usatlas1/gram_scratch_hHq4Ns/condorg_oqmHdWxz) environ['PILOT_SOURCE_DIR'] = args.sourcedir # TODO: replace with singleton # set the pilot user (e.g. ATLAS) environ['PILOT_USER'] = args.pilot_user # TODO: replace with singleton # internal pilot state environ['PILOT_JOB_STATE'] = 'startup' # TODO: replace with singleton # set the pilot version environ['PILOT_VERSION'] = get_pilot_version() # set the default wrap-up/finish instruction environ['PILOT_WRAP_UP'] = 'NORMAL' # proxy verifications environ['PILOT_PROXY_VERIFICATION'] = '%s' % args.verify_proxy # environ['PILOT_PAYLOAD_PROXY_VERIFICATION'] = '%s' % args.verify_payload_proxy # keep track of the server updates, if any environ['SERVER_UPDATE'] = SERVER_UPDATE_NOT_DONE # set the (HPC) resource name (if set in options) environ['PILOT_RESOURCE_NAME'] = args.hpc_resource # keep track of the PanDA server url environ['QUEUEDATA_SERVER_URL'] = '%s' % args.queuedata_url
def pilot_version_banner(): """ Print a pilot version banner. :return: """ logger = logging.getLogger(__name__) version = '*** PanDA Pilot version %s ***' % get_pilot_version() logger.info('*' * len(version)) logger.info(version) logger.info('*' * len(version)) logger.info('') if is_virtual_machine(): logger.info('pilot is running in a VM') display_architecture_info() logger.info('*' * len(version))
def __init__(self, *args, **kwargs): event_version = "%s+%s" % (get_pilot_version(), get_rucio_client_version()) defs = { # for reference, see Tracing report document in wiki area of Pilot GitHub repository 'eventType': '', 'eventVersion': event_version, # Pilot+Rucio client version 'protocol': None, # set by specific copy tool 'clientState': 'INIT_REPORT', 'localSite': environ.get('RUCIO_LOCAL_SITE_ID', ''), 'remoteSite': '', 'timeStart': None, 'catStart': None, 'relativeStart': None, 'transferStart': None, 'validateStart': None, 'timeEnd': None, 'dataset': '', 'version': None, 'duid': None, 'filename': None, 'guid': None, 'filesize': None, 'usr': None, 'appid': None, 'hostname': '', 'ip': '', 'suspicious': '0', 'usrdn': '', 'url': None, 'stateReason': None, 'uuid': None, 'taskid': '', 'pq': environ.get('PILOT_SITENAME', '') } super(TraceReport, self).__init__(defs) self.update(dict(*args, **kwargs)) # apply extra input
def __init__(self, *args, **kwargs): event_version = "%s+%s" % (get_pilot_version(), get_rucio_client_version()) defs = { 'eventType': '', # sitemover 'eventVersion': event_version, # Pilot+Rucio client version 'protocol': None, # set by specific copy tool 'clientState': 'INIT_REPORT', 'localSite': '', # localsite 'remoteSite': '', # equals remotesite 'timeStart': None, # time to start 'catStart': None, 'relativeStart': None, 'transferStart': None, 'validateStart': None, 'timeEnd': None, 'dataset': '', 'version': None, 'duid': None, 'filename': None, 'guid': None, 'filesize': None, 'usr': None, 'appid': None, 'hostname': '', 'ip': '', 'suspicious': '0', 'usrdn': '', 'url': None, 'stateReason': None, 'uuid': None, 'taskid': '' } super(TraceReport, self).__init__(defs) self.update(dict(*args, **kwargs)) # apply extra input
def set_environment_variables(args, mainworkdir): """ Set environment variables. To be replaced with singleton implementation. This function sets PILOT_WORK_DIR, PILOT_HOME, PILOT_SITENAME, PILOT_USER and PILOT_VERSION and others. :param args: :param mainworkdir: :return: """ # working directory as set with a pilot option (e.g. ..) environ['PILOT_WORK_DIR'] = args.workdir # TODO: replace with singleton # main work directory (e.g. /scratch/PanDA_Pilot2_3908_1537173670) environ['PILOT_HOME'] = mainworkdir # TODO: replace with singleton # pilot source directory (e.g. /cluster/home/usatlas1/gram_scratch_hHq4Ns/condorg_oqmHdWxz) environ[ 'PILOT_SOURCE_DIR'] = args.sourcedir # TODO: replace with singleton # store the site name as set with a pilot option environ['PILOT_SITENAME'] = args.site # TODO: replace with singleton # set the pilot user (e.g. ATLAS) environ['PILOT_USER'] = args.pilot_user # TODO: replace with singleton # internal pilot state environ['PILOT_JOB_STATE'] = 'startup' # TODO: replace with singleton # set the pilot version environ['PILOT_VERSION'] = get_pilot_version() # set the default wrap-up/finish instruction environ['PILOT_WRAP_UP'] = 'NORMAL' # keep track of the server updates, if any environ['SERVER_UPDATE'] = SERVER_UPDATE_NOT_DONE