def replace_lfns_with_turls(cmd, workdir, filename, infiles, writetofile=""): """ Replace all LFNs with full TURLs in the payload execution command. This function is used with direct access in production jobs. Athena requires a full TURL instead of LFN. :param cmd: payload execution command (string). :param workdir: location of metadata file (string). :param filename: metadata file name (string). :param infiles: list of input files. :param writetofile: :return: updated cmd (string). """ turl_dictionary = {} # { LFN: TURL, ..} path = os.path.join(workdir, filename) if os.path.exists(path): file_info_dictionary = get_file_info_from_xml(workdir, filename=filename) for inputfile in infiles: if inputfile in cmd: turl = file_info_dictionary[inputfile][0] turl_dictionary[inputfile] = turl # if turl.startswith('root://') and turl not in cmd: if turl not in cmd: cmd = cmd.replace(inputfile, turl) logger.info("replaced '%s' with '%s' in the run command" % (inputfile, turl)) # replace the LFNs with TURLs in the writetofile input file list (if it exists) if writetofile and turl_dictionary: filenames = get_writetoinput_filenames(writetofile) logger.info("filenames=%s" % filenames) for fname in filenames: new_lines = [] path = os.path.join(workdir, fname) if os.path.exists(path): f = read_file(path) for line in f.split('\n'): fname = os.path.basename(line) if fname in turl_dictionary: turl = turl_dictionary[fname] new_lines.append(turl) else: if line: new_lines.append(line) lines = '\n'.join(new_lines) if lines: write_file(path, lines) logger.info("lines=%s" % lines) else: logger.warning("file does not exist: %s" % path) else: logger.warning( "could not find file: %s (cannot locate TURLs for direct access)" % filename) return cmd
def create_release_setup_old(cmd, atlas_setup, full_atlas_setup, release, imagename, workdir, is_cvmfs): """ Get the proper release setup script name, and create the script if necessary. This function also updates the cmd string (removes full asetup from payload command). Note: for stand-alone containers, the function will return /release_setup.sh and assume that this script exists in the container. The pilot will only create a my_release_setup.sh script for OS containers. In case the release setup is not present in an unpacked container, the function will reset the cmd string. :param cmd: Payload execution command (string). :param atlas_setup: asetup command (string). :param full_atlas_setup: full asetup command (string). :param release: software release, needed to determine Athena environment (string). :param imagename: container image name (string). :param workdir: job workdir (string). :param is_cvmfs: does the queue have cvmfs? (Boolean). :return: proper release setup name (string), updated cmd (string). """ release_setup_name = get_release_setup_name(release, imagename) # note: if release_setup_name.startswith('/'), the pilot will NOT create the script if not release_setup_name.startswith('/'): # extracted_asetup should be written to 'my_release_setup.sh' and cmd to 'container_script.sh' content = 'echo \"INFO: sourcing %s inside the container. ' \ 'This should not run if it is a ATLAS standalone container\"' % release_setup_name if is_cvmfs: content, cmd = extract_full_atlas_setup(cmd, atlas_setup) if not content: content = full_atlas_setup if not content: logger.debug( 'will create an empty (almost) release setup file since asetup could not be extracted from command' ) logger.debug( 'command to be written to release setup file:\n\n%s:\n\n%s\n', release_setup_name, content) try: write_file(os.path.join(workdir, release_setup_name), content, mute=False) except FileHandlingFailure as exc: logger.warning('exception caught: %s', exc) else: # reset cmd in case release_setup.sh does not exist in unpacked image (only for those containers) cmd = cmd.replace(';;', ';') if is_release_setup( release_setup_name, imagename) else '' # add the /srv for OS containers if not release_setup_name.startswith('/'): release_setup_name = os.path.join('/srv', release_setup_name) return release_setup_name, cmd
def create_input_file_metadata(file_dictionary, workdir, filename="PoolFileCatalog.xml"): """ Create a Pool File Catalog for the files listed in the input dictionary. The function creates properly formatted XML (pretty printed) and writes the XML to file. Note: any environment variables in the pfn tags will be expanded (see pilot/control/data::get_input_file_dictionary()). Format: dictionary = {'guid': 'pfn', ..} -> <POOLFILECATALOG> <!DOCTYPE POOLFILECATALOG SYSTEM "InMemory"> <File ID="guid"> <physical> <pfn filetype="ROOT_All" name="surl"/> </physical> <logical/> </File> <POOLFILECATALOG> :param file_dictionary: file dictionary. :param workdir: job work directory (string). :param filename: PFC file name (string). :return: xml (string) """ # create the file structure data = ElementTree.Element('POOLFILECATALOG') for fileid in list(file_dictionary.keys()): # Python 2/3 _file = ElementTree.SubElement(data, 'File') _file.set('ID', fileid) _physical = ElementTree.SubElement(_file, 'physical') _pfn = ElementTree.SubElement(_physical, 'pfn') _pfn.set('filetype', 'ROOT_All') _pfn.set('name', file_dictionary.get(fileid)) ElementTree.SubElement(_file, 'logical') # create a new XML file with the results xml = ElementTree.tostring(data, encoding='utf8') xml = minidom.parseString(xml).toprettyxml(indent=" ") # add escape character for & (needed for google turls) if '&' in xml: xml = xml.replace('&', '&') # stitch in the DOCTYPE xml = xml.replace( '<POOLFILECATALOG>', '<!DOCTYPE POOLFILECATALOG SYSTEM "InMemory">\n<POOLFILECATALOG>') write_file(os.path.join(workdir, filename), xml, mute=False) return xml
def create_release_setup(cmd, atlas_setup, full_atlas_setup, release, imagename, workdir, is_cvmfs): """ Get the proper release setup script name, and create the script if necessary. This function also updates the cmd string (removes full asetup from payload command). Note: for stand-alone containers, the function will return /release_setup.sh and assume that this script exists in the container. The pilot will only create a my_release_setup.sh script for OS containers. In case the release setup is not present in an unpacked container, the function will reset the cmd string. :param cmd: Payload execution command (string). :param atlas_setup: asetup command (string). :param full_atlas_setup: full asetup command (string). :param release: software release, needed to determine Athena environment (string). :param imagename: container image name (string). :param workdir: job workdir (string). :param is_cvmfs: does the queue have cvmfs? (Boolean). :return: proper release setup name (string), updated cmd (string). """ release_setup_name = '/srv/my_release_setup.sh' # extracted_asetup should be written to 'my_release_setup.sh' and cmd to 'container_script.sh' content = 'echo \"INFO: sourcing %s inside the container. ' \ 'This should not run if it is a ATLAS standalone container\"' % release_setup_name if is_cvmfs and release and release != 'NULL': content, cmd = extract_full_atlas_setup(cmd, atlas_setup) if not content: content = full_atlas_setup content += '\nreturn $?' logger.debug( 'command to be written to release setup file:\n\n%s:\n\n%s\n' % (release_setup_name, content)) try: write_file(os.path.join(workdir, os.path.basename(release_setup_name)), content, mute=False) except Exception as e: logger.warning('exception caught: %s' % e) # reset cmd in case release_setup.sh does not exist in unpacked image (only for those containers) if imagename and release and release != 'NULL': cmd = cmd.replace(';;', ';') if is_release_setup( release_setup_name, imagename) else '' return release_setup_name, cmd
def write_output(filename, output): """ Write command output to file. :param filename: file name (string). :param output: command stdout/stderr (string). :return: """ try: write_file(filename, output, unique=True) except PilotException as error: logger.warning('failed to write utility output to file: %s, %s', error, output) else: logger.debug('wrote %s', filename)
def create_root_container_command(workdir, cmd): """ :param workdir: :param cmd: :return: """ command = 'cd %s;' % workdir content = get_root_container_script(cmd) script_name = 'open_file.sh' try: status = write_file(os.path.join(workdir, script_name), content) except PilotException as exc: raise exc else: if status: # generate the final container command x509 = os.environ.get('X509_USER_PROXY', '') if x509: command += 'export X509_USER_PROXY=%s;' % x509 command += 'export ALRB_CONT_RUNPAYLOAD=\"source /srv/%s\";' % script_name command += get_asetup( alrb=True ) # export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase; command += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c CentOS7' logger.debug('container command: %s', command) return command
def _generate_override_script(self, jupyter=False, servicetype='LoadBalancer'): """ Generate a values yaml script, unless it already exists. :param jupyter: False if jupyter notebook server should be disabled (Boolean). :param servicetype: name of service type (string). :return: """ filename = os.path.join(self._workdir, self.overrides) if os.path.exists(filename): logger.info('file \'%s\' already exists - will not override', filename) return script = "" if not jupyter: script += 'jupyter:\n enabled: false\n\n' if servicetype: script += 'scheduler:\n serviceType: \"%s\"\n' % servicetype if script: status = write_file(filename, script) if status: logger.debug('generated script: %s', filename) else: self.overrides = None
def create_setup_file(version, path): """ Create the DBRelease setup file. :param version: DBRelease version (string). :param path: path to local DBReleases (string). :return: Boolean (True if DBRelease setup file was successfully created). """ status = False # get the DBRelease directory _dir = get_dbrelease_dir() if _dir != "" and version != "": # create the python code string to be written to file txt = "import os\n" txt += "os.environ['DBRELEASE'] = '%s'\n" % version txt += "os.environ['DATAPATH'] = '%s/%s:' + os.environ['DATAPATH']\n" % (_dir, version) txt += "os.environ['DBRELEASE_REQUIRED'] = '%s'\n" % version txt += "os.environ['DBRELEASE_REQUESTED'] = '%s'\n" % version txt += "os.environ['CORAL_DBLOOKUP_PATH'] = '%s/%s/XMLConfig'\n" % (_dir, version) try: status = write_file(path, txt) except FileHandlingFailure as exc: logger.warning('failed to create DBRelease setup file: %s', exc) else: logger.info("Created setup file with the following content:.................................\n%s", txt) logger.info("...............................................................................") else: logger.warning('failed to create %s for DBRelease version=%s and directory=%s', path, version, _dir) return status
def create_middleware_container_command(workdir, cmd, container_options, label='stagein'): """ Create the stage-in/out container command. The function takes the isolated stage-in/out command, adds bits and pieces needed for the containerisation and stores it in a stage[in|out].sh script file. It then generates the actual command that will execute the stage-in/out script in a container. new cmd: lsetup rucio davis xrootd old cmd exit $? write new cmd to stage[in|out].sh script create container command and return it :param workdir: working directory where script will be stored (string). :param cmd: isolated stage-in/out command (string). :param container_options: container options from queuedata (string). :param label: 'stage-[in|out]' (string). :return: container command to be executed (string). """ command = 'cd %s;' % workdir # add bits and pieces for the containerisation middleware_container = get_middleware_container() content = get_middleware_container_script(middleware_container, cmd) # store it in setup.sh script_name = 'stagein.sh' if label == 'stage-in' else 'stageout.sh' try: status = write_file(os.path.join(workdir, script_name), content) except PilotException as e: raise e else: if status: # generate the final container command x509 = os.environ.get('X509_USER_PROXY', '') if x509: command += 'export X509_USER_PROXY=%s;' % x509 command += 'export ALRB_CONT_RUNPAYLOAD=\"source /srv/%s\";' % script_name command += get_asetup( alrb=True ) # export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase; command += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c %s' % middleware_container command += ' ' + get_container_options(container_options) command = command.replace(' ', ' ') logger.debug('container command: %s' % command) return command
def write_utility_output(self, workdir, step, stdout, stderr): """ Write the utility command output to stdout, stderr files to the job.workdir for the current step. -> <step>_stdout.txt, <step>_stderr.txt Example of step: preprocess, postprocess. :param workdir: job workdir (string). :param step: utility step (string). :param stdout: command stdout (string). :param stderr: command stderr (string). :return: """ # dump to file try: name_stdout = step + '_stdout.txt' name_stderr = step + '_stderr.txt' if step == 'preprocess': self.__preprocess_stdout_name = name_stdout self.__preprocess_stderr_name = name_stderr elif step == 'postprocess': self.__postprocess_stdout_name = name_stdout self.__postprocess_stderr_name = name_stderr name = os.path.join(workdir, step + '_stdout.txt') write_file(name, stdout, unique=True) except PilotException as error: logger.warning('failed to write utility stdout to file: %s, %s', error, stdout) else: logger.debug('wrote %s', name) try: name = os.path.join(workdir, step + '_stderr.txt') write_file(name, stderr, unique=True) except PilotException as error: logger.warning('failed to write utility stderr to file: %s, %s', error, stderr) else: logger.debug('wrote %s', name)
def get_payload_proxy(proxy_outfile_name, voms_role='atlas'): """ :param proxy_outfile_name: specify the file to store proxy :param voms_role: what proxy (role) to request. It should exist on Panda node :return: True on success """ try: # it assumes that https_setup() was done already url = os.environ.get('PANDA_SERVER_URL', config.Pilot.pandaserver) res = https.request( '{pandaserver}/server/panda/getProxy'.format(pandaserver=url), data={'role': voms_role}) if res is None: logger.error( "Unable to get proxy with role '%s' from panda server", voms_role) return False if res['StatusCode'] != 0: logger.error( "When get proxy with role '%s' panda server returned: %s", voms_role, res['errorDialog']) return False proxy_contents = res['userProxy'] except Exception as exc: logger.error("Get proxy from panda server failed: %s, %s", exc, traceback.format_exc()) return False res = False try: # pre-create empty proxy file with secure permissions. Prepare it for write_file() which can not # set file permission mode, it will writes to the existing file with correct permissions. _file = os.open(proxy_outfile_name, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) os.close(_file) res = write_file(proxy_outfile_name, proxy_contents, mute=False) # returns True on success except (IOError, OSError, FileHandlingFailure) as exc: logger.error( "Exception when try to save proxy to the file '%s': %s, %s", proxy_outfile_name, exc, traceback.format_exc()) return res
def containerise_middleware(job, xdata, queue, eventtype, localsite, remotesite, container_options, external_dir, label='stage-in', container_type='container'): """ Containerise the middleware by performing stage-in/out steps in a script that in turn can be run in a container. Note: a container will only be used for option container_type='container'. If this is 'bash', then stage-in/out will still be done by a script, but not containerised. Note: this function is tailor made for stage-in/out. :param job: job object. :param xdata: list of FileSpec objects. :param queue: queue name (string). :param eventtype: :param localsite: :param remotesite: :param container_options: container options from queuedata (string). :param external_dir: input or output files directory (string). :param label: optional 'stage-in/out' (String). :param container_type: optional 'container/bash' :raises StageInFailure: for stage-in failures :raises StageOutFailure: for stage-out failures :return: """ cwd = getcwd() # get the name of the stage-in/out isolation script script = config.Container.middleware_container_stagein_script if label == 'stage-in' else config.Container.middleware_container_stageout_script try: cmd = get_command(job, xdata, queue, script, eventtype, localsite, remotesite, external_dir, label=label, container_type=container_type) except PilotException as e: raise e if container_type == 'container': # add bits and pieces needed to run the cmd in a container pilot_user = environ.get('PILOT_USER', 'generic').lower() user = __import__('pilot.user.%s.container' % pilot_user, globals(), locals(), [pilot_user], 0) # Python 2/3 try: cmd = user.create_middleware_container_command(job.workdir, cmd, container_options, label=label) except PilotException as e: raise e else: logger.warning( '%s will not be done in a container (but it will be done by a script)', label) try: logger.info('*** executing %s (logging will be redirected) ***', label) exit_code, stdout, stderr = execute(cmd, job=job, usecontainer=False) except Exception as exc: logger.info('*** %s has failed ***', label) logger.warning('exception caught: %s', exc) else: if exit_code == 0: logger.info('*** %s has finished ***', label) else: logger.info('*** %s has failed ***', label) logger.warning('stderr:\n%s', stderr) logger.warning('stdout:\n%s', stdout) logger.debug('%s script returned exit_code=%d', label, exit_code) # write stdout+stderr to files try: _stdout_name, _stderr_name = get_logfile_names(label) write_file(path.join(job.workdir, _stdout_name), stdout, mute=False) write_file(path.join(job.workdir, _stderr_name), stderr, mute=False) except PilotException as exc: msg = 'exception caught: %s' % exc if label == 'stage-in': raise StageInFailure(msg) else: raise StageOutFailure(msg) # handle errors, file statuses, etc (the stage-in/out scripts write errors and file status to a json file) try: handle_updated_job_object(job, xdata, label=label) except PilotException as exc: raise exc
def alrb_wrapper(cmd, workdir, job=None): """ Wrap the given command with the special ALRB setup for containers E.g. cmd = /bin/bash hello_world.sh -> export thePlatform="x86_64-slc6-gcc48-opt" export ALRB_CONT_RUNPAYLOAD="cmd' setupATLAS -c $thePlatform :param cmd (string): command to be executed in a container. :param workdir: (not used) :param job: job object. :return: prepended command with singularity execution command (string). """ if not job: logger.warning( 'the ALRB wrapper did not get a job object - cannot proceed') return cmd queuedata = job.infosys.queuedata container_name = queuedata.container_type.get( "pilot") # resolve container name for user=pilot if container_name: # first get the full setup, which should be removed from cmd (or ALRB setup won't work) _asetup = get_asetup() # get_asetup() # -> export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase;source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh # --quiet;source $AtlasSetup/scripts/asetup.sh # atlas_setup = $AtlasSetup/scripts/asetup.sh # clean_asetup = export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase;source # ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh --quiet; atlas_setup, clean_asetup = extract_atlas_setup(_asetup, job.swrelease) full_atlas_setup = get_full_asetup( cmd, 'source ' + atlas_setup) if atlas_setup and clean_asetup else '' # do not include 'clean_asetup' in the container script if clean_asetup and full_atlas_setup: cmd = cmd.replace(clean_asetup, '') # for stand-alone containers, do not include the full atlas setup either if job.imagename: cmd = cmd.replace(full_atlas_setup, '') # get_asetup(asetup=False) # -> export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase;source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh --quiet; # get simplified ALRB setup (export) alrb_setup = get_asetup(alrb=True, add_if=True) # get_asetup(alrb=True) # -> export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase; # get_asetup(alrb=True, add_if=True) # -> if [ -z "$ATLAS_LOCAL_ROOT_BASE" ]; then export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase; fi; # add user proxy if necessary (actually it should also be removed from cmd) exit_code, diagnostics, alrb_setup, cmd = update_for_user_proxy( alrb_setup, cmd, is_analysis=job.is_analysis()) if exit_code: job.piloterrordiag = diagnostics job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( exit_code) # set the platform info alrb_setup = set_platform(job, alrb_setup) # add the jobid to be used as an identifier for the payload running inside the container # it is used to identify the pid for the process to be tracked by the memory monitor if 'export PandaID' not in alrb_setup: alrb_setup += "export PandaID=%s;" % job.jobid # add TMPDIR cmd = "export TMPDIR=/srv;export GFORTRAN_TMPDIR=/srv;" + cmd cmd = cmd.replace(';;', ';') # get the proper release setup script name, and create the script if necessary release_setup, cmd = create_release_setup(cmd, atlas_setup, full_atlas_setup, job.swrelease, job.imagename, job.workdir, queuedata.is_cvmfs) if not cmd: diagnostics = 'payload setup was reset due to missing release setup in unpacked container' logger.warning(diagnostics) job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( errors.MISSINGRELEASEUNPACKED) return "" # correct full payload command in case preprocess command are used (ie replace trf with setupATLAS -c ..) if job.preprocess and job.containeroptions: cmd = replace_last_command( cmd, job.containeroptions.get('containerExec')) logger.debug('updated cmd with containerExec: %s', cmd) # write the full payload command to a script file container_script = config.Container.container_script logger.debug( 'command to be written to container script file:\n\n%s:\n\n%s\n', container_script, cmd) try: write_file(os.path.join(job.workdir, container_script), cmd, mute=False) os.chmod(os.path.join(job.workdir, container_script), 0o755) # Python 2/3 # except (FileHandlingFailure, FileNotFoundError) as exc: # Python 3 except (FileHandlingFailure, OSError) as exc: # Python 2/3 logger.warning('exception caught: %s', exc) return "" # also store the command string in the job object job.command = cmd # add atlasLocalSetup command + options (overwrite the old cmd since the new cmd is the containerised version) cmd = add_asetup(job, alrb_setup, queuedata.is_cvmfs, release_setup, container_script, queuedata.container_options) # add any container options if set execargs = job.containeroptions.get('execArgs', None) if execargs: cmd += ' ' + execargs logger.debug('\n\nfinal command:\n\n%s\n', cmd) else: logger.warning('container name not defined in CRIC') return cmd
def alrb_wrapper(cmd, workdir, job=None): """ Wrap the given command with the special ALRB setup for containers E.g. cmd = /bin/bash hello_world.sh -> export thePlatform="x86_64-slc6-gcc48-opt" export ALRB_CONT_RUNPAYLOAD="cmd' setupATLAS -c $thePlatform :param cmd (string): command to be executed in a container. :param workdir: (not used) :param job: job object. :return: prepended command with singularity execution command (string). """ if not job: logger.warning('the ALRB wrapper did not get a job object - cannot proceed') return cmd log = get_logger(job.jobid) queuedata = job.infosys.queuedata container_name = queuedata.container_type.get("pilot") # resolve container name for user=pilot if container_name == 'singularity': # first get the full setup, which should be removed from cmd (or ALRB setup won't work) _asetup = get_asetup() cmd = cmd.replace(_asetup, "asetup ") # get simplified ALRB setup (export) asetup = get_asetup(alrb=True) # Get the singularity options singularity_options = queuedata.container_options log.debug( "resolved singularity_options from queuedata.container_options: %s" % singularity_options) _cmd = asetup # do not include the X509_USER_PROXY in the command the container will execute x509 = os.environ.get('X509_USER_PROXY') if x509 != "": cmd = cmd.replace("export X509_USER_PROXY=%s;" % x509, "") # add it instead to the container setup command _cmd = "export X509_USER_PROXY=%s;" % x509 + _cmd if job.alrbuserplatform: _cmd += 'export thePlatform=\"%s\";' % job.alrbuserplatform elif job.platform: _cmd += 'export thePlatform=\"%s\";' % job.platform #elif '--containerImage' in job.jobparams: # if job.alrbuserplatform: # _cmd += 'export thePlatform=\"%s\";' % job.alrbuserplatform # else: # # set a default platform for user defined containers # _cmd += 'export thePlatform=\"centos7\";' #if '--containall' not in singularity_options: # singularity_options += ' --containall' if singularity_options != "": _cmd += 'export ALRB_CONT_CMDOPTS=\"%s\";' % singularity_options else: # consider using options "-c -i -p" instead of "-C". The difference is that the latter blocks all environment # variables by default and the former does not _cmd += 'export ALRB_CONT_CMDOPTS=\"$ALRB_CONT_CMDOPTS -C\";' # add the jobid to be used as an identifier for the payload running inside the container _cmd += "export PANDAID=%s;" % job.jobid # add TMPDIR cmd = "export TMPDIR=/srv;export GFORTRAN_TMPDIR=/srv;" + cmd # write the full payload command to a script file script_file = config.Container.script_file status = write_file(os.path.join(job.workdir, script_file), cmd, mute=False) if status: script_cmd = '. /srv/' + script_file _cmd += "export ALRB_CONT_RUNPAYLOAD=\'%s\';" % script_cmd else: log.warning('attempting to quote command instead') _cmd += 'export ALRB_CONT_RUNPAYLOAD=%s;' % pipes.quote(cmd) # also store the command string in the job object job.command = cmd # this should not be necessary after the extract_container_image() in JobData update # containerImage should have been removed already if '--containerImage' in job.jobparams: job.jobparams, container_path = remove_container_string(job.jobparams) if job.alrbuserplatform: _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c %s' % job.alrbuserplatform elif container_path != "": _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c %s' % container_path #if not job.platform: # # add the default platform if not set by the job definition # _cmd += ' $thePlatform' else: log.warning('failed to extract container path from %s' % job.jobparams) _cmd = "" else: # _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh -c images' _cmd += 'source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh ' if job.platform or job.alrbuserplatform: # _cmd += '+$thePlatform' _cmd += '-c $thePlatform' _cmd = _cmd.replace(' ', ' ') cmd = _cmd log.info("Updated command: %s" % cmd) else: log.warning('container %s not supported' % container_name) return cmd