def _do_stageout(job, xdata, activity, title): """ Use the `StageOutClient` in the Data API to perform stage-out. :param job: job object. :param xdata: list of FileSpec objects. :param activity: copytool activity or preferred list of activities to resolve copytools :param title: type of stage-out (output, log) (string). :return: True in case of success transfers """ log = get_logger(job.jobid) log.info('prepare to stage-out %d %s file(s)' % (len(xdata), title)) event_type = "put_sm" #if log_transfer: # eventType += '_logs' #if special_log_transfer: # eventType += '_logs_os' if job.is_analysis(): event_type += "_a" rse = get_rse(xdata) localsite = remotesite = rse trace_report = TraceReport(pq=os.environ.get('PILOT_SITENAME', ''), localSite=localsite, remoteSite=remotesite, dataset="", eventType=event_type) trace_report.init(job) try: client = StageOutClient(job.infosys, logger=log, trace_report=trace_report) kwargs = dict(workdir=job.workdir, cwd=job.workdir, usecontainer=False, job=job) #, mode='stage-out') # prod analy unification: use destination preferences from PanDA server for unified queues if job.infosys.queuedata.type != 'unified': client.prepare_destinations(xdata, activity) ## FIX ME LATER: split activities: for astorages and for copytools (to unify with ES workflow) client.transfer(xdata, activity, **kwargs) except PilotException as error: import traceback error_msg = traceback.format_exc() log.error(error_msg) msg = errors.format_diagnostics(error.get_error_code(), error_msg) job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(error.get_error_code(), msg=msg) except Exception: import traceback log.error(traceback.format_exc()) # do not raise the exception since that will prevent also the log from being staged out # error = PilotException("stageOut failed with error=%s" % e, code=ErrorCodes.STAGEOUTFAILED) else: log.debug('stage-out client completed') log.info('summary of transferred files:') for e in xdata: if not e.status: status = "(not transferred)" else: status = e.status log.info(" -- lfn=%s, status_code=%s, status=%s" % (e.lfn, e.status_code, status)) remain_files = [e for e in xdata if e.status not in ['transferred']] log.debug('remain_files=%s' % str(remain_files)) log.debug('xdata=%s' % str(xdata)) return not remain_files
def create_trace_report(job, label='stage-in'): """ Create the trace report object. :param job: job object. :param label: 'stage-[in|out]' (string). :return: trace report object. """ event_type, localsite, remotesite = get_trace_report_variables(job, label=label) trace_report = TraceReport(pq=os.environ.get('PILOT_SITENAME', ''), localSite=localsite, remoteSite=remotesite, dataset="", eventType=event_type) trace_report.init(job) return trace_report
def __init__(self, infosys_instance=None, acopytools=None, logger=None, default_copytools='rucio', trace_report=None): """ If `acopytools` is not specified then it will be automatically resolved via infosys. In this case `infosys` requires initialization. :param acopytools: dict of copytool names per activity to be used for transfers. Accepts also list of names or string value without activity passed. :param logger: logging.Logger object to use for logging (None means no logging) :param default_copytools: copytool name(s) to be used in case of unknown activity passed. Accepts either list of names or single string value. """ super(StagingClient, self).__init__() if not logger: logger = logging.getLogger('%s.%s' % (__name__, 'null')) logger.disabled = True self.logger = logger self.infosys = infosys_instance or infosys if isinstance(acopytools, basestring): acopytools = {'default': [acopytools]} if acopytools else {} if isinstance(acopytools, (list, tuple)): acopytools = {'default': acopytools} if acopytools else {} self.acopytools = acopytools or {} if self.infosys.queuedata: if not self.acopytools: ## resolve from queuedata.acopytools using infosys self.acopytools = (self.infosys.queuedata.acopytools or {}).copy() if not self.acopytools: ## resolve from queuedata.copytools using infosys self.acopytools = dict( default=(self.infosys.queuedata.copytools or {}).keys()) if not self.acopytools.get('default'): if isinstance(default_copytools, basestring): default_copytools = [default_copytools ] if default_copytools else [] self.acopytools['default'] = default_copytools if not self.acopytools: msg = 'failed to initilize StagingClient: no acopytools options found, acopytools=%s' % self.acopytools logger.error(msg) self.trace_report.update(clientState='BAD_COPYTOOL', stateReason=msg) self.trace_report.send() raise PilotException("failed to resolve acopytools settings") logger.info('configured copytools per activity: acopytools=%s' % self.acopytools) # get an initialized trace report (has to be updated for get/put if not defined before) self.trace_report = trace_report if trace_report else TraceReport( pq=os.environ.get('PILOT_SITENAME', ''))
# scopes = file_list_dictionary.get('scopes') # filesizes = file_list_dictionary.get('filesizes') # checksums = file_list_dictionary.get('checksums') # allowlans = file_list_dictionary.get('allowlans') # allowwans = file_list_dictionary.get('allowwans') # directaccesslans = file_list_dictionary.get('directaccesslans') # directaccesswans = file_list_dictionary.get('directaccesswans') # istars = file_list_dictionary.get('istars') # accessmodes = file_list_dictionary.get('accessmodes') # storagetokens = file_list_dictionary.get('storagetokens') # guids = file_list_dictionary.get('guids') # generate the trace report trace_report = TraceReport(pq=os.environ.get('PILOT_SITENAME', ''), localSite=args.localsite, remoteSite=args.remotesite, dataset="", eventType=args.eventtype) job = Job(produserid=args.produserid, jobid=args.jobid, taskid=args.taskid, jobdefinitionid=args.jobdefinitionid) trace_report.init(job) try: infoservice = InfoService() infoservice.init(args.queuename, infosys.confinfo, infosys.extinfo) infosys.init( args.queuename ) # is this correct? otherwise infosys.queuedata doesn't get set except Exception as e:
def _stage_in(args, job): """ :return: True in case of success """ log = get_logger(job.jobid) # tested ok: #log.info('testing sending SIGUSR1') #import signal #os.kill(os.getpid(), signal.SIGUSR1) # write time stamps to pilot timing file add_to_pilot_timing(job.jobid, PILOT_PRE_STAGEIN, time.time(), args) # any DBRelease files should not be staged in for fspec in job.indata: if 'DBRelease' in fspec.lfn: fspec.status = 'no_transfer' event_type = "get_sm" #if log_transfer: # eventType += '_logs' #if special_log_transfer: # eventType += '_logs_os' if job.is_analysis(): event_type += "_a" rse = get_rse(job.indata) localsite = remotesite = rse trace_report = TraceReport(pq='', localSite=localsite, remoteSite=remotesite, dataset="", eventType=event_type) trace_report.init(job) # now that the trace report has been created, remove any files that are not to be transferred (DBRelease files) from the indata list toberemoved = [] for fspec in job.indata: if fspec.status == 'no_transfer': toberemoved.append(fspec) for fspec in toberemoved: logger.info('removing fspec object (lfn=%s) from list of input files' % fspec.lfn) job.indata.remove(fspec) try: if job.is_eventservicemerge: client = StageInESClient(job.infosys, logger=log, trace_report=trace_report) activity = 'es_events_read' else: client = StageInClient(job.infosys, logger=log, trace_report=trace_report) activity = 'pr' kwargs = dict(workdir=job.workdir, cwd=job.workdir, usecontainer=False, job=job) #, mode='stage-in') client.transfer(job.indata, activity=activity, **kwargs) except PilotException as error: log.error('PilotException caught: %s' % error) job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(error.get_error_code()) except Exception as error: log.error('failed to stage-in: error=%s' % error) log.info('summary of transferred files:') for e in job.indata: if not e.status: status = "(not transferred)" else: status = e.status log.info(" -- lfn=%s, status_code=%s, status=%s" % (e.lfn, e.status_code, status)) # write time stamps to pilot timing file add_to_pilot_timing(job.jobid, PILOT_POST_STAGEIN, time.time(), args) remain_files = [e for e in job.indata if e.status not in ['remote_io', 'transferred', 'no_transfer']] if not remain_files: log.info("stage-in finished") else: log.info("stage-in failed") return not remain_files
def _stage_in(args, job): """ :return: True in case of success """ log = get_logger(job.jobid) # tested ok: #log.info('testing sending SIGUSR1') #import signal #os.kill(os.getpid(), signal.SIGUSR1) # write time stamps to pilot timing file add_to_pilot_timing(job.jobid, PILOT_PRE_STAGEIN, time.time(), args) # any DBRelease files should not be staged in for fspec in job.indata: if 'DBRelease' in fspec.lfn: fspec.status = 'no_transfer' event_type = "get_sm" if job.is_analysis(): event_type += "_a" rse = get_rse(job.indata) localsite = remotesite = rse trace_report = TraceReport(pq=os.environ.get('PILOT_SITENAME', ''), localSite=localsite, remoteSite=remotesite, dataset="", eventType=event_type) trace_report.init(job) # now that the trace report has been created, remove any files that are not to be transferred (DBRelease files) from the indata list toberemoved = [] for fspec in job.indata: if fspec.status == 'no_transfer': toberemoved.append(fspec) for fspec in toberemoved: logger.info('removing fspec object (lfn=%s) from list of input files' % fspec.lfn) job.indata.remove(fspec) ########### bulk transfer test # THE FOLLOWING WORKS BUT THERE IS AN ISSUE WITH TRACES, CHECK STAGEIN SCRIPT IF STORED CORRECTLY #filename = 'initial_trace_report.json' #tpath = os.path.join(job.workdir, filename) #write_json(tpath, trace_report) #lfns, scopes = get_filedata_strings(job.indata) #script = 'stagein.py' #srcdir = os.environ.get('PILOT_SOURCE_DIR') #scriptpath = os.path.join(os.path.join(srcdir, 'pilot/scripts'), script) #copy(scriptpath, srcdir) #cmd = 'python %s --lfns=%s --scopes=%s --tracereportname=%s -w %s -d -q %s' %\ # (os.path.join(srcdir, script), lfns, scopes, tpath, job.workdir, args.queue) #logger.debug('could have executed: %s' % script) #exit_code, stdout, stderr = execute(cmd, mode='python') #logger.debug('exit_code=%d' % exit_code) #logger.debug('stdout=%s' % stdout) #logger.debug('stderr=%s' % stderr) ########### bulk transfer test try: if job.is_eventservicemerge: client = StageInESClient(job.infosys, logger=log, trace_report=trace_report) activity = 'es_events_read' else: client = StageInClient(job.infosys, logger=log, trace_report=trace_report) activity = 'pr' kwargs = dict(workdir=job.workdir, cwd=job.workdir, usecontainer=False, job=job, use_bulk=False) client.prepare_sources(job.indata) client.transfer(job.indata, activity=activity, **kwargs) except PilotException as error: import traceback error_msg = traceback.format_exc() log.error(error_msg) msg = errors.format_diagnostics(error.get_error_code(), error_msg) job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( error.get_error_code(), msg=msg) except Exception as error: log.error('failed to stage-in: error=%s' % error) log.info('summary of transferred files:') for e in job.indata: status = e.status if e.status else "(not transferred)" log.info(" -- lfn=%s, status_code=%s, status=%s" % (e.lfn, e.status_code, status)) # write time stamps to pilot timing file add_to_pilot_timing(job.jobid, PILOT_POST_STAGEIN, time.time(), args) remain_files = [ e for e in job.indata if e.status not in ['remote_io', 'transferred', 'no_transfer'] ] if not remain_files: log.info("stage-in finished") else: log.info("stage-in failed") return not remain_files
def test_copy_out_rucio(self): from pilot.copytool.rucio import copy_out trace_report = TraceReport() trace_report.update(eventType='unit test') copy_out(self.outdata, trace_report=trace_report) os.remove(self.outdata[0].pfn)