args.queuename ) # is this correct? otherwise infosys.queuedata doesn't get set except Exception as e: message(e) # perform stage-in (single transfers) err = "" errcode = 0 if args.eventservicemerge: client = StageInESClient(infoservice, logger=logger, trace_report=trace_report) activity = 'es_events_read' else: client = StageInClient(infoservice, logger=logger, trace_report=trace_report) activity = 'pr' kwargs = dict(workdir=args.workdir, cwd=args.workdir, usecontainer=False, use_pcache=args.usepcache, use_bulk=False, use_vp=args.usevp, input_dir=args.inputdir, catchall=args.catchall) xfiles = [] for lfn in replica_dictionary: files = [{ 'scope': replica_dictionary[lfn]['scope'],
def _stage_in(args, job): """ :return: True in case of success """ log = get_logger(job.jobid) # tested ok: #log.info('testing sending SIGUSR1') #import signal #os.kill(os.getpid(), signal.SIGUSR1) # write time stamps to pilot timing file add_to_pilot_timing(job.jobid, PILOT_PRE_STAGEIN, time.time(), args) # any DBRelease files should not be staged in for fspec in job.indata: if 'DBRelease' in fspec.lfn: fspec.status = 'no_transfer' event_type = "get_sm" #if log_transfer: # eventType += '_logs' #if special_log_transfer: # eventType += '_logs_os' if job.is_analysis(): event_type += "_a" rse = get_rse(job.indata) localsite = remotesite = rse trace_report = TraceReport(pq='', localSite=localsite, remoteSite=remotesite, dataset="", eventType=event_type) trace_report.init(job) # now that the trace report has been created, remove any files that are not to be transferred (DBRelease files) from the indata list toberemoved = [] for fspec in job.indata: if fspec.status == 'no_transfer': toberemoved.append(fspec) for fspec in toberemoved: logger.info('removing fspec object (lfn=%s) from list of input files' % fspec.lfn) job.indata.remove(fspec) try: if job.is_eventservicemerge: client = StageInESClient(job.infosys, logger=log, trace_report=trace_report) activity = 'es_events_read' else: client = StageInClient(job.infosys, logger=log, trace_report=trace_report) activity = 'pr' kwargs = dict(workdir=job.workdir, cwd=job.workdir, usecontainer=False, job=job) #, mode='stage-in') client.transfer(job.indata, activity=activity, **kwargs) except PilotException as error: log.error('PilotException caught: %s' % error) job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(error.get_error_code()) except Exception as error: log.error('failed to stage-in: error=%s' % error) log.info('summary of transferred files:') for e in job.indata: if not e.status: status = "(not transferred)" else: status = e.status log.info(" -- lfn=%s, status_code=%s, status=%s" % (e.lfn, e.status_code, status)) # write time stamps to pilot timing file add_to_pilot_timing(job.jobid, PILOT_POST_STAGEIN, time.time(), args) remain_files = [e for e in job.indata if e.status not in ['remote_io', 'transferred', 'no_transfer']] if not remain_files: log.info("stage-in finished") else: log.info("stage-in failed") return not remain_files
def _stage_in(args, job): """ :return: True in case of success """ # tested ok: #logger.info('testing sending SIGUSR1') #import signal #os.kill(os.getpid(), signal.SIGUSR1) # write time stamps to pilot timing file add_to_pilot_timing(job.jobid, PILOT_PRE_STAGEIN, time.time(), args) # any DBRelease files should not be staged in skip_special_files(job) # now that the trace report has been created, remove any files that are not to be transferred (DBRelease files) from the indata list update_indata(job) label = 'stage-in' # should stage-in be done by a script (for containerisation) or by invoking the API (ie classic mode)? use_container = pilot.util.middleware.use_middleware_script( job.infosys.queuedata.container_type.get("middleware")) if use_container: logger.info('stage-in will be done by a script') try: eventtype, localsite, remotesite = get_trace_report_variables( job, label=label) pilot.util.middleware.containerise_middleware( job, job.indata, args.queue, eventtype, localsite, remotesite, job.infosys.queuedata.container_options, args.input_dir, label=label, container_type=job.infosys.queuedata.container_type.get( "middleware")) except PilotException as error: logger.warning( 'stage-in containerisation threw a pilot exception: %s', error) except Exception as error: import traceback logger.warning('stage-in containerisation threw an exception: %s', error) logger.error(traceback.format_exc()) else: try: logger.info('stage-in will not be done in a container') # create the trace report trace_report = create_trace_report(job, label=label) if job.is_eventservicemerge: client = StageInESClient(job.infosys, logger=logger, trace_report=trace_report) activity = 'es_events_read' else: client = StageInClient(job.infosys, logger=logger, trace_report=trace_report) activity = 'pr' use_pcache = job.infosys.queuedata.use_pcache kwargs = dict(workdir=job.workdir, cwd=job.workdir, usecontainer=False, use_pcache=use_pcache, use_bulk=False, input_dir=args.input_dir, use_vp=job.use_vp, catchall=job.infosys.queuedata.catchall) client.prepare_sources(job.indata) client.transfer(job.indata, activity=activity, **kwargs) except PilotException as error: import traceback error_msg = traceback.format_exc() logger.error(error_msg) msg = errors.format_diagnostics(error.get_error_code(), error_msg) job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( error.get_error_code(), msg=msg) except Exception as error: logger.error('failed to stage-in: error=%s', error) logger.info('summary of transferred files:') for infile in job.indata: status = infile.status if infile.status else "(not transferred)" logger.info(" -- lfn=%s, status_code=%s, status=%s", infile.lfn, infile.status_code, status) # write time stamps to pilot timing file add_to_pilot_timing(job.jobid, PILOT_POST_STAGEIN, time.time(), args) remain_files = [ infile for infile in job.indata if infile.status not in ['remote_io', 'transferred', 'no_transfer'] ] logger.info("stage-in finished") if not remain_files else logger.info( "stage-in failed") return not remain_files
def _stage_in(args, job): """ :return: True in case of success """ log = get_logger(job.jobid) # tested ok: #log.info('testing sending SIGUSR1') #import signal #os.kill(os.getpid(), signal.SIGUSR1) # write time stamps to pilot timing file add_to_pilot_timing(job.jobid, PILOT_PRE_STAGEIN, time.time(), args) # any DBRelease files should not be staged in for fspec in job.indata: if 'DBRelease' in fspec.lfn: fspec.status = 'no_transfer' event_type = "get_sm" if job.is_analysis(): event_type += "_a" rse = get_rse(job.indata) localsite = remotesite = rse trace_report = TraceReport(pq=os.environ.get('PILOT_SITENAME', ''), localSite=localsite, remoteSite=remotesite, dataset="", eventType=event_type) trace_report.init(job) # now that the trace report has been created, remove any files that are not to be transferred (DBRelease files) from the indata list toberemoved = [] for fspec in job.indata: if fspec.status == 'no_transfer': toberemoved.append(fspec) for fspec in toberemoved: logger.info('removing fspec object (lfn=%s) from list of input files' % fspec.lfn) job.indata.remove(fspec) ########### bulk transfer test # THE FOLLOWING WORKS BUT THERE IS AN ISSUE WITH TRACES, CHECK STAGEIN SCRIPT IF STORED CORRECTLY #filename = 'initial_trace_report.json' #tpath = os.path.join(job.workdir, filename) #write_json(tpath, trace_report) #lfns, scopes = get_filedata_strings(job.indata) #script = 'stagein.py' #srcdir = os.environ.get('PILOT_SOURCE_DIR') #scriptpath = os.path.join(os.path.join(srcdir, 'pilot/scripts'), script) #copy(scriptpath, srcdir) #cmd = 'python %s --lfns=%s --scopes=%s --tracereportname=%s -w %s -d -q %s' %\ # (os.path.join(srcdir, script), lfns, scopes, tpath, job.workdir, args.queue) #logger.debug('could have executed: %s' % script) #exit_code, stdout, stderr = execute(cmd, mode='python') #logger.debug('exit_code=%d' % exit_code) #logger.debug('stdout=%s' % stdout) #logger.debug('stderr=%s' % stderr) ########### bulk transfer test try: if job.is_eventservicemerge: client = StageInESClient(job.infosys, logger=log, trace_report=trace_report) activity = 'es_events_read' else: client = StageInClient(job.infosys, logger=log, trace_report=trace_report) activity = 'pr' kwargs = dict(workdir=job.workdir, cwd=job.workdir, usecontainer=False, job=job, use_bulk=False) client.prepare_sources(job.indata) client.transfer(job.indata, activity=activity, **kwargs) except PilotException as error: import traceback error_msg = traceback.format_exc() log.error(error_msg) msg = errors.format_diagnostics(error.get_error_code(), error_msg) job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( error.get_error_code(), msg=msg) except Exception as error: log.error('failed to stage-in: error=%s' % error) log.info('summary of transferred files:') for e in job.indata: status = e.status if e.status else "(not transferred)" log.info(" -- lfn=%s, status_code=%s, status=%s" % (e.lfn, e.status_code, status)) # write time stamps to pilot timing file add_to_pilot_timing(job.jobid, PILOT_POST_STAGEIN, time.time(), args) remain_files = [ e for e in job.indata if e.status not in ['remote_io', 'transferred', 'no_transfer'] ] if not remain_files: log.info("stage-in finished") else: log.info("stage-in failed") return not remain_files