def ganga_job_submitted(application_name, backend_name, plain_job, master_job, sub_jobs): host = getConfig('System')['GANGA_HOSTNAME'] user = getConfig('Configuration')['user'] runtime_packages = ':'.join(map(os.path.basename, filter(lambda x: x, config['RUNTIME_PATH'].split(':')))) start = long(time.time() * 1000) job_submitted_message = {'application': application_name, 'backend': backend_name, 'user': user, 'host': host, 'start': start, 'plain_job': plain_job, 'master_job': master_job, 'sub_jobs': sub_jobs, 'runtime_packages': runtime_packages} if config['UsageMonitoringMSG']: from Ganga.GPIDev.MonitoringServices import MSGUtil msg_config = getConfig('MSGMS') p = MSGUtil.createPublisher( msg_config['server'], msg_config['port'], msg_config['username'], msg_config['password']) # start publisher thread and enqueue usage message for sending p.start() p.send(msg_config['job_submission_message_destination'], repr(job_submitted_message), {'persistent': 'true'}) # p.send('/queue/test.ganga.jobsubmission',repr(job_submitted_message),{'persistent':'true'}) # ask publisher thread to stop. it will send queued message anyway. p.stop() p._finalize(10.) if hasattr(p, 'unregister'): p.unregister() del p
def ganga_started(session_type, **extended_attributes): host = getConfig('System')['GANGA_HOSTNAME'] version = getConfig('System')['GANGA_VERSION'] user = getConfig('Configuration')['user'] runtime_packages = ':'.join( map(os.path.basename, filter(lambda x: x, config['RUNTIME_PATH'].split(':')))) start = long(time.time() * 1000) usage_message = {'user': user, 'host': host, 'start': start, 'session': session_type, 'runtime_packages': runtime_packages, 'version': version} usage_message.update(extended_attributes) if config['UsageMonitoringMSG']: from Ganga.GPIDev.MonitoringServices import MSGUtil msg_config = getConfig('MSGMS') p = MSGUtil.createPublisher( msg_config['server'], msg_config['port'], msg_config['username'], msg_config['password']) # start publisher thread and enqueue usage message for sending p.start() p.send(msg_config['usage_message_destination'], repr(usage_message), {'persistent': 'true'}) # ask publisher thread to stop. it will send queued message anyway. p.stop() p._finalize(10.) if hasattr(p, 'unregister'): p.unregister() del p
def ganga_started(session_type,**extended_attributes): host = getConfig('System')['GANGA_HOSTNAME'] version = getConfig('System')['GANGA_VERSION'] user = getConfig('Configuration')['user'] runtime_packages = ':'.join(map(os.path.basename,filter(lambda x:x, config['RUNTIME_PATH'].split(':')))) start = long(time.time()*1000) usage_message = {'user':user,'host':host,'start':start,'session':session_type,'runtime_packages':runtime_packages,'version':version} usage_message.update(extended_attributes) if config['UsageMonitoringURL']: import ApMon.apmon global monitor # the ApMon constructor may start background threads to refresh the configuration from URL # NOTE: the configuration (including defaultLogLevel) is overriden from the config file specified in URL monitor = ApMon.apmon.ApMon(config['UsageMonitoringURL'], defaultLogLevel=ApMon.apmon.Logger.FATAL) monitor.sendParameters('GangaUsage','%s@%s_%s'%(user,host,start),usage_message) # stop any background threads started by the ApMon constructor monitor.free() if config['UsageMonitoringMSG']: from Ganga.Lib.MonitoringServices.MSGMS import MSGUtil msg_config = getConfig('MSGMS') p = MSGUtil.createPublisher( msg_config['server'], msg_config['port'], msg_config['username'], msg_config['password']) # start publisher thread and enqueue usage message for sending p.start() p.send(msg_config['usage_message_destination'],repr(usage_message),{'persistent':'true'}) # ask publisher thread to stop. it will send queued message anyway. p.stop()
def prepare(self, force=False): super(Ostap, self).prepare(force) self._check_inputs() share_dir = os.path.join ( expandfilename ( getConfig('Configuration')['gangadir'] ) , 'shared' , getConfig('Configuration')['user'] , self.is_prepared.name ) input_sandbox_tar = os.path.join ( share_dir , 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name ) input_sandbox_tgz = os.path.join ( share_dir , 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name ) fillPackedSandbox ( self.scripts , input_sandbox_tar ) gzipFile ( input_sandbox_tar , input_sandbox_tgz , True ) # add the newly created shared directory into the metadata system if # the app is associated with a persisted object self.checkPreparedHasParent(self) self.post_prepare() logger.debug("Finished Preparing Application in %s" % share_dir)
def getDiracEnv(sourceFile = None): """ Returns the dirac environment stored in a global dictionary by Ganga. Once loaded and stored this is used for executing all DIRAC code in future Args: sourceFile (str): This is an optional file path which points to the env which should be sourced for this DIRAC """ global DIRAC_ENV with Dirac_Env_Lock: if sourceFile is None: sourceFile = 'default' cache_file = getConfig('DIRAC')['DiracEnvJSON'] source_command = getConfig('DIRAC')['DiracEnvSource'] if not cache_file and not source_command: source_command = getConfig('DIRAC')['DiracEnvFile'] else: # Needed for backwards compatibility with old configs... cache_file = None source_command = sourceFile if sourceFile not in DIRAC_ENV: if cache_file: DIRAC_ENV[sourceFile] = read_env_cache(cache_file) elif source_command: DIRAC_ENV[sourceFile] = get_env(source_command) else: logger.error("'DiracEnvSource' config variable empty") logger.error("%s %s" % (getConfig('DIRAC')['DiracEnvJSON'], getConfig('DIRAC')['DiracEnvSource'])) return DIRAC_ENV[sourceFile]
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.gLite import gLiteJobConfig prepared_exe = app.exe if app.is_prepared is not None: shared_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) if isinstance(app.exe, str): # we have a file. is it an absolute path? if os.path.abspath(app.exe) == app.exe: logger.info("Submitting a prepared application; taking any input files from %s" % ( app.is_prepared.name)) prepared_exe = File(os.path.join(os.path.join( shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name))) # else assume it's a system binary, so we don't need to # transport anything to the sharedir else: prepared_exe = app.exe elif isinstance(app.exe, File): logger.info("Submitting a prepared application; taking any input files from %s" % ( app.is_prepared.name)) prepared_exe = File(os.path.join(os.path.join( shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name))) return gLiteJobConfig(prepared_exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env)
def __init__(self, **kwargs): """ Construct a voms proxy requirement and assign the default vo from the config if none has been provided """ super(VomsProxy, self).__init__(**kwargs) if 'vo' not in kwargs and getConfig('LCG')['VirtualOrganisation']: self.vo = getConfig('LCG')['VirtualOrganisation']
def tearDown(self): wipe_temp_dir() from Ganga.Utility.Config import getConfig getConfig('Configuration').revertToDefault('gangadir') super(Savannah9638, self).tearDown()
def ganga_started(session_type, **extended_attributes): host = getConfig('System')['GANGA_HOSTNAME'] version = getConfig('System')['GANGA_VERSION'] user = getConfig('Configuration')['user'] runtime_packages = ':'.join( map(os.path.basename, filter(lambda x: x, config['RUNTIME_PATH'].split(':')))) start = long(time.time() * 1000) usage_message = {'user': user, 'host': host, 'start': start, 'session': session_type, 'runtime_packages': runtime_packages, 'version': version} usage_message.update(extended_attributes) #if config['UsageMonitoringURL']: # from Ganga.GPI import queues # # Lets move the actual monitoring out of the main thread for some performance # msg = '%s@%s_%s' % (user, host, start) # queues.add( _setupMonitor, (config['UsageMonitoringURL'], msg, usage_message) ) if config['UsageMonitoringMSG']: from Ganga.Lib.MonitoringServices.MSGMS import MSGUtil msg_config = getConfig('MSGMS') p = MSGUtil.createPublisher( msg_config['server'], msg_config['port'], msg_config['username'], msg_config['password']) # start publisher thread and enqueue usage message for sending p.start() p.send(msg_config['usage_message_destination'], repr( usage_message), {'persistent': 'true'}) # ask publisher thread to stop. it will send queued message anyway. p.stop()
def getDiracEnv(force=False): """ Returns the dirac environment stored in a global dictionary by Ganga. This is expected to be stored as some form of 'source'-able file on disk which can be used to get the printenv after sourcing Once loaded and stored his is used for executing all DIRAC code in future Args: force (bool): This triggers a compulsory reload of the env from disk """ global DIRAC_ENV with Dirac_Env_Lock: if DIRAC_ENV == {} or force: config_file = getConfig('DIRAC')['DiracEnvFile'] if not os.path.exists(config_file): absolute_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../..', config_file) else: absolute_path = config_file if getConfig('DIRAC')['DiracEnvFile'] != "" and os.path.exists(absolute_path): env_dict = get_env_from_arg(this_arg = absolute_path, def_env_on_fail = False) if env_dict is not None: DIRAC_ENV = env_dict else: logger.error("Error determining DIRAC environment") raise GangaException("Error determining DIRAC environment") else: logger.error("'DiracEnvFile' config variable empty or file not present") logger.error("Tried looking in : '%s' Please check your config" % absolute_path) logger.debug("Dirac Env: %s" % DIRAC_ENV) return DIRAC_ENV
def test_voms_proxy_life_cycle(gpi): from Ganga.GPI import VomsProxy, credential_store assert len(credential_store) == 0 cred = credential_store.create(VomsProxy()) assert cred.is_valid() assert len(credential_store) == 1 assert os.path.isfile(cred.location) assert cred.vo == getConfig("LCG")["VirtualOrganisation"] cred.destroy() assert not cred.is_valid() credential_store.clear() assert len(credential_store) == 0 with pytest.raises(KeyError): _ = credential_store[VomsProxy()] default_cred = credential_store.create(VomsProxy()) explicit_default_cred = credential_store.create(VomsProxy(vo=getConfig("LCG")["VirtualOrganisation"])) assert explicit_default_cred == default_cred assert len(credential_store) == 1
def getJobInfo(self): """Create job_info from Job object.""" j = self.job_info # called on client, so job_info is Job object from Ganga.Utility.Config import getConfig batch = getConfig('Configuration')['Batch'] host = getConfig('System')['GANGA_HOSTNAME'] jj = self.job_info if jj.master: jj = jj.master ji = { 'fqid': j.fqid, 'EXECUTION_BACKEND': self.dynamic_util.cl_execution_backend(j), 'OWNERDN': self.dynamic_util.cl_ownerdn(), 'JOB_ID_INSIDE_THE_TASK': self.dynamic_util.cl_job_id_inside_the_task(j), 'TASKNAME': self.dynamic_util.cl_task_name(j), 'UNIQUEJOBID': self.dynamic_util.cl_unique_job_id(j), 'BACKEND': self.job_info.backend.__class__.__name__, 'BATCH': batch, 'GANGA_HOSTNAME': host, 'JOB_UUID': jj.info.uuid, 'JOB_NAME': jj.name } return ji
def stop_ganga(): from Ganga.Utility.logging import getLogger logger = getLogger() logger.info("Deciding how to shutdown") # Do we want to empty the repository on shutdown? from Ganga.Utility.Config import getConfig if 'AutoCleanup' in getConfig('TestingFramework'): whole_cleanup = getConfig('TestingFramework')['AutoCleanup'] else: whole_cleanup = True logger.info("AutoCleanup: %s" % whole_cleanup) if whole_cleanup is True: # empty repository so we start again at job 0 when we restart logger.info("Clearing the Job and Template repositories") from Ganga.GPI import jobs, templates for j in jobs: try: j.remove() except: pass for t in templates: try: t.remove() except: pass if hasattr(jobs, 'clean'): jobs.clean(confirm=True, force=True) if hasattr(templates, 'clean'): templates.clean(confirm=True, force=True) logger.info("Shutting Down Internal Services") # Disable internal services such as monitoring and other tasks #from Ganga.Core.InternalServices import Coordinator # if Coordinator.servicesEnabled: # Coordinator.disableInternalServices() # Coordinator.servicesEnabled = False logger.info("Mimicking ganga exit") from Ganga.Core.InternalServices import ShutdownManager import Ganga.Core Ganga.Core.change_atexitPolicy(interactive_session=False, new_policy='batch') # This should now be safe ShutdownManager._ganga_run_exitfuncs() # Undo any manual editing of the config and revert to defaults from Ganga.Utility.Config import allConfigs for package in allConfigs.values(): package.revertToDefaultOptions() # Finished logger.info("Test Finished")
def dirac_inputdata(app): job = stripProxy(app).getJobObject() input_data = None parametricinput_data = None inputLFNs = [] if hasattr(job.inputdata, "getLFNs"): inputLFNs = job.inputdata.getLFNs() if job.master: logger.debug("job.master.inputdata: %s " % str(job.master.inputdata)) logger.debug("job.inputdata: %s" % str(job.inputdata)) if hasattr(job.inputdata, "getLFNs"): logger.debug("getLFNs(): %s" % job.inputdata.getLFNs()) has_input_DiracFile = False for this_file in job.inputfiles: if isType(this_file, DiracFile): has_input_DiracFile = True break if job.master and not has_input_DiracFile: for this_file in job.master.inputfiles: if isType(this_file, DiracFile): has_input_DiracFile = True break if len(inputLFNs) > 0: # master job with a splitter reaching prepare, hence bulk submit if not job.master and job.splitter: parametricinput_data = dirac_parametric_split(app) if parametricinput_data is not None and len(parametricinput_data) > getConfig("DIRAC")["MaxDiracBulkJobs"]: raise BackendError( "Dirac", "Number of bulk submission jobs '%s' exceeds the maximum allowed '%s' if more are needed please modify your config. Note there is a hard limit in Dirac of currently 1000." % (len(parametricinput_data), getConfig("DIRAC")["MaxDiracBulkJobs"]), ) # master job with no splitter or subjob already split proceed as normal else: input_data = job.inputdata.getLFNs() elif "Destination" not in job.backend.settings and not has_input_DiracFile: ##THIS IS NOT VERY DIRAC CENTRIC ##PLEASE WHEN TIME MOVE TO LHCBDIRAC where T1 is more applicable rcurrie ##Also editing the settings on the fly is asking for potential problems, should avoid t1_sites = getConfig("DIRAC")["noInputDataBannedSites"] logger.info("Job has no inputdata (T1 sites will be banned to help avoid overloading them).") if "BannedSites" in job.backend.settings: job.backend.settings["BannedSites"].extend(t1_sites) job.backend.settings["BannedSites"] = unique(job.backend.settings["BannedSites"]) else: job.backend.settings["BannedSites"] = t1_sites[:] # import traceback # traceback.print_stack() return input_data, parametricinput_data
def split(self, job): """ Method to do the splitting work Args: job (Job): master job to be used as a template to split subjobs """ from GangaLHCb.Lib.Applications import Gauss if not isinstance(job.application, (Gauss, GaudiExec)): logger.warning("This application is of type: '%s', be careful how you use it with the GaussSplitter!" % type(job.application)) subjobs = [] inputdata = job.inputdata if not isinstance(job.application, GaudiExec): # I'm assuming this file is created by the Gauss Application at some stage? if not job.inputdata: share_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], job.application.is_prepared.name, 'inputdata', 'options_data.pkl') if os.path.exists(share_path): f = open(share_path, 'r+b') #FIXME should this have been an addition? inputdata = pickle.load(f) f.close() for i in range(self.numberOfJobs): j = self._create_subjob(job, inputdata) # FIXME this starts from the 1st event and not zero, is it clear why? first = self.firstEventNumber + i * self.eventsPerJob + 1 opts = 'from Configurables import GenInit \n' if isinstance(job.application, Gauss): opts += 'from Gaudi.Configuration import * \n' opts += 'ApplicationMgr().EvtMax = %d\n' % self.eventsPerJob else: opts += 'from Configurables import LHCbApp \n' opts += 'LHCbApp().EvtMax = %d\n' % self.eventsPerJob opts += 'GenInit("GaussGen").FirstEventNumber = %d\n' % first spillOver = ["GaussGenPrev", "GaussGenPrevPrev", "GaussGenNext"] for s in spillOver: opts += 'GenInit("%s").FirstEventNumber = %d\n' % (s, first) #j.application.extra.input_buffers['data.py'] += opts if isinstance(job.application, GaudiExec): j.application.extraOpts = j.application.extraOpts + '\n' + opts else: j._splitter_data = opts # j.inputsandbox.append(File(FileBuffer(path,opts).create().name)) logger.debug("Creating job %d w/ FirstEventNumber = %d" % (i, first)) subjobs.append(j) return subjobs
def repairJobRepository(jobId): """ Repairs job repository for the comment attribute (migration from Comment object to string) """ def repairFilePath(filePath): fileRead = open(filePath, 'r') index = -1 found = False lines = fileRead.readlines() fileRead.close() for line in lines: index += 1 if line.find('<class name="Comment"') > -1: found = True break if found: newLines = [] for i in range(index): newLines.append(lines[i]) newLines.append(lines[index+1].replace('<attribute name="comment"> ', '')) for i in range(index+4, len(lines)): newLines.append(lines[i]) fileWrite = open(filePath, 'w') fileWrite.write(''.join(newLines)) fileWrite.close() from Ganga.Utility.Config import getConfig from Ganga.Utility.files import expandfilename import os if not isinstance(jobId,int): return repositoryPath = "repository/$usr/LocalXML/6.0/jobs/$thousandsNumxxx" repositoryPath = repositoryPath.replace('$usr', getConfig('Configuration')['user']) repositoryPath = repositoryPath.replace('$thousandsNum', str(jobId/1000)) repositoryFullPath = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), repositoryPath, str(jobId)) #repair also the subjobs data files for subjobDir in os.listdir(repositoryFullPath): repositorySubJobFullPath = os.path.join(repositoryFullPath, subjobDir) if os.path.isdir(repositorySubJobFullPath): repairFilePath(os.path.join(repositorySubJobFullPath, 'data')) repairFilePath(os.path.join(repositoryFullPath, 'data'))
def getGridStorage( self, gridhome = "" ): vo = getConfig( "LCG" )[ "VirtualOrganisation" ] if not gridhome: gridhome = self.gridhome job = self._getParent() id = job.id username = getConfig( "Configuration" )[ "user" ] userletter = username[ 0 ] gridStorage = os.path.join\ ( gridhome, "user", userletter, username, "ganga", str( id ) ) return gridStorage
def get_share_path(app=None): if app is None or app == "": return os.path.join( expandfilename(getConfig("Configuration")["gangadir"]), "shared", getConfig("Configuration")["user"] ) return os.path.join( expandfilename(getConfig("Configuration")["gangadir"]), "shared", getConfig("Configuration")["user"], app.is_prepared.name, )
def load_config_files(): """ Load the config files as a normal Ganga session would, taking into account environment variables etc. """ from Ganga.Utility.Config import getConfig, setSessionValuesFromFiles from Ganga.Runtime import GangaProgram system_vars = {} for opt in getConfig('System'): system_vars[opt] = getConfig('System')[opt] config_files = GangaProgram.get_config_files(os.path.expanduser('~/.gangarc')) setSessionValuesFromFiles(config_files, system_vars)
def test_GaudiPython_prepare(self): g = self.job.application g.prepare() assert g.is_prepared is not None, 'is_prepared not correctly set' g.is_prepared.ls() share_path = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], g.is_prepared.name, 'inputsandbox') os.system( 'cd ' + share_path + ';tar -xzvf _input_sandbox_' + g.is_prepared.name + '.tgz') assert os.path.exists(os.path.join(share_path, 'dummy.script'))
def test_optionsString(self): if not getConfig('Output')['ForbidLegacyInput']: ds = make_dataset(['lfn:a', 'lfn:b', 'pfn:c']) else: ds = make_dataset(['lfn:a', 'lfn:b', 'c']) this_str = ds.optionsString() print("---\n%s\n---\n" % this_str) assert this_str.find('LFN:a') >= 0 assert this_str.find('LFN:b') >= 0 if getConfig('Output')['ForbidLegacyInput']: assert this_str.find('PFN:') >= 0 else: assert this_str.find('c') >= 0
def test_hasLFNs(self): if not getConfig('Output')['ForbidLegacyInput']: ds = make_dataset(['lfn:a']) assert ds.hasLFNs() if not getConfig('Output')['ForbidLegacyInput']: ds = make_dataset(['pfn:a']) else: ds = make_dataset(['a']) assert not ds.hasLFNs() else: ds = make_new_dataset([DiracFile(lfn='a')]) assert ds.hasLFNs() ds = make_new_dataset([LocalFile('/some/local/file')]) assert ds.hasPFNs()
def stop_ganga(): """ This test stops Ganga and shuts it down Most of the logic is weapped in ShutdownManager._ganga_run_exitfuncs but additional code is used to cleanup repos and such between tests """ from Ganga.Utility.logging import getLogger logger = getLogger() logger.info("Deciding how to shutdown") # Do we want to empty the repository on shutdown? from Ganga.Utility.Config import getConfig if 'AutoCleanup' in getConfig('TestingFramework'): whole_cleanup = getConfig('TestingFramework')['AutoCleanup'] else: whole_cleanup = True logger.info("AutoCleanup: %s" % whole_cleanup) if whole_cleanup is True: emptyRepositories() logger.info("Shutting Down Internal Services") # Disable internal services such as monitoring and other tasks #from Ganga.Core.InternalServices import Coordinator # if Coordinator.servicesEnabled: # Coordinator.disableInternalServices() # Coordinator.servicesEnabled = False logger.info("Mimicking ganga exit") from Ganga.Core.InternalServices import ShutdownManager # make sure we don't have an interactive shutdown policy from Ganga.Core.GangaThread import GangaThreadPool GangaThreadPool.shutdown_policy = 'batch' # This should now be safe ShutdownManager._ganga_run_exitfuncs() logger.info("Clearing Config") # Undo any manual editing of the config and revert to defaults clear_config() # Finished logger.info("Test Finished")
def dirac_inputdata(app, hasOtherInputData=False): """ Construct the JDL component which requests the inputdata for a job Args: app (IApplication): app which belongs to the job of interest hasOtherInputData (bool): This is used to stop BannedSites being added to the JDL structure through backend.settings """ job = app.getJobObject() input_data = None parametricinput_data = None inputLFNs = [] input_data = None parametricinput_data = None if not job.inputdata and (not job.master or not job.master.inputdata): return input_data, parametricinput_data wanted_job = job if not job.inputdata and job.master and job.master.inputdata is not None and job.master.inputdata: wanted_job = job.master inputLFNs = ['LFN:'+this_file.lfn for this_file in wanted_job.inputdata if isinstance(this_file, DiracFile)] # master job with a splitter reaching prepare, hence bulk submit if not job.master and job.splitter: parametricinput_data = dirac_parametric_split(app) if parametricinput_data is not None and len(parametricinput_data) > getConfig('DIRAC')['MaxDiracBulkJobs']: raise BackendError('Dirac', 'Number of bulk submission jobs \'%s\' exceeds the maximum allowed \'%s\' if more are needed please modify your config. Note there is a hard limit in Dirac of currently 1000.' % ( len(parametricinput_data), getConfig('DIRAC')['MaxDiracBulkJobs'])) # master job with no splitter or subjob already split proceed as normal else: input_data = inputLFNs if 'Destination' not in job.backend.settings and not inputLFNs and not hasOtherInputData: t1_sites = getConfig('DIRAC')['noInputDataBannedSites'] logger.info('Job has no inputdata (T1 sites will be banned to help avoid overloading them).') if 'BannedSites' in job.backend.settings: job.backend.settings['BannedSites'].extend(t1_sites) job.backend.settings['BannedSites'] = unique(job.backend.settings['BannedSites']) else: if t1_sites: job.backend.settings['BannedSites'] = t1_sites[:] if not input_data and not parametricinput_data: input_data = inputLFNs return input_data, parametricinput_data
def postBootstrapHook(): from Ganga.Utility.logging import getLogger logger = getLogger() from Ganga.Utility.Config import getConfig cfg = getConfig('MonitoringServices') MONITORING_DEFAULT = "Ganga.Lib.MonitoringServices.Dashboard.LCGAthenaMS.LCGAthenaMS" for name in cfg.options: value = cfg[name] if 'Athena' in name.split('/') and ('LCG' in name.split('/') or 'CREAM' in name.split('/')): if not MONITORING_DEFAULT in value.split(','): logger.error('''*** Outdated monitoring configuration - check your configuration files *** *** Outdated monitoring configuration - check your configuration files *** Maybe your ~/.gangarc contains old entries which override new defaults? You may also check the configuration files defined by $GANGA_CONFIG_PATH or $GANGA_CONFIG environment variables. To fix this problem simply remove (or comment out) the following lines in [MonitoringServices] section: Athena/LCG=... Athena/CREAM=... For now I will add the correct default settings (%s) to the configuration of this Ganga session. Note that in the future you won't be able to start Ganga until these issues are corrected manually. '''%MONITORING_DEFAULT) cfg.setUserValue(name,value+','+MONITORING_DEFAULT)
def DiracSizeSplitter(inputs, filesPerJob, maxSize, ignoremissing): """ Generator that yields a datasets for LHCbdirac split jobs by size """ #logger.debug( "DiracSplitter" ) #logger.debug( "inputs: %s" % str( inputs ) ) split_files = [] i = inputs.__class__() if inputs.getLFNs() != len(inputs.files): raise SplittingError( "Error trying to split dataset using DIRAC backend with non-DiracFile in the inputdata") all_files = igroup(inputs.files[:maxFiles], getConfig('DIRAC')['splitFilesChunks'], leftovers=True) #logger.debug( "Looping over all_files" ) #logger.debug( "%s" % str( all_files ) ) for files in all_files: i.files = files LFNsToSplit = i.getLFNs() if(len(LFNsToSplit)) > 1: result = execute('splitInputDataBySize(%s,%d)' % (i.getLFNs(), filesPerJob)) if not result_ok(result): logger.error('DIRAC:: Error splitting files: %s' % str(result)) raise SplittingError('Error splitting files.') split_files += result.get('Value', []) else: split_files = [LFNsToSplit] if len(split_files) == 0: raise SplittingError('An unknown error occured.') # FIXME # check that all files were available on the grid big_list = [] for l in split_files: big_list.extend(l) diff = set(inputs.getFileNames()[:maxFiles]).difference(big_list) if len(diff) > 0: for f in diff: logger.warning('Ignored file: %s' % f) if not ignoremissing: raise SplittingError('Some files not found!') ### logger.debug("Split Files: %s" % str(split_files)) for dataset in split_files: yield dataset
def dorun(self): """Executes a run of actions and sleep periods. Initialises runid to the current UTC ID. """ self.runid = Utility.utcid() while 1: logger.info("Start run %s with id '%s'.", self.run, self.runid) for action in self.run: try: self._doaction(action) except GangaRobotContinueError as e: logger.warning("Continue Error in Action '%s' with message '%s'. Run continued", action, e) except GangaRobotBreakError as e: logger.warning("Break Error in Action '%s' with message '%s'. Run ended", action, e) break except GangaRobotFatalError as e: logger.error("Fatal Error in Action '%s' with message '%s'. Run aborted", action, e) raise except Exception as e: config = getConfig('Robot') if (config['ExceptionBehaviour'] == 'Continue'): logger.error("Error in Action '%s' with message '%s'. Run continued", action, e) elif (config['ExceptionBehaviour'] == 'Break'): logger.error("Error in Action '%s' with message '%s'. Run continued", action, e) break else: logger.error("Abort run id '%s'. Action '%s' failed with message %s.", self.runid, action, e) raise logger.info("Finish run id '%s'.", self.runid) if not self.repeat: break
def setLocation(self): """ Sets the location of output files that were uploaded to mass storage from the WN """ postprocessLocationsPath = os.path.join(self.joboutputdir, getConfig('Output')['PostProcessLocationsFileName']) if not os.path.exists(postprocessLocationsPath): return postprocesslocations = open(postprocessLocationsPath, 'r') for line in postprocesslocations.readlines(): if line.strip() == '': continue lineParts = line.split(' ') outputPattern = lineParts[1] outputPath = lineParts[2] if line.startswith('massstorage'): if outputPattern == self.namePattern: massStorageLocation = outputPath.strip('\n') if massStorageLocation not in self.locations: self.locations.append(massStorageLocation) postprocesslocations.close()
def getSharedPath(): # Required to be initialized for ShareDir object from Ganga.Utility.files import expandfilename import os.path Conf_config = getConfig('Configuration') root_default = os.path.join(expandfilename(Conf_config['gangadir']), 'shared', Conf_config['user']) return root_default
def setLocation(self): """ """ logger.debug("DiracFile: setLocation") if not stripProxy(self).getJobObject(): logger.error("No job assocaited with DiracFile: %s" % str(self)) return job = self.getJobObject() postprocessLocationsPath = os.path.join(job.outputdir, getConfig('Output')['PostProcessLocationsFileName']) postprocesslocations = None try: postprocesslocations = open(postprocessLocationsPath, 'r') self.subfiles = [] for line in postprocesslocations.readlines(): if line.startswith('DiracFile'): if self.dirac_line_processor(line, self, os.path.dirname(postprocessLocationsPath)) and regex.search(self.namePattern) is None: logger.error("Error processing line:\n%s\nAND: namePattern: %s is NOT matched" % (str(line), str(self.namePattern))) except Exception, err: logger.warning("unexpected Error: %s" % str(err))
from Ganga.Utility.Config import getConfig from Ganga.Utility.logging import getLogger from Ganga.Utility.GridShell import getShell from Ganga.GPIDev.Credentials.ICredential import ICredential logger = getLogger() logger.critical('LCG Grid Simulator ENABLED') ########################################################################## # GRID SIMULATOR ########################################################################## config = getConfig("GridSimulator") def sleep(val): import time time.sleep(get_number(val)) def failed(val): t = get_number(val) import random return random.random() < t def get_number(val): import random
from Ganga.Utility.Config import getConfig outputconfig = getConfig("Output") def getSharedPath(): # Required to be initialized for ShareDir object from Ganga.Utility.files import expandfilename import os.path Conf_config = getConfig('Configuration') root_default = os.path.join(expandfilename(Conf_config['gangadir']), 'shared', Conf_config['user']) return root_default
# The absolute path to the job's temporary working directory. #============================= sge_config.addOption('preexecute', 'os.chdir(os.environ["TMPDIR"])\nos.environ["PATH"]+=":."', "String contains commands executing before submiting job to queue") sge_config.addOption('postexecute', '', "String contains commands executing before submiting job to queue") sge_config.addOption('jobnameopt', 'N', "String contains option name for name of job in batch system") sge_config.addOption('timeout', 600, 'Timeout in seconds after which a job is declared killed if it has not touched its heartbeat file. Heartbeat is touched every 30s so do not set this below 120 or so.') # ------------------------------------------------ # Mergers merge_config = makeConfig('Mergers', 'parameters for mergers') merge_config.addOption('associate', {'log': 'TextMerger', 'root': 'RootMerger', 'text': 'TextMerger', 'txt': 'TextMerger'}, 'Dictionary of file associations') gangadir = getConfig('Configuration')['gangadir'] merge_config.addOption('merge_output_dir', gangadir + '/merge_results', "location of the merger's outputdir") merge_config.addOption('std_merge', 'TextMerger', 'Standard (default) merger') # ------------------------------------------------ # Preparable preparable_config = makeConfig('Preparable', 'Parameters for preparable applications') preparable_config.addOption('unprepare_on_copy', False, 'Unprepare a prepared application when it is copied') # ------------------------------------------------ # GPIComponentFilters gpicomp_config = makeConfig('GPIComponentFilters', """Customization of GPI component object assignment for each category there may be multiple filters registered, the one used being defined in the configuration file in [GPIComponentFilters] e.g: {'datasets':{'lhcbdatasets':lhcbFilter, 'testdatasets':testFilter}...}
param += '-o "%s" ' % ( outfiles ) # must be double quotes, because python prints strings in 'single quotes' for file in jspec.Files: if file.type in ['output', 'log'] and configPanda['chirpconfig']: file.dispatchDBlockToken = configPanda['chirpconfig'] logger.debug('chirp file %s', file) jspec.jobParameters = param return jspec from Ganga.GPIDev.Credentials import GridProxy gridProxy = GridProxy() from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('Executable', 'Panda', ExecutablePandaRTHandler) from Ganga.Utility.Config import getConfig, ConfigError config = getConfig('Athena') configDQ2 = getConfig('DQ2') configPanda = getConfig('Panda') from Ganga.Utility.logging import getLogger logger = getLogger()
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from GangaAtlas.Lib.ATLASDataset import isDQ2SRMSite, getLocationsCE, getIncompleteLocationsCE, getIncompleteLocations from GangaAtlas.Lib.ATLASDataset import ATLASLocalDataset from GangaAtlas.Lib.ATLASDataset import DQ2Dataset from Ganga.Utility.Config import getConfig, makeConfig, ConfigError from Ganga.Utility.logging import getLogger from Ganga.GPIDev.Adapters.IRuntimeHandler import IRuntimeHandler from Ganga.Utility.files import expandfilename from GangaAtlas.Lib.ATLASDataset.DQ2Dataset import dq2outputdatasetname shared_path = os.path.join( expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) __directory__ = os.path.dirname(__file__) def _append_file_buffer(inputbox, name, array): inputbox.append(FileBuffer(name, '\n'.join(array) + '\n')) def _append_files(inputbox, *names): for name in names: inputbox.append(File(os.path.join(__directory__, name)))
class Gaudi(GaudiBase): _name = 'Gaudi' __doc__ = GaudiDocString(_name) _category = 'applications' _exportmethods = GaudiBase._exportmethods[:] _exportmethods += ['prepare', 'unprepare'] _hidden = 1 _schema = GaudiBase._schema.inherit_copy() docstr = 'The gaudirun.py cli args that will be passed at run-time' _schema.datadict['args'] = SimpleItem(defvalue=['-T'], sequence=1, strict_sequence=0, typelist=['str', 'type(None)'], doc=docstr) docstr = 'The name of the optionsfile. Import statements in the file ' \ 'will be expanded at submission time and a full copy made' _schema.datadict['optsfile'] = FileItem(preparable=1, sequence=1, strict_sequence=0, defvalue=[], doc=docstr) docstr = 'A python configurable string that will be appended to the ' \ 'end of the options file. Can be multiline by using a ' \ 'notation like \nHistogramPersistencySvc().OutputFile = ' \ '\"myPlots.root"\\nEventSelector().PrintFreq = 100\n or by ' \ 'using triple quotes around a multiline string.' _schema.datadict['extraopts'] = SimpleItem(preparable=1, defvalue=None, typelist=['str', 'type(None)'], doc=docstr) _schema.version.major += 0 _schema.version.minor += 0 def _auto__init__(self): """bootstrap Gaudi applications. If called via a subclass set up some basic structure like version platform...""" self._init() def _parse_options(self): raise NotImplementedError def prepare(self, force=False): from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList from Ganga.GPIDev.Lib.File.File import File if isType(self.optsfile, (list, tuple, GangaList)): for this_file in self.optsfile: if type(this_file) is str: this_file = File(this_file) else: continue elif type(self.optsfile) is str: self.optsfile = [File(self.optsfile)] try: super(Gaudi, self).prepare(force) except Exception, err: logger.debug("Super Prepare Error:\n%s" % str(err)) raise err logger.debug("Prepare") _is_prepared = self.is_prepared #logger.info("_is_prepared: %s" % _is_prepared) share_dir = os.path.join( expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], _is_prepared.name) # We will return a list of files 'send_to_share' which will be copied into the jobs # inputsandbox when prepare called from job object. NOTE that these files will not go # into an inputsandbox when prepare called on standalone app. # Things in the inputsandbox end up in the working dir at runtime. # Exception is just re-thrown here after setting is_prepared to None # could have done the setting in the actual functions but didnt want # prepared state altered from the readInputData pseudo-static member try: self._check_inputs() except Exception, err: logger.debug("_check_inputs Error:\n%s" % str(err)) self.unprepare() raise err
class CredentialStore(GangaObject, collections.Mapping): """ The central management for all credentials It is not intended to store the credential objects between sessions, rather it will search the filesystem or create new credential files. Its interface is similar to a dictionary with ``ICredentialRequirement`` objects as keys and ``ICredentialInfo`` objects as values. A single instance of this class makes most sense and should be created in the bootstrap and exported. E.g: To create a new Afs Token call:: credential_store.create(AfsToken()) To create a new Dirac Proxy call:: credential_store.create(DiracProxy()) To destroy a proxy which is already in the store call:: credential_store[VomsProxy()].destroy() To get a summary of the available proxies already in the store simply type:: credential_store """ _schema = Schema(Version(1, 0), {}) _category = 'credentials2' _hidden = 1 # This class is hidden since we want a 'singleton' created in the bootstrap retry_limit = 5 enable_caching = True _last_clean = None _clean_delay = getConfig('Credentials')['CleanDelay'] __slots__ = ('credentials', ) def __init__(self): super(CredentialStore, self).__init__() self.credentials = set() @export def create(self, query, create=True, check_file=False): # type: (ICredentialRequirement, bool, bool) -> ICredentialInfo """ Create an ``ICredentialInfo`` for the query. Args: query (ICredentialRequirement): check_file (bool): Raise an exception if the file does not exist create (bool): Create the credential file Returns: The newly created ICredentialInfo object """ try: assert isinstance(query, ICredentialRequirement), "Error checking 'query'" assert isinstance(create, bool), "Error checking 'create'" assert isinstance(check_file, bool), "Error checking 'check_file'" except AssertionError as err: raise CredentialsError( "Requirements to make a Credential are wrong. Please check your arguments. %s" % err) cred = query.info_class(query, check_file=check_file, create=create) self.credentials.add(cred) return cred def remove(self, credential_object): # type: (ICredentialInfo) -> None """ Args: credential_object (ICredentialInfo): """ self.credentials.remove(credential_object) @export def __str__(self, interactive=False): """ This creates a table summary of all credentials known to the store """ self.clean() headers = ['Type', 'Location', 'Valid', 'Time left'] cred_info = [ [ str(f) for f in (type(cred).__name__, cred.location, cred.is_valid(), cred.time_left()) ] # Format each field as a string for cred in self.credentials # for each credential in the store ] return self._create_table(headers, cred_info) @staticmethod def _create_table(headers, data): """ Create a formatted table out of the headers and data Args: headers (List[str]): the strings for the table headers data (List[List[str]]): the body of the table Returns (str): a formatted string displaying the data """ rows = [headers] + data # Get the length of the longest string in each column column_widths = [ max(len(field) for field in column) for column in zip(*rows) ] def pad_row_strings(row, widths, filler=' '): # type: (List[str], List[int], str) -> List[str] """Add padding to each of ``row`` to equal the corresponding ``widths`` entry, padded with ``filler``""" return [ '{field:{filler}<{width}}'.format(field=field[0], filler=filler, width=field[1]) for field in zip(row, widths) ] padded_headers = pad_row_strings(headers, column_widths) padded_dividers = pad_row_strings([''] * len(column_widths), column_widths, filler='-') padded_data = [pad_row_strings(row, column_widths) for row in data] # Concatenate the field strings together def strings_to_row(strings, spacer='|'): # type: (List[str], str) -> str return ' {0} '.format(spacer).join(strings) header = strings_to_row(padded_headers) divider = strings_to_row(padded_dividers, spacer='+') body = '\n'.join( strings_to_row(padded_field_strings) for padded_field_strings in padded_data) return '\n'.join([header, divider, body]) @export def __iter__(self): """Allow iterating over the store directly""" # yield from self.credentialList #In Python 3.3 return iter(self.credentials) @export def __len__(self): """How many credentials are known about in the system""" return len(self.credentials) @export def __getitem__(self, query): # type: (ICredentialRequirement) -> ICredentialInfo """ This function will try quite hard to find and wrap any missing credential but will never create a new file on disk. It should *always* be non-interactive so that it can be run on background threads without issue. Args: query (ICredentialRequirement): Returns: A single ICredentialInfo object which matches the requirements Raises: GangaKeyError: If it could not provide a credential GangaTypeError: If query is of the wrong type """ self.clean() if not isinstance(query, ICredentialRequirement): raise GangaTypeError( 'Credential store query should be of type ICredentialRequirement' ) match = self.match(query) if match: return match try: cred = self.create(query, create=False, check_file=True) except IOError as err: logger.debug(err.strerror) except CredentialsError as err: logger.debug(str(err)) else: self.credentials.add(cred) return cred raise GangaKeyError( 'Matching credential [{query}] not found in store.'.format( query=query)) def get(self, query, default=None): # type: (ICredentialRequirement, Optional[ICredentialInfo]) -> Optional[ICredentialInfo] """ Return the value for ``query`` if ``query`` is in the store, else default. If ``default`` is not given, it defaults to ``None``, so that this method never raises a ``KeyError``. Args: query (ICredentialRequirement): default (ICredentialInfo): Returns: A single ICredentialInfo object which matches the requirements or ``default`` """ try: assert isinstance(query, ICredentialRequirement), "Error checking 'query'" if default is not None: assert isinstance(default, ICredentialInfo), "Error checking 'default'" except AssertionError as err: raise CredentialsError( "Requirements for get-ing a Credential are wrong. Please check your arguments. %s" % err) try: return self[query] except KeyError: return default def get_all_matching_type(self, query): # type: (ICredentialRequirement) -> Sequence[ICredentialInfo] """ Returns all ``ICredentialInfo`` with the type that matches the query Args: query (ICredentialRequirement): Returns: list[ICredentialInfo]: An list of all matching objects """ try: assert isinstance(query, ICredentialRequirement), "Error checking 'query'" except AssertionError as err: raise CredentialsError( "Requirements for matching all Credential are wrong. Please check your arguments. %s" % err) return [ cred for cred in self.credentials if isinstance(cred, query.info_class) ] def matches(self, query): # type: (ICredentialRequirement) -> Sequence[ICredentialInfo] """ Search the credentials in the store for all matches. They must match every condition exactly. Args: query (ICredentialRequirement): Returns: list[ICredentialInfo]: An list of all matching objects """ try: assert isinstance(query, ICredentialRequirement), "Error checking 'query'" except AssertionError as err: raise CredentialsError( "Requirements for matching any Credential are wrong. Please check your arguments. %s" % err) return [ cred for cred in self.get_all_matching_type(query) if cred.check_requirements(query) ] def match(self, query): # type: (ICredentialRequirement) -> ICredentialInfo """ Returns a single match from the store Args: query (ICredentialRequirement): Returns: ICredentialInfo: A single credential object. If more than one is found, the first is returned """ try: assert isinstance(query, ICredentialRequirement), "Error checking 'query'" except AssertionError as err: raise CredentialsError( "Requirements for matching a Credential are wrong. Please check your arguments. %s" % err) matches = self.matches(query) if len(matches) == 1: return matches[0] if len(matches) > 1: logger.debug('More than one match...') # If we have a specific object and a general one. Then we ask for a general one, what should we do. # Does it matter since they've only asked for a general proxy? What are the use cases? return matches[ 0] # TODO For now just return the first one... Though perhaps we should merge them or something? return None @export def renew(self): # type: () -> None """ Renew all credentials which are invalid or will expire soon. It also uses the entries in `needed_credentials` and adds and renews those TODO Should this function be standalone? """ for cred in self.credentials: if not cred.is_valid() or cred.time_left() < timedelta(hours=1): cred.renew() for cred_req in needed_credentials - self.credentials: try: self[cred_req].renew() except KeyError: self.create(cred_req) @export def clear(self): # type: () -> None """ Remove all credentials in the system (without destroying them) """ self.credentials = set() def clean(self): # type: () -> None """ Remove any credentials with missing files """ this_time = time.time() if not CredentialStore._last_clean: CredentialStore._last_clean = this_time self.credentials = set(cred for cred in self.credentials if cred.exists()) elif this_time - CredentialStore._last_clean > CredentialStore._clean_delay: self.credentials = set(cred for cred in self.credentials if cred.exists()) CredentialStore._last_clean = this_time
if not x.exists(): raise ApplicationConfigurationError(None, '%s: file not found' % x.name) except AttributeError as err: raise ApplicationConfigurationError(err, '%s (%s): unsupported type, must be a string or File' % (str(x), str(type(x)))) validate_argument(self.exe, exe=1) for a in self.args: validate_argument(a) return (None, None) # disable type checking for 'exe' property (a workaround to assign File() objects) # FIXME: a cleaner solution, which is integrated with type information in # schemas should be used automatically config = getConfig('defaults_Executable') # _Properties #config.setDefaultOption('exe',Executable._schema.getItem('exe')['defvalue'], type(None),override=True) config.options['exe'].type = type(None) # not needed anymore: # the backend is also required in the option name # so we need a kind of dynamic options (5.0) #mc = getConfig('MonitoringServices') #mc['Executable'] = None def convertIntToStringArgs(args): result = [] for arg in args:
def _display(self, colour): from Ganga.Utility.ColourText import ANSIMarkup, NoMarkup, getColour, Foreground, Effects if colour: markup = ANSIMarkup() else: markup = NoMarkup() fg = Foreground() display_config = getConfig('Display') name_colour = getColour(display_config['config_name_colour']) docstring_colour = getColour( display_config['config_docstring_colour']) # fg.boldgrey value_colour = getColour( display_config['config_value_colour']) # fx.normal levels = ['**', '* ', ' '] levels = map(lambda x: markup(x, fg.red), levels) from cStringIO import StringIO sio = StringIO() sio.write('%s' % markup(stripProxy(self).name, name_colour) + ' : ' + markup(stripProxy(self).docstring, docstring_colour) + '\n') opts = sorted(stripProxy(self).options.keys()) INDENT = ' ' * 2 p = re.compile('[\.\w]*\.') for o in opts: sio.write( levels[stripProxy(self).getEffectiveLevel(o)] + ' ' + markup(o, name_colour) + ' = ' + markup(p.sub('', repr(stripProxy(self)[o])), value_colour) + '\n') sio.write( textwrap.fill(markup( stripProxy(self).options[o].docstring.strip(), docstring_colour), width=80, initial_indent=INDENT, subsequent_indent=INDENT) + '\n') typelist = stripProxy(self).options[o].typelist if not typelist: typedesc = 'Type: ' + \ p.sub('',str(type(stripProxy(self).options[o].default_value))) else: typedesc = 'Allowed types: ' + \ str([p.sub('',str(t)) for t in typelist]) sio.write(markup(INDENT + typedesc, docstring_colour) + '\n') filter = stripProxy(self).options[o].filter if filter: filter_doc = filter.__doc__ if not filter_doc: filter_doc = "undocumented" sio.write( markup(INDENT + "Filter: " + filter_doc, docstring_colour) + '\n') examples = stripProxy(self).options[o].examples if examples: sio.write( markup(INDENT + "Examples:", docstring_colour) + '\n') for e in examples.splitlines(): sio.write( markup(INDENT + e.strip(), docstring_colour) + '\n') return sio.getvalue()
def _getRootEnvSys(self, version, usepython=False): """Returns an environment suitable for running Root and sometimes Python.""" from os.path import join from os import environ from Ganga.Lib.Root.shared import setEnvironment, findPythonVersion rootsys = getrootsys(version) logger.info("rootsys: %s" % str(rootsys)) rootenv = {} # propagate from localhost if 'PATH' in environ: setEnvironment('PATH', environ['PATH'], update=True, environment=rootenv) if 'LD_LIBRARY_PATH' in environ: setEnvironment('LD_LIBRARY_PATH', environ['LD_LIBRARY_PATH'], update=True, environment=rootenv) setEnvironment('LD_LIBRARY_PATH', join(rootsys, 'lib'), update=True, environment=rootenv) setEnvironment('PATH', join(rootsys, 'bin'), update=True, environment=rootenv) setEnvironment('ROOTSYS', rootsys, update=False, environment=rootenv) logger.debug('Have set Root variables. rootenv is now %s', str(rootenv)) if usepython: # first get from config python_version = '' try: python_version = getConfig('ROOT')['pythonversion'] except ConfigError as e: logger.debug( 'There was a problem trying to get [ROOT]pythonversion: %s.', e) logger.debug('Found version of python: %s', str(python_version)) # now try grepping files if not python_version: python_version = findPythonVersion(rootsys) if (python_version is None): logger.warn( 'Unable to find the Python version needed for Root version %s. See the [ROOT] section of your .gangarc file.', version) else: logger.debug('Python version found was %s', python_version) python_home = getpythonhome(pythonversion=python_version) logger.info("Looking in: %s" % python_home) logger.debug('PYTHONHOME is being set to %s', python_home) python_bin = join(python_home, 'bin') setEnvironment('PATH', python_bin, update=True, environment=rootenv) setEnvironment('PYTHONPATH', join(rootsys, 'lib'), update=True, environment=rootenv) logger.debug('Added PYTHONPATH. rootenv is now %s', str(rootenv)) if join(python_bin, 'python') != sys.executable: # only try to do all this if the python currently running isn't # going to be used logger.debug('Using a different Python - %s.', python_home) python_lib = join(python_home, 'lib') if not os.path.exists(python_bin) or not os.path.exists( python_lib): logger.error( 'The PYTHONHOME specified does not have the expected structure. See the [ROOT] section of your .gangarc file.' ) logger.error('PYTHONPATH is: ' + str(os.path)) setEnvironment('LD_LIBRARY_PATH', python_lib, update=True, environment=rootenv) setEnvironment('PYTHONHOME', python_home, update=False, environment=rootenv) setEnvironment('PYTHONPATH', python_lib, update=True, environment=rootenv) return (rootenv, rootsys)
def file_contains(filename, string): return file(filename).read().find(string) != -1 def write_file(filename, content): """ Open,write and close the file descriptor""" f = open(filename, 'w') try: return f.write(content) finally: f.close() def read_file(filename): """ read the file, and safely close the file at the end""" f = open(filename) try: return "\n%s\n" % f.read() finally: f.close() import unittest failureException = unittest.TestCase.failureException try: from Ganga.Utility.Config import getConfig config = getConfig('TestingFramework') except: # if we are outside Ganga, use a simple dict config = {}
from Ganga.GPIDev.Credentials import credential_store from Ganga.Utility.Config import getConfig from Ganga.Utility.logging import getLogger from Ganga.Utility.GridShell import getShell from Ganga.Lib.LCG.GridftpSandboxCache import GridftpFileIndex, GridftpSandboxCache from Ganga.Lib.LCG.Utility import get_uuid from Ganga.Lib.Root import randomString # global variables logger = getLogger() config = getConfig('LCG') def __set_submit_option__(): submit_option = '' if config['Config']: submit_option += ' --config %s' % config['Config'] elif config['GLITE_ALLOWED_WMS_LIST']: wms_conf_path = os.path.join(os.environ['GLITE_WMS_LOCATION'], 'etc', config['VirtualOrganisation'], 'glite_wmsui.conf') temp_wms_conf = tempfile.NamedTemporaryFile(suffix='.conf', delete=False)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList from Ganga.GPIDev.Schema import Schema, Version, SimpleItem, ComponentItem from Ganga.GPIDev.Adapters.IGangaFile import IGangaFile from Ganga.GPIDev.Lib.File import FileUtils from Ganga.GPIDev.Lib.Job.Job import Job from Ganga.Utility.files import expandfilename from Ganga.Core.exceptions import GangaFileError from GangaDirac.Lib.Utilities.DiracUtilities import getDiracEnv, execute, GangaDiracError import Ganga.Utility.Config from Ganga.Runtime.GPIexport import exportToGPI from Ganga.GPIDev.Credentials import require_credential from GangaDirac.Lib.Credentials.DiracProxy import DiracProxy, DiracProxyInfo from Ganga.Utility.Config import getConfig from Ganga.Utility.logging import getLogger from GangaDirac.Lib.Backends.DiracUtils import getAccessURLs configDirac = getConfig('DIRAC') logger = getLogger() regex = re.compile('[*?\[\]]') global stored_list_of_sites stored_list_of_sites = [] class DiracFile(IGangaFile): """ File stored on a DIRAC storage element Usage: Some common use cases:
def getShell(middleware='EDG', force=False): """ Utility function for getting Grid Shell. Caller should take responsiblity of credential checking if proxy is needed. Argumennts: middleware - grid m/w used force - False : if the shell already exists in local cache return the previous created instance True : recreate the shell and if not None update the cache """ logger = getLogger() if not middleware: logger.debug('No middleware specified, assuming default EDG') middleware = 'EDG' if middleware in _allShells.keys() and not force: return _allShells[middleware] values = {} for key in ['X509_USER_PROXY', 'X509_CERT_DIR', 'X509_VOMS_DIR']: try: values[key] = os.environ[key] except KeyError: pass configname = "" if middleware == 'EDG' or middleware == 'GLITE': configname = 'LCG' else: configname = middleware config = None try: config = getConfig(configname) except: logger.warning( '[%s] configuration section not found. Cannot set up a proper grid shell.' % configname) return None s = None key = '%s_SETUP' % middleware # 1. check if the *_SETUP is changed by user -> take the user's value as session value # 2. else check if *_LOCATION is defined as env. variable -> do nothing (ie. create shell without any lcg setup) # 3. else take the default *_SETUP as session value MIDDLEWARE_LOCATION = '%s_LOCATION' % middleware if config.getEffectiveLevel(key) == 2 and MIDDLEWARE_LOCATION in os.environ: s = Shell() else: if os.path.exists(config[key]): # FIXME: Hardcoded rule for ARC middleware setup (pass explicitly # the $ARC_LOCATION as argument), this is hardcoded to maintain # backwards compatibility (and avoid any side effects) for EDG and # GLITE setup scripts which did not take any arguments if key.startswith('ARC') and MIDDLEWARE_LOCATION in os.environ: s = Shell( config[key], setup_args=[os.environ[MIDDLEWARE_LOCATION]]) else: s = Shell(config[key]) else: logger.warning("Configuration of %s for %s: " % (middleware, configname)) logger.warning("File not found: %s" % config[key]) if s: for key, val in values.items(): s.env[key] = val # check and set env. variables for default LFC setup if 'LFC_HOST' not in s.env: try: s.env['LFC_HOST'] = config['DefaultLFC'] except ConfigError: pass if 'LFC_CONNTIMEOUT' not in s.env: s.env['LFC_CONNTIMEOUT'] = '20' if 'LFC_CONRETRY' not in s.env: s.env['LFC_CONRETRY'] = '0' if 'LFC_CONRETRYINT' not in s.env: s.env['LFC_CONRETRYINT'] = '1' _allShells[middleware] = s return s
from Ganga.Utility.Config import getConfig, ConfigError from Ganga.Utility.root import getrootsys, getpythonhome from Ganga.Core.exceptions import ApplicationPrepareError import Ganga.Utility.logging import inspect import os import sys import tempfile from Ganga.Utility.files import expandfilename from Ganga.GPIDev.Base.Proxy import getName logger = Ganga.Utility.logging.getLogger() config = getConfig('ROOT') def getDefaultScript(): name = os.path.join( os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))), 'defaultRootScript.C') return File(name=name) class Root(IPrepareApp): """ Root application -- running ROOT To run a job in ROOT you need to specify the CINT script to be
def preparejob(self, jobconfig, master_input_sandbox): job = self.getJobObject() # print str(job.backend_output_postprocess) mon = job.getMonitoringService() import Ganga.Core.Sandbox as Sandbox from Ganga.GPIDev.Lib.File import File from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR import inspect fileutils = File(inspect.getsourcefile(Ganga.Utility.files), subdir=PYTHON_DIR) sharedfiles = jobconfig.getSharedFiles() subjob_input_sandbox = job.createPackedInputSandbox( jobconfig.getSandboxFiles() + [fileutils]) appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings() if self.nice: appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath if self.nice < 0: logger.warning( 'increasing process priority is often not allowed, your job may fail due to this' ) sharedoutputpath = job.getOutputWorkspace().getPath() ## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method. outputpatterns = jobconfig.outputbox environment = dict() if jobconfig.env is None else jobconfig.env import tempfile workdir = tempfile.mkdtemp(dir=config['location']) import inspect script_location = os.path.join( os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))), 'LocalHostExec.py.template') from Ganga.GPIDev.Lib.File import FileUtils script = FileUtils.loadScript(script_location, '') script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox)) from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation from Ganga.Utility.Config import getConfig jobidRepr = repr(job.getFQID('.')) script = script.replace( '###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr)) script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, '')) script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, '')) script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, '')) script = script.replace('###APPLICATION_NAME###', repr(getName(job.application))) script = script.replace( '###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox + sharedfiles)) script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath)) script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath)) script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns)) script = script.replace('###JOBID###', jobidRepr) script = script.replace('###ENVIRONMENT###', repr(environment)) script = script.replace('###WORKDIR###', repr(workdir)) script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir())) self.workdir = workdir script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH'])) wrkspace = job.getInputWorkspace() scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1) return scriptPath
from Ganga.GPIDev.Adapters.IApplication import IApplication from Ganga.GPIDev.Adapters.IPrepareApp import IPrepareApp from Ganga.GPIDev.Adapters.IRuntimeHandler import IRuntimeHandler from Ganga.GPIDev.Schema import * from Ganga.Utility.Config import getConfig from Ganga.GPIDev.Lib.File import * from Ganga.GPIDev.Lib.Registry.PrepRegistry import ShareRef from Ganga.GPIDev.Base.Proxy import isType from Ganga.Core import ApplicationConfigurationError import os, shutil, commands, re from Ganga.Utility.files import expandfilename shared_path = os.path.join( expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) class Highland(IApplication): """ Highland application running any highland executables. The required input for this module are: app.exe = 'RunNumuCCAnalysis.exe' app.cmtsetup = '/home/me/myT2KWork/setup.sh' app.outputfile = 'myFantasticResults.root' NOTE: Ganga will run a bash script so the CMT setup script must be in sh/bash. It is also possible to pass command line arguments to the executable.
def updateMonitoringInformation(jobs): jobDict = {} for job in jobs: if job.backend.id: jobDict[job.backend.id] = job idList = jobDict.keys() if not idList: return queryCommand = " ".join\ ([ "condor_q -global" if getConfig( "Condor")["query_global_queues"] else "condor_q", "-format \"%s \" GlobalJobId", "-format \"%s \" RemoteHost", "-format \"%d \" JobStatus", "-format \"%f\\n\" RemoteUserCpu" ]) status, output = commands.getstatusoutput(queryCommand) if 0 != status: logger.error("Problem retrieving status for Condor jobs") return if ("All queues are empty" == output): infoList = [] else: infoList = output.split("\n") allDict = {} for infoString in infoList: tmpList = infoString.split() id, host, status, cputime = ("", "", "", "") if 3 == len(tmpList): id, status, cputime = tmpList if 4 == len(tmpList): id, host, status, cputime = tmpList if id: allDict[id] = {} allDict[id]["status"] = Condor.statusDict[status] allDict[id]["cputime"] = cputime allDict[id]["host"] = host fg = Foreground() fx = Effects() status_colours = {'submitted': fg.orange, 'running': fg.green, 'completed': fg.blue} for id in idList: printStatus = False if jobDict[id].status == "killed": continue localId = id.split("#")[-1] globalId = id if globalId == localId: queryCommand = " ".join\ ([ "condor_q -global" if getConfig( "Condor")["query_global_queues"] else "condor_q", "-format \"%s\" GlobalJobId", id ]) status, output = commands.getstatusoutput(queryCommand) if 0 == status: globalId = output if globalId in allDict.keys(): status = allDict[globalId]["status"] host = allDict[globalId]["host"] cputime = allDict[globalId]["cputime"] if status != jobDict[id].backend.status: printStatus = True stripProxy(jobDict[id])._getWriteAccess() jobDict[id].backend.status = status if jobDict[id].backend.status == "Running": jobDict[id].updateStatus("running") if host: if jobDict[id].backend.actualCE != host: jobDict[id].backend.actualCE = host jobDict[id].backend.cputime = cputime else: jobDict[id].backend.status = "" outDir = jobDict[id].getOutputWorkspace().getPath() condorLogPath = "".join([outDir, "condorLog"]) checkExit = True if os.path.isfile(condorLogPath): checkExit = False for line in open(condorLogPath): if -1 != line.find("terminated"): checkExit = True break if -1 != line.find("aborted"): checkExit = True break if checkExit: printStatus = True stdoutPath = "".join([outDir, "stdout"]) jobStatus = "failed" if os.path.isfile(stdoutPath): with open(stdoutPath) as stdout: lineList = stdout.readlines() try: exitLine = lineList[-1] exitCode = exitLine.strip().split()[-1] except IndexError: exitCode = -1 if exitCode.isdigit(): jobStatus = "completed" else: logger.error("Problem extracting exit code from job %s. Line found was '%s'." % ( jobDict[id].fqid, exitLine)) jobDict[id].updateStatus(jobStatus) if printStatus: if jobDict[id].backend.actualCE: hostInfo = jobDict[id].backend.actualCE else: hostInfo = "Condor" status = jobDict[id].status if status in status_colours: colour = status_colours[status] else: colour = fg.magenta if "submitted" == status: preposition = "to" else: preposition = "on" if jobDict[id].backend.status: backendStatus = "".join\ ([" (", jobDict[id].backend.status, ") "]) else: backendStatus = "" logger.info(colour + 'Job %s %s%s %s %s - %s' + fx.normal, jobDict[ id].fqid, status, backendStatus, preposition, hostInfo, time.strftime('%c')) return None
def report_inner(job=None, isJob=False, isTask=False): userInfoDirName = "userreport" tempDirName = "reportsRepository" # job relevant info jobSummaryFileName = "jobsummary.txt" jobFullPrintFileName = "jobfullprint.txt" repositoryPath = "repository/$usr/LocalXML/6.0/jobs/$thousandsNumxxx" # task relevant info taskSummaryFileName = "tasksummary.txt" taskFullPrintFileName = "taskfullprint.txt" tasksRepositoryPath = "repository/$usr/LocalXML/6.0/tasks/$thousandsNumxxx" # user's info environFileName = "environ.txt" userConfigFileName = "userconfig.txt" defaultConfigFileName = "gangarc.txt" ipythonHistoryFileName = "ipythonhistory.txt" gangaLogFileName = "gangalog.txt" jobsListFileName = "jobslist.txt" tasksListFileName = "taskslist.txt" from Ganga.Utility import Config uploadFileServer = Config.getConfig('Feedback')['uploadServer'] #uploadFileServer= "http://gangamon.cern.ch/django/errorreports/" #uploadFileServer= "http://ganga-ai-02.cern.ch/django/errorreports/" #uploadFileServer= "http://127.0.0.1:8000/errorreports" def printDictionary(dictionary, file=sys.stdout): for k, v in dictionary.iteritems(): print('%s: %s' % (k, v), file=file) if k == 'PYTHONPATH': global PYTHON_PATH PYTHON_PATH = v def extractFileObjects(fileName, targetDirectoryName): try: fileToRead = open(fileName, 'r') try: fileText = fileToRead.read() import re pattern = "File\(name=\'(.+?)\'" matches = re.findall(pattern, fileText) for fileName in matches: fileName = os.path.expanduser(fileName) targetFileName = os.path.join( targetDirectoryName, os.path.basename(fileName)) shutil.copyfile(fileName, targetFileName) finally: fileToRead.close() # except IOError, OSError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) def writeErrorLog(errorMessage): try: fileToWrite = open(errorLogPath, 'a') try: fileToWrite.write(errorMessage) fileToWrite.write("\n") except Exception as err: logger.debug("Err: %s" % str(err)) raise finally: fileToWrite.close() except Exception as err2: logger.debug("Err: %s" % str(err2)) pass def writeStringToFile(fileName, stringToWrite): try: # uncomment this to try the error logger #fileName = '~/' + fileName fileToWrite = open(fileName, 'w') try: fileToWrite.write(stringToWrite) except Exception as err: logger.debug("Err: %s" % str(err)) raise err finally: fileToWrite.close() # except IOError: except Exception as err: logger.debug("Err2: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) def renameDataFiles(directory): for fileName in os.listdir(directory): fullFileName = os.path.join(directory, fileName) if os.path.isfile(fullFileName): if fileName == 'data': os.rename(fullFileName, fullFileName + '.txt') else: renameDataFiles(fullFileName) import shutil import tarfile import tempfile import os userHomeDir = os.getenv("HOME") tempDir = tempfile.mkdtemp() errorLogPath = os.path.join(tempDir, 'reportErrorLog.txt') fullPathTempDir = os.path.join(tempDir, tempDirName) fullLogDirName = '' # create temp dir and specific dir for the job/user try: if not os.path.exists(fullPathTempDir): os.mkdir(fullPathTempDir) import datetime now = datetime.datetime.now() userInfoDirName = userInfoDirName + \ now.strftime("%Y-%m-%d-%H:%M:%S") fullLogDirName = os.path.join(fullPathTempDir, userInfoDirName) # if report directory exists -> delete it's content(we would like # last version of the report) if os.path.exists(fullLogDirName): shutil.rmtree(fullLogDirName) os.mkdir(fullLogDirName) # except OSError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import os.environ in a file fullEnvironFileName = os.path.join(fullLogDirName, environFileName) try: inputFile = open(fullEnvironFileName, 'w') try: printDictionary(os.environ, file=inputFile) print('OS VERSION : ' + platform.platform(), file=inputFile) finally: inputFile.close() # except IOError except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import user config in a file userConfigFullFileName = os.path.join( fullLogDirName, userConfigFileName) try: inputFile = open(userConfigFullFileName, 'w') try: print("#GANGA_VERSION = %s" % config.System.GANGA_VERSION, file=inputFile) global GANGA_VERSION GANGA_VERSION = config.System.GANGA_VERSION # this gets the default values # Ganga.GPIDev.Lib.Config.Config.print_config_file() # this should get the changed values for c in config: print(config[c], file=inputFile) finally: inputFile.close() # except IOError does not catch the exception ??? except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # write gangarc - default configuration defaultConfigFullFileName = os.path.join( fullLogDirName, defaultConfigFileName) try: outputFile = open(os.path.join(userHomeDir, '.gangarc'), 'r') try: writeStringToFile(defaultConfigFullFileName, outputFile.read()) finally: outputFile.close() # except IOError does not catch the exception ??? except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import ipython history in a file try: ipythonFile = open( os.path.join(os.environ['IPYTHONDIR'], 'history'), 'r') try: lastIPythonCommands = ipythonFile.readlines()[-20:] writeStringToFile(os.path.join( fullLogDirName, ipythonHistoryFileName), '\n'.join(lastIPythonCommands)) #writeStringToFile(os.path.join(fullLogDirName, ipythonHistoryFileName), ipythonFile.read()) finally: ipythonFile.close() # except IOError does not catch the exception ??? except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import gangalog in a file userLogFileLocation = config["Logging"]._logfile userLogFileLocation = os.path.expanduser(userLogFileLocation) try: gangaLogFile = open(userLogFileLocation, 'r') try: writeStringToFile( os.path.join(fullLogDirName, gangaLogFileName), gangaLogFile.read()) finally: gangaLogFile.close() # except IOError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import the result of jobs command in the report jobsListFullFileName = os.path.join(fullLogDirName, jobsListFileName) try: outputFile = open(jobsListFullFileName, 'w') try: from Ganga.GPI import jobs print(jobs, file=outputFile) finally: outputFile.close() # except IOError does not catch the exception ??? except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import the result of tasks command in the report tasksListFullFileName = os.path.join(fullLogDirName, tasksListFileName) try: outputFile = open(tasksListFullFileName, 'w') try: from Ganga.GPI import tasks print(tasks, file=outputFile) finally: outputFile.close() # except IOError does not catch the exception ??? except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # save it here because we will change fullLogDirName, but we want this # to be the archive and to be deleted folderToArchive = fullLogDirName # import job relevant info if (job is not None and isJob): global JOB_REPORT, APPLICATION_NAME, BACKEND_NAME JOB_REPORT = True APPLICATION_NAME = getName(job.application) BACKEND_NAME = getName(job.backend) # create job folder jobFolder = 'job_%s' % str(job.fqid) fullLogDirName = os.path.join(fullLogDirName, jobFolder) os.mkdir(fullLogDirName) # import job summary in a file fullJobSummaryFileName = os.path.join( fullLogDirName, jobSummaryFileName) writeStringToFile(fullJobSummaryFileName, str(job)) # import job full print in a file fullJobPrintFileName = os.path.join( fullLogDirName, jobFullPrintFileName) try: inputFile = open(fullJobPrintFileName, 'w') try: full_print(job, inputFile) finally: inputFile.close() # except IOError, OSError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # extract file objects try: fileObjectsPath = os.path.join(fullLogDirName, 'fileobjects') os.mkdir(fileObjectsPath) extractFileObjects(fullJobSummaryFileName, fileObjectsPath) # except OSError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # copy dir of the job ->input/output and subjobs try: parentDir, currentDir = os.path.split(job.inputdir[:-1]) workspaceDir = os.path.join(fullLogDirName, 'workspace') shutil.copytree(parentDir, workspaceDir) # except IOError, OSError except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # copy shared area of the job try: if hasattr(job.application, 'is_prepared'): if job.application.is_prepared is not None and job.application.is_prepared is not True: import os from Ganga.Utility.Config import getConfig from Ganga.Utility.files import expandfilename shared_path = os.path.join(expandfilename(getConfig( 'Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) shareddir = os.path.join( shared_path, job.application.is_prepared.name) if os.path.isdir(shareddir): sharedAreaDir = os.path.join( fullLogDirName, 'sharedarea') shutil.copytree(shareddir, sharedAreaDir) # except IOError, OSError except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # copy repository job file try: indexFileName = str(job.id) + '.index' repositoryPath = repositoryPath.replace( '$usr', os.getenv("USER")) # check if the job is subjob -> different way of forming the # path to the repository is_subjob = job.fqid.find('.') > -1 if is_subjob: jobid, subjobid = job.fqid.split( '.')[0], job.fqid.split('.')[1] repositoryPath = repositoryPath.replace( '$thousandsNum', str(int(jobid) / 1000)) repositoryPath = os.path.join(repositoryPath, jobid) else: repositoryPath = repositoryPath.replace( '$thousandsNum', str(job.id / 1000)) repositoryFullPath = os.path.join( config.Configuration.gangadir, repositoryPath) indexFileSourcePath = os.path.join( repositoryFullPath, indexFileName) repositoryFullPath = os.path.join( repositoryFullPath, str(job.id)) repositoryTargetPath = os.path.join( fullLogDirName, 'repository', str(job.id)) os.mkdir(os.path.join(fullLogDirName, 'repository')) shutil.copytree(repositoryFullPath, repositoryTargetPath) # data files are copied but can not be opened -> add .txt to # their file names renameDataFiles(repositoryTargetPath) if not is_subjob: # copy .index file indexFileTargetPath = os.path.join( fullLogDirName, 'repository', indexFileName) shutil.copyfile(indexFileSourcePath, indexFileTargetPath) # except OSError, IOError: except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # import task relevant info if (job is not None and isTask): # job is actually a task object task = job # create task folder taskFolder = 'task_%s' % str(task.id) fullLogDirName = os.path.join(fullLogDirName, taskFolder) os.mkdir(fullLogDirName) # import task summary in a file fullTaskSummaryFileName = os.path.join( fullLogDirName, taskSummaryFileName) writeStringToFile(fullTaskSummaryFileName, str(task)) # import task full print in a file fullTaskPrintFileName = os.path.join( fullLogDirName, taskFullPrintFileName) try: inputFile = open(fullTaskPrintFileName, 'w') try: full_print(task, inputFile) except Exception as err: logger.debug("Err: %s" % str(err)) raise err finally: inputFile.close() # except IOError, OSError: except Exception as err: logger.debug("Err2: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # copy shared area of the task try: if len(task.transforms) > 0: if hasattr(task.transforms[0], 'application') and hasattr(task.transforms[0].application, 'is_prepared'): if task.transforms[0].application.is_prepared is not None and task.transforms[0].application.is_prepared is not True: import os from Ganga.Utility.Config import getConfig from Ganga.Utility.files import expandfilename shared_path = os.path.join(expandfilename(getConfig( 'Configuration')['gangadir']), 'shared', getConfig('Configuration')['user']) shareddir = os.path.join( shared_path, task.transforms[0].application.is_prepared.name) if os.path.isdir(shareddir): sharedAreaDir = os.path.join( fullLogDirName, 'sharedarea') shutil.copytree(shareddir, sharedAreaDir) # except IOError, OSError except Exception as err: logger.debug("Err: %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) # copy repository task file try: indexFileName = str(task.id) + '.index' tasksRepositoryPath = tasksRepositoryPath.replace( '$usr', os.getenv("USER")) tasksRepositoryPath = tasksRepositoryPath.replace( '$thousandsNum', str(task.id / 1000)) repositoryFullPath = os.path.join( config.Configuration.gangadir, tasksRepositoryPath) indexFileSourcePath = os.path.join( repositoryFullPath, indexFileName) repositoryFullPath = os.path.join( repositoryFullPath, str(task.id)) repositoryTargetPath = os.path.join( fullLogDirName, 'repository', str(task.id)) os.mkdir(os.path.join(fullLogDirName, 'repository')) shutil.copytree(repositoryFullPath, repositoryTargetPath) # data files are copied but can not be opened -> add .txt to # their file names renameDataFiles(repositoryTargetPath) # copy .index file indexFileTargetPath = os.path.join( fullLogDirName, 'repository', indexFileName) shutil.copyfile(indexFileSourcePath, indexFileTargetPath) # except OSError, IOError: except Exception as err: logger.debug("Err %s" % str(err)) writeErrorLog(str(sys.exc_info()[1])) resultArchive = '%s.tar.gz' % folderToArchive try: resultFile = tarfile.TarFile.open(resultArchive, 'w:gz') try: resultFile.add( folderToArchive, arcname=os.path.basename(folderToArchive)) # put the error log in the archive if(os.path.exists(errorLogPath)): resultFile.add( errorLogPath, arcname=os.path.basename(errorLogPath)) except Exception as err: logger.debug("Err: %s" % str(err)) raise finally: resultFile.close() except Exception as err: logger.debug("Err2: %s" % str(err)) raise # pass # remove temp dir if(os.path.exists(folderToArchive)): shutil.rmtree(folderToArchive) # print the error if there is something if os.path.exists(errorLogPath): logger.error('') logger.error('An error occured while collecting report information : ' + open(errorLogPath, 'r').read()) logger.error('') # delete the errorfile from user's pc if(os.path.exists(errorLogPath)): os.remove(errorLogPath) # return the path to the archive and the path to the upload server return (resultArchive, uploadFileServer, tempDir)
def remove(self, force=False, removeLocal=False): """ Removes file from remote storage ONLY by default """ massStorageConfig = getConfig( 'Output')['MassStorageFile']['uploadOptions'] rm_cmd = massStorageConfig['rm_cmd'] if force == True: _auto_delete = True else: _auto_delete = False for i in self.locations: if not _auto_delete: keyin = None while keyin is None: keyin = raw_input( "Do you want to delete file %s at Location: %s ? [y/n] " % (str(self.namePattern), str(i))) if keyin == 'y': _delete_this = True elif keyin == 'n': _delete_this = False else: logger.warning("y/n please!") keyin = None else: _delete_this = True if _delete_this: logger.info("Deleting File at Location: %s") self.execSyscmdSubprocess('%s %s' % (rm_cmd, i)) self.locations.pop(i) if removeLocal: sourceDir = '' if self.localDir == '': import os _CWD = os.getcwd() if os.path.isfile(os.path.join(_CWD, self.namePattern)): sourceDir = _CWD else: sourceDir = self.localDir _localFile = os.path.join(sourceDir, self.namePattern) if os.path.isfile(_localFile): if force: _actual_delete = True else: keyin = None while keyin is None: keyin = raw_input( "Do you want to remove the local File: %s ? ([y]/n) " % str(_localFile)) if keyin in ['y', '']: _actual_delete = True elif keyin == 'n': _actual_delete = False else: logger.warning("y/n please!") keyin = None if _actual_delete: import time remove_filename = _localFile + "_" + str( time.time()) + '__to_be_deleted_' try: os.rename(_localFile, remove_filename) except OSError as err: logger.warning( "Error in first stage of removing file: %s" % remove_filename) remove_filename = _localFile try: os.remove(remove_filename) except OSError as err: if err.errno != errno.ENOENT: logger.error("Error in removing file: %s" % str(remove_filename)) raise pass return
from __future__ import print_function from Ganga.Utility.Config import getConfig, setSessionValuesFromFiles if __name__ == "__main__": import Ganga.Utility.logging Ganga.Utility.logging.config['Ganga.Utility.Config'] = 'DEBUG' Ganga.Utility.logging.bootstrap() print('Basic Test') import sys c1 = getConfig('C1') c1.setDefaultOptions({'a': 1, 'aa': 'xx'}) c2 = getConfig('C2') c2['b'] = 2 c2['bb'] = 'yy' print('path=', sys.path[0]) setSessionValuesFromFiles([sys.path[0] + '/ConfigTest.ini']) print("C1") print(c1.getEffectiveOptions()) print() print("C2") print(c2.getEffectiveOptions()) assert(c1['a'] == 3)
import time import traceback import sys import Ganga.GPIDev.Lib.Registry.RegistrySlice from Ganga.GPIDev.Lib.Registry.JobRegistry import JobRegistrySliceProxy from Ganga.Core.GangaRepository.Registry import Registry, RegistryError, RegistryKeyError, RegistryAccessError, RegistryFlusher from Ganga.GPIDev.Base.Proxy import stripProxy, getName, isType from Ganga.Utility.ColourText import ANSIMarkup, overview_colours, status_colours, fgcol from Ganga.Utility.logging import getLogger logger = getLogger() from Ganga.Utility.Config import getConfig config = getConfig('Tasks') markup = ANSIMarkup() str_run = markup("run", overview_colours["running"]) str_fail = markup("fail", overview_colours["failed"]) str_hold = markup("hold", overview_colours["hold"]) str_bad = markup("bad", overview_colours["bad"]) class TaskRegistry(Registry): def __init__(self, name, doc): super(TaskRegistry, self).__init__(name, doc) self._main_thread = None
except Exception as err: logger.error("problems with loading plugins for %s -- ignored" % r.name) logger.error('Reason: %s' % str(err)) # ------------------------------------------------------------------------------------ # make all plugins visible in GPI for k in allPlugins.allCategories(): for n in allPlugins.allClasses(k): cls = allPlugins.find(k, n) if not cls._declared_property('hidden'): exportToPublicInterface(n, cls._proxyClass, 'Classes') # ------------------------------------------------------------------------------------ # set the default value for the plugins default_plugins_cfg = getConfig("Plugins") for opt in default_plugins_cfg: try: category, tag = opt.split('_') except ValueError, err: logger.warning("do not understand option %s in [Plugins]", opt) logger.debug('Reason: want %s' % str(err)) else: if tag == 'default': try: allPlugins.setDefault(category, default_plugins_cfg[opt]) except Ganga.Utility.Plugin.PluginManagerError as x: logger.warning('cannot set the default plugin "%s": %s' % (opt, x)) else:
def _parse_options(self): try: parser = self._get_parser() except ApplicationConfigurationError as err: logger.debug("_get_parser Error:\n%s" % str(err)) raise err share_dir = os.path.join( expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], self.is_prepared.name) # Need to remember to create the buffer as the perpare methods returns # are merely copied to the inputsandbox so must alread exist. # share_path = os.path.join(share_dir,'inputsandbox') # if not os.path.isdir(share_path): os.makedirs(share_path) fillPackedSandbox([FileBuffer('options.pkl', parser.opts_pkl_str)], os.path.join( share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name)) # FileBuffer(os.path.join(share_path,'options.pkl'), # parser.opts_pkl_str).create() # self.prep_inputbox.append(File(os.path.join(share_dir,'options.pkl'))) # Check in any input datasets defined in optsfiles and allow them to be # read into the inputdata = parser.get_input_data() if len(inputdata.files) > 0: logger.warning( 'Found inputdataset defined in optsfile, ' 'this will get pickled up and stored in the ' 'prepared state. Any change to the options/data will ' 'therefore require an unprepare first.') logger.warning( 'NOTE: the prefered way of working ' 'is to define inputdata in the job.inputdata field. ') logger.warning( 'Data defined in job.inputdata will superseed optsfile data!') logger.warning( 'Inputdata can be transfered from optsfiles to the job.inputdata field ' 'using job.inputdata = job.application.readInputData(optsfiles)' ) share_path = os.path.join(share_dir, 'inputdata') if not os.path.isdir(share_path): os.makedirs(share_path) f = open(os.path.join(share_path, 'options_data.pkl'), 'w+b') pickle.dump(inputdata, f) f.close() # store the outputsandbox/outputdata defined in the options file # Can remove this when no-longer need to define outputdata in optsfiles # Can remove the if job: when look into how to do prepare for standalone app # move into RuntimeHandler move whole parsing into options maybe? # try and get the job object # not present if preparing standalone app # must change this as prepare should be seperate from the jpb.inputdata share_path = os.path.join(share_dir, 'output') if not os.path.isdir(share_path): os.makedirs(share_path) f = open(os.path.join(share_path, 'options_parser.pkl'), 'w+b') pickle.dump(parser, f) f.close()
import pytest try: import unittest.mock as mock except ImportError: import mock from Ganga.Utility.Config import getConfig, makeConfig makeConfig('defaults_DiracProxy', '') getConfig('defaults_DiracProxy').addOption('group', 'some_group', '') from GangaDirac.Lib.Credentials.DiracProxy import DiracProxy, DiracProxyInfo getConfig('defaults_DiracProxy').setSessionValue('group', 'some_group') class FakeShell(object): """ A mock version of Shell which allows to customise the return values Examples: >>> s = FakeShell() >>> assert s.cmd1.call_count == 0 >>> assert s.cmd1('foo') == (0, '', '') >>> assert s.cmd1.call_count == 1 >>> assert s.cmd1('something -vo') == (0, 'some_group', '') >>> assert s.cmd1.call_count == 2 """ vo = 'some_group' timeleft = 100 def __init__(self): self.env = {} self.cmd1 = mock.Mock(wraps=self._cmd1)
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): """ Prepare the RTHandler in order to submit to the Dirac backend Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appconfig (unknown): Output passed from the application configuration call appmasterconfig (unknown): Output passed from the application master_configure call jobmasterconfig (tuple): Output from the master job prepare step """ cred_req = app.getJobObject().backend.credential_requirements check_creds(cred_req) # NB this needs to be removed safely # Get the inputdata and input/output sandbox in a sorted way inputsandbox, outputsandbox = sandbox_prepare(app, appsubconfig, appmasterconfig, jobmasterconfig) input_data, parametricinput_data = dirac_inputdata(app) # We know we don't need this one inputsandbox = [] job = app.getJobObject() # We can support inputfiles and opts_file here. Locally should be submitted once, remotely can be referenced. all_opts_files = app.getOptsFiles() for opts_file in all_opts_files: if isinstance(opts_file, DiracFile): inputsandbox += ['LFN:' + opts_file.lfn] # Sort out inputfiles we support for file_ in job.inputfiles: if isinstance(file_, DiracFile): inputsandbox += ['LFN:' + file_.lfn] if isinstance(file_, LocalFile): if job.master is not None and file_ not in job.master.inputfiles: shutil.copy( os.path.join(file_.localDir, file_.namePattern), app.getSharedPath()) inputsandbox += [ os.path.join(app.getSharedPath(), file_.namePattern) ] else: logger.error( "Filetype: %s nor currently supported, please contact Ganga Devs if you require support for this with the DIRAC backend" % getName(file_)) raise ApplicationConfigurationError( "Unsupported filetype: %s with DIRAC backend" % getName(file_)) master_job = job.master or job app.uploadedInput = master_job.application.uploadedInput app.jobScriptArchive = master_job.application.jobScriptArchive logger.debug("uploadedInput: %s" % app.uploadedInput) rep_data = app.uploadedInput.getReplicas() logger.debug("Replica info: %s" % rep_data) inputsandbox += ['LFN:' + app.uploadedInput.lfn] inputsandbox += ['LFN:' + app.jobScriptArchive.lfn] logger.debug("Input Sand: %s" % inputsandbox) logger.debug("input_data: %s" % input_data) outputfiles = [ this_file for this_file in job.outputfiles if isinstance(this_file, DiracFile) ] scriptToRun = getScriptName(app) # Already added to sandbox uploaded as LFN # This code deals with the outputfiles as outputsandbox and outputdata for us lhcbdirac_outputfiles = lhcbdirac_outputfile_jdl(outputfiles) # NOTE special case for replicas: replicate string must be empty for no # replication dirac_script = script_generator( lhcbdiracAPI_script_template(), DIRAC_IMPORT= 'from LHCbDIRAC.Interfaces.API.DiracLHCb import DiracLHCb', DIRAC_JOB_IMPORT= 'from LHCbDIRAC.Interfaces.API.LHCbJob import LHCbJob', DIRAC_OBJECT='DiracLHCb()', JOB_OBJECT='LHCbJob()', NAME=mangle_job_name(app), EXE=os.path.join('jobScript', scriptToRun), EXE_ARG_STR='', EXE_LOG_FILE='Ganga_GaudiExec.log', ENVIRONMENT=None, # app.env, INPUTDATA=input_data, PARAMETRIC_INPUTDATA=parametricinput_data, OUTPUT_SANDBOX=API_nullifier(outputsandbox), OUTPUTFILESSCRIPT=lhcbdirac_outputfiles, OUTPUT_PATH="", # job.fqid, OUTPUT_SE=[], PLATFORM=app.platform, SETTINGS=diracAPI_script_settings(app), DIRAC_OPTS=job.backend.diracOpts, REPLICATE='True' if getConfig('DIRAC')['ReplicateOutputData'] else '', # leave the sandbox for altering later as needs # to be done in backend.submit to combine master. # Note only using 2 #s as auto-remove 3 INPUT_SANDBOX=repr([f for f in inputsandbox]), ) # NB # inputsandbox here isn't used by the DIRAC backend as we explicitly define the INPUT_SANDBOX here! # Return the output needed for the backend to submit this job return StandardJobConfig(dirac_script, inputbox=[], outputbox=[])
def __setattr__(self, attr, value): if attr == 'outputfiles': if value != []: if self.outputdata is not None: logger.error( 'ITransform.outputdata is set, you can\'t set ITransform.outputfiles') return elif self.outputsandbox != []: logger.error( 'ITransform.outputsandbox is set, you can\'t set ITransform.outputfiles') return # reduce duplicate values here, leave only duplicates for LCG, # where we can have replicas uniqueValuesDict = [] uniqueValues = [] for val in value: key = '%s%s' % (getName(val), val.namePattern) if key not in uniqueValuesDict: uniqueValuesDict.append(key) uniqueValues.append(val) elif getName(val) == 'LCGSEFile': uniqueValues.append(val) super(ITransform, self).__setattr__(attr, uniqueValues) elif attr == 'inputfiles': if value != []: if self.inputsandbox != []: logger.error( 'ITransform.inputsandbox is set, you can\'t set ITransform.inputfiles') return super(ITransform, self).__setattr__(attr, value) elif attr == 'outputsandbox': if value != []: if getConfig('Output')['ForbidLegacyOutput']: logger.error( 'Use of ITransform.outputsandbox is forbidden, please use ITransform.outputfiles') return if self.outputfiles != []: logger.error( 'ITransform.outputfiles is set, you can\'t set ITransform.outputsandbox') return super(ITransform, self).__setattr__(attr, value) elif attr == 'inputsandbox': if value != []: if getConfig('Output')['ForbidLegacyInput']: logger.error( 'Use of ITransform.inputsandbox is forbidden, please use ITransform.inputfiles') return if self.inputfiles != []: logger.error( 'ITransform.inputfiles is set, you can\'t set ITransform.inputsandbox') return super(ITransform, self).__setattr__(attr, value) elif attr == 'outputdata': if value is not None: if getConfig('Output')['ForbidLegacyOutput']: logger.error( 'Use of ITransform.outputdata is forbidden, please use ITransform.outputfiles') return if self.outputfiles != []: logger.error( 'ITransform.outputfiles is set, you can\'t set ITransform.outputdata') return super(ITransform, self).__setattr__(attr, value) else: super(ITransform, self).__setattr__(attr, value)
def _internal_job_finalisation(job, updated_dirac_status): logger = getLogger() if updated_dirac_status == 'completed': # firstly update job to completing DiracBase._getStateTime(job, 'completing') if job.status in ['removed', 'killed']: return if (job.master and job.master.status in ['removed', 'killed']): return # user changed it under us job.updateStatus('completing') if job.master: job.master.updateMasterJobStatus() import time start = time.time() # contact dirac for information job.backend.normCPUTime = execute('normCPUTime(%d)' % job.backend.id) getSandboxResult = execute("getOutputSandbox(%d,'%s')" % (job.backend.id, job.getOutputWorkspace().getPath())) file_info_dict = execute('getOutputDataInfo(%d)' % job.backend.id) now = time.time() logger.debug('Job ' + job.fqid + ' Time for Dirac metadata : ' + str(now - start)) logger.debug('Job ' + job.fqid + ' OutputDataInfo: ' + str(file_info_dict)) logger.debug('Job ' + job.fqid + ' OutputSandbox: ' + str(getSandboxResult)) # Set DiracFile metadata wildcards = [f.namePattern for f in job.outputfiles.get(DiracFile) if regex.search(f.namePattern) is not None] with open(os.path.join(job.getOutputWorkspace().getPath(), getConfig('Output')['PostProcessLocationsFileName']), 'ab') as postprocesslocationsfile: if not hasattr(file_info_dict, 'keys'): logger.error("Error understanding OutputDataInfo: %s" % str(file_info_dict)) from Ganga.Core.exceptions import GangaException raise GangaException("Error understanding OutputDataInfo: %s" % str(file_info_dict)) for file_name in file_info_dict.get('Value', []): file_name = os.path.basename(file_name) info = file_info_dict.get(file_name) logger.debug("file_name: %s,\tinfo: %s" % (str(file_name), str(info))) valid_wildcards = [wc for wc in wildcards if fnmatch.fnmatch(file_name, wc)] if len(valid_wildcards) == 0: valid_wildcards.append('') if not hasattr(info, 'get'): logger.error("Error getting OutputDataInfo for: %s" % str(job.getFQID('.'))) logger.error("Please check the Dirac Job still exists or attempt a job.backend.reset() to try again!") logger.error("Err: %s" % str(info)) logger.error("file_info_dict: %s" % str(file_info_dict)) from Ganga.Core.exceptions import GangaException raise GangaException("Error getting OutputDataInfo") for wc in valid_wildcards: logger.debug("wildcard: %s" % str(wc)) DiracFileData = 'DiracFile:::%s&&%s->%s:::%s:::%s\n' % (wc, file_name, info.get('LFN', 'Error Getting LFN!'), str(info.get('LOCATIONS', ['NotAvailable'])), info.get('GUID', 'NotAvailable') ) logger.debug("DiracFileData: %s" % str(DiracFileData)) postprocesslocationsfile.write(DiracFileData) # check outputsandbox downloaded correctly if not result_ok(getSandboxResult): logger.warning('Problem retrieving outputsandbox: %s' % str(getSandboxResult)) DiracBase._getStateTime(job, 'failed') if job.status in ['removed', 'killed']: return if (job.master and job.master.status in ['removed', 'killed']): return # user changed it under us job.updateStatus('failed') if job.master: job.master.updateMasterJobStatus() raise BackendError('Problem retrieving outputsandbox: %s' % str(getSandboxResult)) # finally update job to completed DiracBase._getStateTime(job, 'completed') if job.status in ['removed', 'killed']: return if (job.master and job.master.status in ['removed', 'killed']): return # user changed it under us job.updateStatus('completed') if job.master: job.master.updateMasterJobStatus() now = time.time() logger.debug('Job ' + job.fqid + ' Time for complete update : ' + str(now - start)) elif updated_dirac_status == 'failed': # firstly update status to failed DiracBase._getStateTime(job, 'failed') if job.status in ['removed', 'killed']: return if (job.master and job.master.status in ['removed', 'killed']): return # user changed it under us job.updateStatus('failed') if job.master: job.master.updateMasterJobStatus() # if requested try downloading outputsandbox anyway if getConfig('DIRAC')['failed_sandbox_download']: execute("getOutputSandbox(%d,'%s')" % (job.backend.id, job.getOutputWorkspace().getPath())) else: logger.error("Unexpected dirac status '%s' encountered" % updated_dirac_status)
from .GaudiPython import GaudiPython from .Bender import Bender from .BenderScript import BenderScript from .Ostap import Ostap from .BenderBox import BenderModule, BenderRun, OstapRun from .GaudiExec import GaudiExec # Add any additional Packages required by the user in the .gangarc file from Ganga.Utility.Config import getConfig logger = getLogger() logger.debug("User Added Apps") config = getConfig('LHCb') user_added = config['UserAddedApplications'] user_apps = user_added.split(':') if user_apps == user_added and len(user_added) > 0: AppsBaseUtils.addNewLHCbapp(user_apps) for app in user_apps: if len(app) > 0: AppsBaseUtils.addNewLHCbapp(app) logger.debug("Constructing AppsBase Apps") f = open(os.path.join(os.path.dirname(__file__), 'AppsBase.py'), 'r') cls = f.read() f.close() all_apps = '' for app in AppsBaseUtils.available_apps(): if app in dir():
def DiracSplitter(inputs, filesPerJob, maxFiles, ignoremissing): """ Generator that yields a datasets for dirac split jobs """ #logger.debug( "DiracSplitter" ) #logger.debug( "inputs: %s" % str( inputs ) ) split_files = [] i = inputs.__class__() if len(inputs.getLFNs()) != len(inputs.files): raise SplittingError( "Error trying to split dataset using DIRAC backend with non-DiracFile in the inputdata" ) all_files = igroup(inputs.files[:maxFiles], getConfig('DIRAC')['splitFilesChunks'], leftovers=True) #logger.debug( "Looping over all_files" ) #logger.debug( "%s" % str( all_files ) ) for files in all_files: i.files = files LFNsToSplit = i.getLFNs() if (len(LFNsToSplit)) > 1: result = execute('splitInputData(%s, %d)' % (i.getLFNs(), filesPerJob)) if not result_ok(result): logger.error('DIRAC:: Error splitting files: %s' % str(result)) raise SplittingError('Error splitting files.') split_files += result.get('Value', []) else: split_files = [LFNsToSplit] if len(split_files) == 0: raise SplittingError('An unknown error occured.') # FIXME # check that all files were available on the grid big_list = [] for l in split_files: big_list.extend(l) diff = set(inputs.getFileNames()[:maxFiles]).difference(big_list) if len(diff) > 0: for f in diff: logger.warning('Ignored file: %s' % f) if not ignoremissing: raise SplittingError('Some files not found!') ### logger.debug("Split Files: %s" % str(split_files)) for _dataset in split_files: dataset = [] for _lfn in _dataset: dataset.append(DiracFile(lfn=_lfn)) yield dataset