Example #1
0
	def __init__(self, config, name):
		NamedPlugin.__init__(self, config, name)
		self._workDir = config.getWorkPath()
		# Initialise task module
		self.task = config.getPlugin(['task', 'module'], cls = TaskModule, tags = [self]).getInstance()
		utils.vprint('Current task ID: %s' % self.task.taskID, -1)
		utils.vprint('Task started on %s' % self.task.taskDate, -1)

		# Initialise monitoring module
		self.monitor = config.getCompositePlugin('monitor', 'scripts', 'MultiMonitor',
			cls = Monitoring, tags = [self, self.task]).getInstance(self.task)

		# Initialise workload management interface
		self.wms = config.getCompositePlugin('backend', 'grid', 'MultiWMS',
			cls = WMS, tags = [self, self.task]).getInstance()

		# Initialise job database
		jobManagerCls = config.getPlugin('job manager', 'SimpleJobManager',
			cls = JobManager, tags = [self, self.task, self.wms])
		self.jobManager = jobManagerCls.getInstance(self.task, self.monitor)

		# Prepare work package
		self.wms.deployTask(self.task, self.monitor)

		configJobs = config.changeView(viewClass = TaggedConfigView, addSections = ['jobs'], addTags = [self])
		self._actionList = configJobs.getList('action', ['check', 'retrieve', 'submit'], onChange = None)
		self.runContinuous = configJobs.getBool('continuous', False, onChange = None)

		self._checkSpace = config.getInt('workdir space', 10, onChange = None)
		self._submitFlag = config.getBool('submission', True, onChange = None)
		guiClass = config.getPlugin('gui', 'SimpleConsole', cls = GUI, onChange = None)
		self._gui = guiClass.getInstance(config, self)
Example #2
0
	def __init__(self, config, name):
		NamedPlugin.__init__(self, config, name)
		initSandbox = changeInitNeeded('sandbox')

		# Task requirements
		configJobs = config.changeView(viewClass = TaggedConfigView, addSections = ['jobs'], addTags = [self]) # Move this into parameter manager?
		self.wallTime = configJobs.getTime('wall time', onChange = None)
		self.cpuTime = configJobs.getTime('cpu time', self.wallTime, onChange = None)
		self.cpus = configJobs.getInt('cpus', 1, onChange = None)
		self.memory = configJobs.getInt('memory', -1, onChange = None)
		self.nodeTimeout = configJobs.getTime('node timeout', -1, onChange = initSandbox)

		# Compute / get task ID
		self.taskID = config.get('task id', 'GC' + md5(str(time())).hexdigest()[:12], persistent = True)
		self.taskDate = config.get('task date', strftime('%Y-%m-%d'), persistent = True, onChange = initSandbox)
		self.taskConfigName = config.getConfigName()

		# Storage setup
		configStorage = config.changeView(viewClass = TaggedConfigView,
			setClasses = None, setNames = None, addSections = ['storage'], addTags = [self])
		self.taskVariables = {
			# Space limits
			'SCRATCH_UL': configStorage.getInt('scratch space used', 5000, onChange = initSandbox),
			'SCRATCH_LL': configStorage.getInt('scratch space left', 1, onChange = initSandbox),
			'LANDINGZONE_UL': configStorage.getInt('landing zone space used', 100, onChange = initSandbox),
			'LANDINGZONE_LL': configStorage.getInt('landing zone space left', 1, onChange = initSandbox),
		}
		configStorage.set('se output pattern', 'job_@GC_JOB_ID@_@X@')
		self.seMinSize = configStorage.getInt('se min size', -1, onChange = initSandbox)

		self.sbInputFiles = config.getPaths('input files', [], onChange = initSandbox)
		self.sbOutputFiles = config.getList('output files', [], onChange = initSandbox)
		self.gzipOut = config.getBool('gzip output', True, onChange = initSandbox)

		self.substFiles = config.getList('subst files', [], onChange = initSandbox)
		self.dependencies = map(str.lower, config.getList('depends', [], onChange = initSandbox))

		# Get error messages from gc-run.lib comments
		self.errorDict = dict(self.updateErrorDict(utils.pathShare('gc-run.lib')))

		# Init parameter source manager
		pm = config.getPlugin('parameter factory', 'SimpleParameterFactory',
			cls = ParameterFactory, inherit = True).getInstance()
		configParam = config.changeView(viewClass = TaggedConfigView, addSections = ['parameters'], addTags = [self])
		self.setupJobParameters(configParam, pm)
		self.source = pm.getSource(configParam)
	def __init__(self, config, name, task, eventhandler):
		NamedPlugin.__init__(self, config, name)
		(self._task, self._eventhandler) = (task, eventhandler)
		self.jobLimit = config.getInt('jobs', -1, onChange = None)
		selected = JobSelector.create(config.get('selected', '', onChange = None), task = self._task)
		jobDBClass = config.getPlugin('jobdb', 'JobDB', cls = JobDB)
		self.jobDB = jobDBClass.getInstance(config, self.getMaxJobs(self._task), selected)
		self.disableLog = config.getWorkPath('disabled')
		outputProcessorClass = config.getPlugin('output processor', 'SandboxProcessor', cls = TaskOutputProcessor)
		self._outputProcessor = outputProcessorClass.getInstance(task)

		self.timeout = config.getTime('queue timeout', -1, onChange = None)
		self.inFlight = config.getInt('in flight', -1, onChange = None)
		self.inQueue = config.getInt('in queue', -1, onChange = None)
		self.doShuffle = config.getBool('shuffle', False, onChange = None)
		self.maxRetry = config.getInt('max retry', -1, onChange = None)
		self.continuous = config.getBool('continuous', False, onChange = None)
		self._reportClass = config.getPlugin('abort report', 'LocationReport', cls = Report, onChange = None)
Example #4
0
	def __init__(self, config, name, userOpt, itemName, discoverFun):
		NamedPlugin.__init__(self, config, name)
		(self._itemsStart, self._itemsDiscovered, self._itemName) = (None, False, itemName)
		self._nEntries = config.getInt('%s entries' % userOpt, 0, onChange = None)
		self._nRandom = config.getBool('%s randomize' % userOpt, False, onChange = None)
Example #5
0
	def __init__(self, config, name, task, submodules = []):
		NamedPlugin.__init__(self, config, name)
		self._log = logging.getLogger('monitoring')
		(self.config, self.task, self.submodules) = (config, task, submodules)
	def __init__(self, config, name):
		NamedPlugin.__init__(self, config, name)
		self.adapter = config.get('parameter adapter', 'TrackedParameterAdapter')
		self.paramConfig = ParameterConfig(config.changeView(setSections = ['parameters']), self.adapter != 'TrackedParameterAdapter')
Example #7
0
	def __init__(self, config, wmsName):
		wmsName = utils.QM(wmsName, wmsName, self.__class__.__name__).upper().replace('.', '_')
		NamedPlugin.__init__(self, config, wmsName)
		(self.config, self.wmsName) = (config, wmsName)
		self._wait_idle = config.getInt('wait idle', 60, onChange = None)
		self._wait_work = config.getInt('wait work', 10, onChange = None)