return cmp(x[DataProvider.Dataset], y[DataProvider.Dataset])
		oldBlocks.sort(cmpBlock)
		newBlocks.sort(cmpBlock)

		def onMatchingBlock(blocksAdded, blocksMissing, blocksMatching, oldBlock, newBlock):
			# Compare different files according to their name - NOT full content
			def cmpFiles(x, y):
				return cmp(x[DataProvider.URL], y[DataProvider.URL])
			oldBlock[DataProvider.FileList].sort(cmpFiles)
			newBlock[DataProvider.FileList].sort(cmpFiles)

			def onMatchingFile(filesAdded, filesMissing, filesMatched, oldFile, newFile):
				filesMatched.append((oldFile, newFile))

			(filesAdded, filesMissing, filesMatched) = \
				utils.DiffLists(oldBlock[DataProvider.FileList], newBlock[DataProvider.FileList], cmpFiles, onMatchingFile, isSorted = True)
			if filesAdded: # Create new block for added files in an existing block
				tmpBlock = copy.copy(newBlock)
				tmpBlock[DataProvider.FileList] = filesAdded
				tmpBlock[DataProvider.NEntries] = sum(map(lambda x: x[DataProvider.NEntries], filesAdded))
				blocksAdded.append(tmpBlock)
			blocksMatching.append((oldBlock, newBlock, filesMissing, filesMatched))

		return utils.DiffLists(oldBlocks, newBlocks, cmpBlock, onMatchingBlock, isSorted = True)
	resyncSources = staticmethod(resyncSources)

DataProvider.providers = {}
# To uncover errors, the enums of DataProvider / DataSplitter do *NOT* match type wise
utils.makeEnum(['NEntries', 'BlockName', 'Dataset', 'Locations', 'URL', 'FileList',
	'Nickname', 'DatasetID', 'Metadata', 'Provider', 'ResyncInfo'], DataProvider)
	def process(self, dn):
		return WMS.parseJobInfo(os.path.join(dn, 'job.info'))

class FileInfoProcessor(JobInfoProcessor):
	def process(self, dn):
		jobInfo = JobInfoProcessor.process(self, dn)
		if jobInfo:
			(jobNumStored, jobExitCode, jobData) = jobInfo
			result = {}
			# parse old job info data format for files
			oldFileFormat = [FileInfoProcessor.Hash, FileInfoProcessor.NameLocal, FileInfoProcessor.NameDest, FileInfoProcessor.Path]
			for (fileKey, fileData) in filter(lambda (key, value): key.startswith('FILE'), jobData.items()):
				fileIdx = fileKey.replace('FILE', '').rjust(1, '0')
				result[int(fileIdx)] = dict(zip(oldFileFormat, fileData.strip('"').split('  ')))
			# parse new job info data format
			for (fileKey, fileData) in filter(lambda (key, value): key.startswith('OUTPUT_FILE'), jobData.items()):
				(fileIdx, fileProperty) = fileKey.replace('OUTPUT_FILE_', '').split('_')
				if isinstance(fileData, str):
					fileData = fileData.strip('"')
				result.setdefault(int(fileIdx), {})[FileInfoProcessor.str2enum(fileProperty)] = fileData
			return result.values()
utils.makeEnum(['Hash', 'NameLocal', 'NameDest', 'Path'], FileInfoProcessor)

class TaskOutputProcessor(OutputProcessor):
	def __init__(self, task):
		self._task = task

class SandboxProcessor(TaskOutputProcessor):
	def process(self, dn):
		return True
示例#3
0
		return self.dict.get(key, default)


	def update(self, state):
		self.state = state
		self.changed = time.time()
		self.history[self.attempt] = self.dict.get('dest', 'N/A')


	def assignId(self, wmsId):
		self.dict['legacy'] = None # Legacy support
		self.wmsId = wmsId
		self.attempt = self.attempt + 1
		self.submitted = time.time()

utils.makeEnum(['INIT', 'SUBMITTED', 'DISABLED', 'READY', 'WAITING', 'QUEUED', 'ABORTED',
		'RUNNING', 'CANCELLED', 'DONE', 'FAILED', 'SUCCESS'], Job, useHash = False)


class JobClass:
	mkJobClass = lambda *fList: (reduce(operator.add, map(lambda f: 1 << f, fList)), fList)
	ATWMS = mkJobClass(Job.SUBMITTED, Job.WAITING, Job.READY, Job.QUEUED)
	RUNNING = mkJobClass(Job.RUNNING)
	PROCESSING = mkJobClass(Job.SUBMITTED, Job.WAITING, Job.READY, Job.QUEUED, Job.RUNNING)
	READY = mkJobClass(Job.INIT, Job.FAILED, Job.ABORTED, Job.CANCELLED)
	DONE = mkJobClass(Job.DONE)
	SUCCESS = mkJobClass(Job.SUCCESS)
	DISABLED = mkJobClass(Job.DISABLED)
	ENDSTATE = mkJobClass(Job.SUCCESS, Job.DISABLED)
	PROCESSED = mkJobClass(Job.SUCCESS, Job.FAILED, Job.CANCELLED, Job.ABORTED)

from hpfwk import AbstractError, Plugin
from python_compat import next

def fast_search(lst, cmp_op):
	def bisect_left_cmp(lst, cmp_op):
		(lo, hi) = (0, len(lst))
		while lo < hi:
			mid = (lo + hi) / 2
			if cmp_op(lst[mid]) < 0: lo = mid + 1
			else: hi = mid
		return lo
	idx = bisect_left_cmp(lst, cmp_op)
	if idx < len(lst) and cmp_op(lst[idx]) == 0:
		return lst[idx]

ResyncMode = utils.makeEnum(['disable', 'complete', 'changed', 'ignore']) # prio: "disable" overrides "complete", etc.
ResyncMode.noChanged = [ResyncMode.disable, ResyncMode.complete, ResyncMode.ignore]
ResyncOrder = utils.makeEnum(['append', 'preserve', 'fillgap', 'reorder']) # reorder mechanism

class DataSplitter(Plugin):
	def __init__(self, config):
		self.config = config
		self.splitSource = None
		self._protocol = {}
		# Resync settings:
		self.interactive = config.getBool('resync interactive', False, onChange = None)
		#   behaviour in case of event size changes
		self.mode_removed = config.getEnum('mode removed', ResyncMode, ResyncMode.complete, subset = ResyncMode.noChanged)
		self.mode_expanded = config.getEnum('mode expand', ResyncMode, ResyncMode.changed)
		self.mode_shrunken = config.getEnum('mode shrink', ResyncMode, ResyncMode.changed)
		self.mode_new = config.getEnum('mode new', ResyncMode, ResyncMode.complete, subset = [ResyncMode.complete, ResyncMode.ignore])
#-#
#-#  Unless required by applicable law or agreed to in writing, software
#-#  distributed under the License is distributed on an "AS IS" BASIS,
#-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#-#  See the License for the specific language governing permissions and
#-#  limitations under the License.

from grid_control import utils
from hpfwk import APIError, NestedException
from python_compat import set, sorted

class ConfigError(NestedException):
	pass

# Placeholder to specify a non-existing default
noDefault = utils.makeEnum(['noDefault'])

# return canonized section or option string
def standardConfigForm(value):
	if value != None:
		if not isinstance(value, list):
			value = [value]
		return map(lambda x: str(x).strip().lower(), value)

def multi_line_format(value):
	value_list = filter(lambda x: x != '', map(str.strip, value.strip().splitlines()))
	if len(value_list) > 1:
		return '\n\t%s' % str.join('\n\t', value_list)
	return str.join('\n\t', value_list)

示例#6
0
#-#  You may obtain a copy of the License at
#-#
#-#      http://www.apache.org/licenses/LICENSE-2.0
#-#
#-#  Unless required by applicable law or agreed to in writing, software
#-#  distributed under the License is distributed on an "AS IS" BASIS,
#-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#-#  See the License for the specific language governing permissions and
#-#  limitations under the License.

import logging
from grid_control import utils
from grid_control.config.config_entry import ConfigEntry, ConfigError, noDefault, standardConfigForm
from python_compat import sorted

selectorUnchanged = utils.makeEnum(['selector_unchanged'])

class ConfigView(object):
	def __init__(self, name, parent = None):
		if not parent:
			parent = self
		self._parent = parent
		self.pathDict = {}
		self.pathDict.update(parent.pathDict) # inherit path dict from parent
		self.setConfigName(name)

	def setConfigName(self, name):
		self.configName = name
		self._log = logging.getLogger('config.%s' % name)

	def getView(self, setSections = selectorUnchanged, **kwargs):
示例#7
0
	def parseJobInfo(fn):
		if not os.path.exists(fn):
			return utils.eprint('Warning: "%s" does not exist.' % fn)
		try:
			info_content = open(fn, 'r').read()
		except Exception, ex:
			return utils.eprint('Warning: Unable to read "%s"!\n%s' % (fn, str(ex)))
		if not info_content:
			return utils.eprint('Warning: "%s" is empty!' % fn)
		try:
			data = utils.DictFormat().parse(info_content, keyParser = {None: str})
			return (data['JOBID'], data['EXITCODE'], data)
		except Exception:
			return utils.eprint('Warning: Unable to parse "%s"!' % fn)
	parseJobInfo = staticmethod(parseJobInfo)
utils.makeEnum(['WALLTIME', 'CPUTIME', 'MEMORY', 'CPUS', 'BACKEND', 'SITES', 'QUEUES', 'SOFTWARE', 'STORAGE'], WMS)


class InactiveWMS(WMS):
	def __init__(self, config, wmsName):
		WMS.__init__(self, config, wmsName)
		self._token = config.getCompositePlugin(['access token', 'proxy'], 'TrivialAccessToken',
			'MultiAccessToken', cls = AccessToken, inherit = True, tags = [self]).getInstance()

	def canSubmit(self, neededTime, canCurrentlySubmit):
		return True

	def getAccessToken(self, wmsId):
		return self._token

	def deployTask(self, task, monitor):
示例#8
0
from grid_control import QM, LoadableObject, AbstractError, RuntimeError, utils, ConfigError, Config, noDefault
from provider_base import DataProvider

def fast_search(lst, cmp_op):
	def bisect_left_cmp(lst, cmp_op):
		(lo, hi) = (0, len(lst))
		while lo < hi:
			mid = (lo + hi) / 2
			if cmp_op(lst[mid]) < 0: lo = mid + 1
			else: hi = mid
		return lo
	idx = bisect_left_cmp(lst, cmp_op)
	if idx < len(lst) and cmp_op(lst[idx]) == 0:
		return lst[idx]

ResyncMode = utils.makeEnum(['disable', 'complete', 'changed', 'ignore']) # prio: "disable" overrides "complete", etc.
ResyncMode.noChanged = [ResyncMode.disable, ResyncMode.complete, ResyncMode.ignore]
ResyncOrder = utils.makeEnum(['append', 'preserve', 'fillgap', 'reorder']) # reorder mechanism

class DataSplitter(LoadableObject):
	splitInfos = ['Dataset', 'Locations', 'NEntries', 'Skipped', 'FileList', 'Nickname', 'DatasetID',
		'CommonPrefix', 'Invalid', 'BlockName', 'MetadataHeader', 'Metadata', 'Comment']
	for idx, splitInfo in enumerate(splitInfos):
		locals()[splitInfo] = idx

	def __init__(self, config):
		self.config = config.addSections(['dataset'])
		self.splitSource = None
		self._protocol = {}

		def getResyncConfig(item, default, opts, cls = ResyncMode):
示例#9
0
#-#  You may obtain a copy of the License at
#-#
#-#      http://www.apache.org/licenses/LICENSE-2.0
#-#
#-#  Unless required by applicable law or agreed to in writing, software
#-#  distributed under the License is distributed on an "AS IS" BASIS,
#-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#-#  See the License for the specific language governing permissions and
#-#  limitations under the License.

from python_compat import sorted, set
from grid_control import APIError, AbstractError, ConfigError, RethrowError, utils, QM
import logging

# Placeholder to specify a non-existing default or not-set value
noDefault = utils.makeEnum(['noDefault'])
notSet = utils.makeEnum(['notSet'])

# return canonized section or option string
standardConfigForm = lambda x: str(x).strip().lower()

def multi_line_format(value):
	value_list = map(str.strip, filter(lambda x: x.strip() != '', value.strip().splitlines()))
	if len(value_list) > 1:
		return '\n\t%s' % str.join('\n\t', value_list)
	return ' %s' % str.join('\n\t', value_list)

# Holder of config information
class ConfigEntry(object):
	def __init__(self, value, source, section = None, option = None, default = notSet, accessed = False):
		(self.section, self.option) = (section, option)