return cmp(x[DataProvider.Dataset], y[DataProvider.Dataset])
oldBlocks.sort(cmpBlock)
newBlocks.sort(cmpBlock)
def onMatchingBlock(blocksAdded, blocksMissing, blocksMatching, oldBlock, newBlock):
# Compare different files according to their name - NOT full content
def cmpFiles(x, y):
return cmp(x[DataProvider.URL], y[DataProvider.URL])
oldBlock[DataProvider.FileList].sort(cmpFiles)
newBlock[DataProvider.FileList].sort(cmpFiles)
def onMatchingFile(filesAdded, filesMissing, filesMatched, oldFile, newFile):
filesMatched.append((oldFile, newFile))
(filesAdded, filesMissing, filesMatched) = \
utils.DiffLists(oldBlock[DataProvider.FileList], newBlock[DataProvider.FileList], cmpFiles, onMatchingFile, isSorted = True)
if filesAdded: # Create new block for added files in an existing block
tmpBlock = copy.copy(newBlock)
tmpBlock[DataProvider.FileList] = filesAdded
tmpBlock[DataProvider.NEntries] = sum(map(lambda x: x[DataProvider.NEntries], filesAdded))
blocksAdded.append(tmpBlock)
blocksMatching.append((oldBlock, newBlock, filesMissing, filesMatched))
return utils.DiffLists(oldBlocks, newBlocks, cmpBlock, onMatchingBlock, isSorted = True)
resyncSources = staticmethod(resyncSources)
DataProvider.providers = {}
# To uncover errors, the enums of DataProvider / DataSplitter do *NOT* match type wise
utils.makeEnum(['NEntries', 'BlockName', 'Dataset', 'Locations', 'URL', 'FileList',
'Nickname', 'DatasetID', 'Metadata', 'Provider', 'ResyncInfo'], DataProvider)
def process(self, dn):
return WMS.parseJobInfo(os.path.join(dn, 'job.info'))
class FileInfoProcessor(JobInfoProcessor):
def process(self, dn):
jobInfo = JobInfoProcessor.process(self, dn)
if jobInfo:
(jobNumStored, jobExitCode, jobData) = jobInfo
result = {}
# parse old job info data format for files
oldFileFormat = [FileInfoProcessor.Hash, FileInfoProcessor.NameLocal, FileInfoProcessor.NameDest, FileInfoProcessor.Path]
for (fileKey, fileData) in filter(lambda (key, value): key.startswith('FILE'), jobData.items()):
fileIdx = fileKey.replace('FILE', '').rjust(1, '0')
result[int(fileIdx)] = dict(zip(oldFileFormat, fileData.strip('"').split(' ')))
# parse new job info data format
for (fileKey, fileData) in filter(lambda (key, value): key.startswith('OUTPUT_FILE'), jobData.items()):
(fileIdx, fileProperty) = fileKey.replace('OUTPUT_FILE_', '').split('_')
if isinstance(fileData, str):
fileData = fileData.strip('"')
result.setdefault(int(fileIdx), {})[FileInfoProcessor.str2enum(fileProperty)] = fileData
return result.values()
utils.makeEnum(['Hash', 'NameLocal', 'NameDest', 'Path'], FileInfoProcessor)
class TaskOutputProcessor(OutputProcessor):
def __init__(self, task):
self._task = task
class SandboxProcessor(TaskOutputProcessor):
def process(self, dn):
return True
return self.dict.get(key, default)
def update(self, state):
self.state = state
self.changed = time.time()
self.history[self.attempt] = self.dict.get('dest', 'N/A')
def assignId(self, wmsId):
self.dict['legacy'] = None # Legacy support
self.wmsId = wmsId
self.attempt = self.attempt + 1
self.submitted = time.time()
utils.makeEnum(['INIT', 'SUBMITTED', 'DISABLED', 'READY', 'WAITING', 'QUEUED', 'ABORTED',
'RUNNING', 'CANCELLED', 'DONE', 'FAILED', 'SUCCESS'], Job, useHash = False)
class JobClass:
mkJobClass = lambda *fList: (reduce(operator.add, map(lambda f: 1 << f, fList)), fList)
ATWMS = mkJobClass(Job.SUBMITTED, Job.WAITING, Job.READY, Job.QUEUED)
RUNNING = mkJobClass(Job.RUNNING)
PROCESSING = mkJobClass(Job.SUBMITTED, Job.WAITING, Job.READY, Job.QUEUED, Job.RUNNING)
READY = mkJobClass(Job.INIT, Job.FAILED, Job.ABORTED, Job.CANCELLED)
DONE = mkJobClass(Job.DONE)
SUCCESS = mkJobClass(Job.SUCCESS)
DISABLED = mkJobClass(Job.DISABLED)
ENDSTATE = mkJobClass(Job.SUCCESS, Job.DISABLED)
PROCESSED = mkJobClass(Job.SUCCESS, Job.FAILED, Job.CANCELLED, Job.ABORTED)
from hpfwk import AbstractError, Plugin
from python_compat import next
def fast_search(lst, cmp_op):
def bisect_left_cmp(lst, cmp_op):
(lo, hi) = (0, len(lst))
while lo < hi:
mid = (lo + hi) / 2
if cmp_op(lst[mid]) < 0: lo = mid + 1
else: hi = mid
return lo
idx = bisect_left_cmp(lst, cmp_op)
if idx < len(lst) and cmp_op(lst[idx]) == 0:
return lst[idx]
ResyncMode = utils.makeEnum(['disable', 'complete', 'changed', 'ignore']) # prio: "disable" overrides "complete", etc.
ResyncMode.noChanged = [ResyncMode.disable, ResyncMode.complete, ResyncMode.ignore]
ResyncOrder = utils.makeEnum(['append', 'preserve', 'fillgap', 'reorder']) # reorder mechanism
class DataSplitter(Plugin):
def __init__(self, config):
self.config = config
self.splitSource = None
self._protocol = {}
# Resync settings:
self.interactive = config.getBool('resync interactive', False, onChange = None)
# behaviour in case of event size changes
self.mode_removed = config.getEnum('mode removed', ResyncMode, ResyncMode.complete, subset = ResyncMode.noChanged)
self.mode_expanded = config.getEnum('mode expand', ResyncMode, ResyncMode.changed)
self.mode_shrunken = config.getEnum('mode shrink', ResyncMode, ResyncMode.changed)
self.mode_new = config.getEnum('mode new', ResyncMode, ResyncMode.complete, subset = [ResyncMode.complete, ResyncMode.ignore])
#-#
#-# Unless required by applicable law or agreed to in writing, software
#-# distributed under the License is distributed on an "AS IS" BASIS,
#-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#-# See the License for the specific language governing permissions and
#-# limitations under the License.
from grid_control import utils
from hpfwk import APIError, NestedException
from python_compat import set, sorted
class ConfigError(NestedException):
pass
# Placeholder to specify a non-existing default
noDefault = utils.makeEnum(['noDefault'])
# return canonized section or option string
def standardConfigForm(value):
if value != None:
if not isinstance(value, list):
value = [value]
return map(lambda x: str(x).strip().lower(), value)
def multi_line_format(value):
value_list = filter(lambda x: x != '', map(str.strip, value.strip().splitlines()))
if len(value_list) > 1:
return '\n\t%s' % str.join('\n\t', value_list)
return str.join('\n\t', value_list)
#-# You may obtain a copy of the License at
#-#
#-# http://www.apache.org/licenses/LICENSE-2.0
#-#
#-# Unless required by applicable law or agreed to in writing, software
#-# distributed under the License is distributed on an "AS IS" BASIS,
#-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#-# See the License for the specific language governing permissions and
#-# limitations under the License.
import logging
from grid_control import utils
from grid_control.config.config_entry import ConfigEntry, ConfigError, noDefault, standardConfigForm
from python_compat import sorted
selectorUnchanged = utils.makeEnum(['selector_unchanged'])
class ConfigView(object):
def __init__(self, name, parent = None):
if not parent:
parent = self
self._parent = parent
self.pathDict = {}
self.pathDict.update(parent.pathDict) # inherit path dict from parent
self.setConfigName(name)
def setConfigName(self, name):
self.configName = name
self._log = logging.getLogger('config.%s' % name)
def getView(self, setSections = selectorUnchanged, **kwargs):
def parseJobInfo(fn):
if not os.path.exists(fn):
return utils.eprint('Warning: "%s" does not exist.' % fn)
try:
info_content = open(fn, 'r').read()
except Exception, ex:
return utils.eprint('Warning: Unable to read "%s"!\n%s' % (fn, str(ex)))
if not info_content:
return utils.eprint('Warning: "%s" is empty!' % fn)
try:
data = utils.DictFormat().parse(info_content, keyParser = {None: str})
return (data['JOBID'], data['EXITCODE'], data)
except Exception:
return utils.eprint('Warning: Unable to parse "%s"!' % fn)
parseJobInfo = staticmethod(parseJobInfo)
utils.makeEnum(['WALLTIME', 'CPUTIME', 'MEMORY', 'CPUS', 'BACKEND', 'SITES', 'QUEUES', 'SOFTWARE', 'STORAGE'], WMS)
class InactiveWMS(WMS):
def __init__(self, config, wmsName):
WMS.__init__(self, config, wmsName)
self._token = config.getCompositePlugin(['access token', 'proxy'], 'TrivialAccessToken',
'MultiAccessToken', cls = AccessToken, inherit = True, tags = [self]).getInstance()
def canSubmit(self, neededTime, canCurrentlySubmit):
return True
def getAccessToken(self, wmsId):
return self._token
def deployTask(self, task, monitor):
from grid_control import QM, LoadableObject, AbstractError, RuntimeError, utils, ConfigError, Config, noDefault
from provider_base import DataProvider
def fast_search(lst, cmp_op):
def bisect_left_cmp(lst, cmp_op):
(lo, hi) = (0, len(lst))
while lo < hi:
mid = (lo + hi) / 2
if cmp_op(lst[mid]) < 0: lo = mid + 1
else: hi = mid
return lo
idx = bisect_left_cmp(lst, cmp_op)
if idx < len(lst) and cmp_op(lst[idx]) == 0:
return lst[idx]
ResyncMode = utils.makeEnum(['disable', 'complete', 'changed', 'ignore']) # prio: "disable" overrides "complete", etc.
ResyncMode.noChanged = [ResyncMode.disable, ResyncMode.complete, ResyncMode.ignore]
ResyncOrder = utils.makeEnum(['append', 'preserve', 'fillgap', 'reorder']) # reorder mechanism
class DataSplitter(LoadableObject):
splitInfos = ['Dataset', 'Locations', 'NEntries', 'Skipped', 'FileList', 'Nickname', 'DatasetID',
'CommonPrefix', 'Invalid', 'BlockName', 'MetadataHeader', 'Metadata', 'Comment']
for idx, splitInfo in enumerate(splitInfos):
locals()[splitInfo] = idx
def __init__(self, config):
self.config = config.addSections(['dataset'])
self.splitSource = None
self._protocol = {}
def getResyncConfig(item, default, opts, cls = ResyncMode):
#-# You may obtain a copy of the License at
#-#
#-# http://www.apache.org/licenses/LICENSE-2.0
#-#
#-# Unless required by applicable law or agreed to in writing, software
#-# distributed under the License is distributed on an "AS IS" BASIS,
#-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#-# See the License for the specific language governing permissions and
#-# limitations under the License.
from python_compat import sorted, set
from grid_control import APIError, AbstractError, ConfigError, RethrowError, utils, QM
import logging
# Placeholder to specify a non-existing default or not-set value
noDefault = utils.makeEnum(['noDefault'])
notSet = utils.makeEnum(['notSet'])
# return canonized section or option string
standardConfigForm = lambda x: str(x).strip().lower()
def multi_line_format(value):
value_list = map(str.strip, filter(lambda x: x.strip() != '', value.strip().splitlines()))
if len(value_list) > 1:
return '\n\t%s' % str.join('\n\t', value_list)
return ' %s' % str.join('\n\t', value_list)
# Holder of config information
class ConfigEntry(object):
def __init__(self, value, source, section = None, option = None, default = notSet, accessed = False):
(self.section, self.option) = (section, option)