def _parseTickets(self, cached = True):
# Return cached results if requested
if cached and self._cache:
return self._cache
# Call klist and parse results
proc = LocalProcess(self._klistExec)
self._cache = {}
try:
for line in proc.stdout.iter(timeout = 10):
if line.count('@') and (line.count(':') > 1):
issued_expires, principal = rsplit(line, ' ', 1)
issued_expires = issued_expires.replace('/', ' ').split()
assert(len(issued_expires) % 2 == 0)
issued_str = str.join(' ', issued_expires[:int(len(issued_expires) / 2)])
expires_str = str.join(' ', issued_expires[int(len(issued_expires) / 2):])
parseDate = lambda value, format: time.mktime(time.strptime(value, format))
if expires_str.count(' ') == 3:
if len(expires_str.split()[2]) == 2:
expires = parseDate(expires_str, '%m %d %y %H:%M:%S')
else:
expires = parseDate(expires_str, '%m %d %Y %H:%M:%S')
elif expires_str.count(' ') == 2: # year information is missing
currentYear = int(time.strftime('%Y'))
expires = parseDate(expires_str + ' %d' % currentYear, '%b %d %H:%M:%S %Y')
issued = parseDate(issued_str + ' %d' % currentYear, '%b %d %H:%M:%S %Y')
if expires < issued: # wraparound at new year
expires = parseDate(expires_str + ' %d' % (currentYear + 1), '%b %d %H:%M:%S %Y')
self._cache.setdefault('tickets', {})[principal] = expires
elif line.count(':') == 1:
key, value = lmap(str.strip, line.split(':', 1))
self._cache[key.lower()] = value
except Exception:
raise AccessTokenError('Unable to parse kerberos ticket information!')
proc.status_raise(timeout = 0)
return self._cache
def _parseFile(self, iterator):
block = None
for idx, line in enumerate(iterator):
try:
# Found start of block:
line = line.strip()
if line.startswith(';'):
continue
elif line.startswith('['):
if block:
yield self._finishBlock(block)
block = self._createBlock(line)
elif line != '':
tmp = lmap(str.strip, utils.QM('[' in line, line.split(' = ', 1), rsplit(line, '=', 1)))
if len(tmp) != 2:
raise DatasetError('Malformed entry in dataset file:\n%s' % line)
key, value = tmp
handlerInfo = self._handleEntry.get(key.lower(), None)
if handlerInfo:
(prop, parser, msg) = handlerInfo
block[prop] = try_apply(value, parser, msg)
else:
block[DataProvider.FileList].append(self._parseEntry(block, key, value))
except Exception:
raise DatasetError('Unable to parse %s:%d\n\t%s' % (repr(self._filename), idx, repr(line)))
if block:
yield self._finishBlock(block)
def parseLine(idx, line):
exceptionText = 'Unable to parse config file %s:%d\n\t%r' % (configFile, idx, line)
try:
line = rsplit(line, ';', 1)[0].rstrip()
except Exception:
raise ConfigError(exceptionText + '\nUnable to strip comments!')
exceptionText = 'Unable to parse config file %s:%d\n\t%r' % (configFile, idx, line)
if not line.strip() or line.lstrip().startswith('#'): # skip empty lines or comment lines
return
elif line[0].isspace():
try:
self._currentValue += '\n' + line.strip()
self._currentLines += [idx]
except Exception:
raise ConfigError(exceptionText + '\nInvalid indentation!')
elif line.startswith('['):
if self._currentOption:
storeOption()
try:
self._currentSection = line[1:line.index(']')].strip()
parseLine(idx, line[line.index(']') + 1:].strip())
except Exception:
raise ConfigError(exceptionText + '\nUnable to parse config section!')
elif '=' in line:
if self._currentOption:
storeOption()
try:
(self._currentOption, self._currentValue) = map(str.strip, line.split('=', 1))
self._currentLines = [idx]
except Exception:
raise ConfigError(exceptionText + '\nUnable to parse config option!')
else:
raise ConfigError(exceptionText + '\nPlease use "key = value" syntax or indent values!')
def _create_blocks(self, iterable):
block = None
for idx, line in enumerate(iterable):
try:
# Found start of block:
line = line.strip()
if line.startswith(';'):
continue
elif line.startswith('['):
if block:
yield self._finish_block(block)
block = self._create_block(line)
elif line != '':
if '[' in line: # metadata on this line -> enforce whitespace '/path/file = ...'
tmp = lmap(str.strip, line.split(' = ', 1))
else: # loose whitespace convention (allow: '/path/file_var=B_test=1')
tmp = lmap(str.strip, rsplit(line, '=', 1))
if len(tmp) != 2:
raise DatasetError(
'Malformed entry in dataset file:\n%s' % line)
self._fill_block(block, *tmp)
except Exception:
raise DatasetError('Unable to parse %s:%d\n\t%s' %
(repr(self._filename), idx, repr(line)))
if block:
yield self._finish_block(block)
def _parseLine(self, exceptionIntro, configContent, configFile, idx, line):
exceptionIntroLineInfo = exceptionIntro + ':%d\n\t%r' % (idx, line)
try:
line = rsplit(line, ';', 1)[0].rstrip()
except Exception:
raise ConfigError(exceptionIntroLineInfo + '\nUnable to strip comments!')
exceptionIntroLineInfo = exceptionIntro + ':%d\n\t%r' % (idx, line) # removed comment
if line.lstrip().startswith(';') or line.lstrip().startswith('#') or not line.strip():
return # skip empty lines or comment lines
elif line[0].isspace():
try:
self._currentValue += '\n' + line.strip()
self._currentIndices += [idx]
except Exception:
raise ConfigError(exceptionIntroLineInfo + '\nInvalid indentation!')
elif line.startswith('['):
if self._currentOption:
self._storeOption(exceptionIntroLineInfo, configContent, configFile)
try:
self._currentSection = line[1:line.index(']')].strip()
self._parseLine(exceptionIntro, configContent, configFile,
idx, line[line.index(']') + 1:].strip())
except Exception:
raise ConfigError(exceptionIntroLineInfo + '\nUnable to parse config section!')
elif '=' in line:
if self._currentOption:
self._storeOption(exceptionIntroLineInfo, configContent, configFile)
try:
(self._currentOption, self._currentValue) = lmap(str.strip, line.split('=', 1))
self._currentIndices = [idx]
except Exception:
raise ConfigError(exceptionIntroLineInfo + '\nUnable to parse config option!')
else:
raise ConfigError(exceptionIntroLineInfo + '\nPlease use "key = value" syntax or indent values!')
def registerObject(cls, searchPath = [], base = None):
if not base:
base = cls
if not hasattr(base, 'moduleMap'):
(base.moduleMap, base.moduleMapDynamic, base.modPaths) = ({}, {}, [])
splitUpFun = lambda x: rsplit(base.__module__, ".", x)[0]
base.modPaths = utils.uniqueListRL(searchPath + base.modPaths + map(splitUpFun, range(base.__module__.count(".") + 1)))
def _parseFile(self, iterator):
block = None
for idx, line in enumerate(iterator):
try:
# Found start of block:
line = line.strip()
if line.startswith(';'):
continue
elif line.startswith('['):
if block:
yield self._finishBlock(block)
block = self._createBlock(line)
elif line != '':
tmp = lmap(
str.strip,
utils.QM('[' in line, line.split(' = ', 1),
rsplit(line, '=', 1)))
if len(tmp) != 2:
raise DatasetError(
'Malformed entry in dataset file:\n%s' % line)
key, value = tmp
handlerInfo = self._handleEntry.get(key.lower(), None)
if handlerInfo:
(prop, parser, msg) = handlerInfo
block[prop] = try_apply(value, parser, msg)
else:
block[DataProvider.FileList].append(
self._parseEntry(block, key, value))
except Exception:
raise DatasetError('Unable to parse %s:%d\n\t%s' %
(repr(self._filename), idx, repr(line)))
if block:
yield self._finishBlock(block)
def _parseLine(self, exceptionIntro, configContent, configFile, idx, line):
exceptionIntroLineInfo = exceptionIntro + ':%d\n\t%r' % (idx, line)
try:
line = rsplit(line, ';', 1)[0].rstrip()
except Exception:
raise ConfigError(exceptionIntroLineInfo + '\nUnable to strip comments!')
exceptionIntroLineInfo = exceptionIntro + ':%d\n\t%r' % (idx, line) # removed comment
if line.lstrip().startswith(';') or line.lstrip().startswith('#') or not line.strip():
return # skip empty lines or comment lines
elif line[0].isspace():
try:
self._currentValue += '\n' + line.strip()
self._currentIndices += [idx]
except Exception:
raise ConfigError(exceptionIntroLineInfo + '\nInvalid indentation!')
elif line.startswith('['):
if self._currentOption:
self._storeOption(exceptionIntroLineInfo, configContent, configFile)
try:
self._currentSection = line[1:line.index(']')].strip()
self._parseLine(exceptionIntro, configContent, configFile,
idx, line[line.index(']') + 1:].strip())
except Exception:
raise ConfigError(exceptionIntroLineInfo + '\nUnable to parse config section!')
elif '=' in line:
if self._currentOption:
self._storeOption(exceptionIntroLineInfo, configContent, configFile)
try:
(self._currentOption, self._currentValue) = lmap(str.strip, line.split('=', 1))
self._currentIndices = [idx]
except Exception:
raise ConfigError(exceptionIntroLineInfo + '\nUnable to parse config option!')
else:
raise ConfigError(exceptionIntroLineInfo + '\nPlease use "key = value" syntax or indent values!')
def _iter_possible_parents(self, current_thread_name): # yield activities in current and parent threads stack = list(Activity.running_by_thread_name.get(current_thread_name, [])) stack.reverse() # in reverse order of creation for item in stack: yield item if '-' in current_thread_name: for item in self._iter_possible_parents(rsplit(current_thread_name, '-', 1)[0]): yield item
def getBlocksInternal(self):
def doFilter(block):
if self._filter:
name = "/%s#%s#" % (block[DataProvider.Dataset], block.get(DataProvider.BlockName, ""))
return self._filter in name
return True
(blockinfo, commonMetadata) = (None, [])
for line in open(self._filename, "rb"):
# Found start of block:
line = line.strip()
if line.startswith(";"):
continue
elif line.startswith("["):
if blockinfo and doFilter(blockinfo):
yield blockinfo
blockinfo = {DataProvider.Locations: None, DataProvider.FileList: []}
blockname = line.lstrip("[").rstrip("]").split("#")
if len(blockname) > 0:
blockinfo[DataProvider.Dataset] = blockname[0]
if len(blockname) > 1:
blockinfo[DataProvider.BlockName] = blockname[1]
commonprefix = self._forcePrefix
commonMetadata = []
elif line != "":
tmp = map(str.strip, utils.QM("[" in line, line.split(" = ", 1), rsplit(line, "=", 1)))
if len(tmp) != 2:
raise ConfigError("Malformed entry in dataset file:\n%s" % line)
key, value = tmp
if key.lower() == "nickname":
blockinfo[DataProvider.Nickname] = value
elif key.lower() == "id":
blockinfo[DataProvider.DatasetID] = int(value)
elif key.lower() == "events":
blockinfo[DataProvider.NEntries] = int(value)
elif key.lower() == "metadata":
blockinfo[DataProvider.Metadata] = eval(value)
elif key.lower() == "metadata common":
commonMetadata = eval(value)
elif key.lower() == "se list":
blockinfo[DataProvider.Locations] = utils.parseList(value)
elif key.lower() == "prefix":
if not self._forcePrefix:
commonprefix = value
else:
if commonprefix:
key = "%s/%s" % (commonprefix, key)
value = value.split(" ", 1)
data = {DataProvider.URL: key, DataProvider.NEntries: int(value[0])}
if commonMetadata:
data[DataProvider.Metadata] = commonMetadata
if len(value) > 1:
data[DataProvider.Metadata] = data.get(DataProvider.Metadata, []) + eval(value[1])
blockinfo[DataProvider.FileList].append(data)
else:
if blockinfo and doFilter(blockinfo):
yield blockinfo
def getBlocksInternal(self):
def doFilter(block):
if self._filter:
name = '/%s#%s#' % (block[DataProvider.Dataset], block.get(DataProvider.BlockName, ''))
return self._filter in name
return True
(blockinfo, commonMetadata) = (None, [])
for line in open(self._filename, 'rb'):
# Found start of block:
line = line.strip()
if line.startswith(';'):
continue
elif line.startswith('['):
if blockinfo and doFilter(blockinfo):
yield blockinfo
blockinfo = { DataProvider.Locations: None, DataProvider.FileList: [] }
blockname = line.lstrip('[').rstrip(']').split('#')
if len(blockname) > 0:
blockinfo[DataProvider.Dataset] = blockname[0]
if len(blockname) > 1:
blockinfo[DataProvider.BlockName] = blockname[1]
commonprefix = self._forcePrefix
commonMetadata = []
elif line != '':
tmp = map(str.strip, QM('[' in line, line.split(' = ', 1), rsplit(line, '=', 1)))
if len(tmp) != 2:
raise ConfigError('Malformed entry in dataset file:\n%s' % line)
key, value = tmp
if key.lower() == 'nickname':
blockinfo[DataProvider.Nickname] = value
elif key.lower() == 'id':
blockinfo[DataProvider.DatasetID] = int(value)
elif key.lower() == 'events':
blockinfo[DataProvider.NEntries] = int(value)
elif key.lower() == 'metadata':
blockinfo[DataProvider.Metadata] = eval(value)
elif key.lower() == 'metadata common':
commonMetadata = eval(value)
elif key.lower() == 'se list':
blockinfo[DataProvider.Locations] = utils.parseList(value)
elif key.lower() == 'prefix':
if not self._forcePrefix:
commonprefix = value
else:
if commonprefix:
key = '%s/%s' % (commonprefix, key)
value = value.split(' ', 1)
data = { DataProvider.URL: key, DataProvider.NEntries: int(value[0]) }
if commonMetadata:
data[DataProvider.Metadata] = commonMetadata
if len(value) > 1:
data[DataProvider.Metadata] = data.get(DataProvider.Metadata, []) + eval(value[1])
blockinfo[DataProvider.FileList].append(data)
else:
if blockinfo and doFilter(blockinfo):
yield blockinfo
def _iter_possible_parents(self, current_thread_name):
# yield activities in current and parent threads
stack = list(
Activity.running_by_thread_name.get(current_thread_name, []))
stack.reverse() # in reverse order of creation
for item in stack:
yield item
if '-' in current_thread_name:
for item in self._iter_possible_parents(
rsplit(current_thread_name, '-', 1)[0]):
yield item
def _parse_tickets(self, cached=True):
# Return cached results if requested
if cached and self._cache:
return self._cache
# Call klist and parse results
proc = LocalProcess(self._klist_exec)
self._cache = {}
try:
for line in proc.stdout.iter(timeout=10):
if line.count('@') and (line.count(':') > 1):
issued_expires, principal = rsplit(line, ' ', 1)
issued_expires = issued_expires.replace('/', ' ').split()
assert len(issued_expires) % 2 == 0
issued_str = str.join(
' ', issued_expires[:int(len(issued_expires) / 2)])
expires_str = str.join(
' ', issued_expires[int(len(issued_expires) / 2):])
if expires_str.count(' ') == 3:
if len(expires_str.split()[2]) == 2:
expires = _parse_date(expires_str,
'%m %d %y %H:%M:%S')
elif len(expires_str.split()[2]) == 4:
expires = _parse_date(expires_str,
'%m %d %Y %H:%M:%S') # here
else:
# On NAF, we get an entirely different format now: Sep 2 12:31:34 2021
expires = _parse_date(expires_str,
'%b %d %H:%M:%S %Y')
elif expires_str.count(
' ') == 2: # year information is missing
cur_year = int(time.strftime('%Y'))
expires = _parse_date(expires_str + ' %d' % cur_year,
'%b %d %H:%M:%S %Y')
issued = _parse_date(issued_str + ' %d' % cur_year,
'%b %d %H:%M:%S %Y')
if expires < issued: # wraparound at new year
expires = _parse_date(
expires_str + ' %d' % (cur_year + 1),
'%b %d %H:%M:%S %Y')
self._cache.setdefault('tickets', {})[principal] = expires
elif line.count(':') == 1:
(key, value) = lmap(str.strip, line.split(':', 1))
self._cache[key.lower()] = value
except Exception:
raise AccessTokenError(
'Unable to parse kerberos ticket information!')
proc.status_raise(timeout=0)
return self._cache
def _create_blocks(self, iterable):
block = None
for idx, line in enumerate(iterable):
try:
# Found start of block:
line = line.strip()
if line.startswith(';'):
continue
elif line.startswith('['):
if block:
yield self._finish_block(block)
block = self._create_block(line)
elif line != '':
if '[' in line: # metadata on this line -> enforce whitespace '/path/file = ...'
tmp = lmap(str.strip, line.split(' = ', 1))
else: # loose whitespace convention (allow: '/path/file_var=B_test=1')
tmp = lmap(str.strip, rsplit(line, '=', 1))
if len(tmp) != 2:
raise DatasetError('Malformed entry in dataset file:\n%s' % line)
self._fill_block(block, *tmp)
except Exception:
raise DatasetError('Unable to parse %s:%d\n\t%s' % (repr(self._filename), idx, repr(line)))
if block:
yield self._finish_block(block)
def _parse_line_strip_comments(self, exception_intro, content_configfile,
config_fn, idx, line):
return rsplit(line, ';', 1)[0].rstrip()
def get_file_name(fn): # Return file name without extension
return rsplit(os.path.basename(str(fn)).lstrip('.'), '.', 1)[0]
def get_file_name(fn): # Return file name without extension
return rsplit(os.path.basename(str(fn)).lstrip('.'), '.', 1)[0]
def _parse_line_strip_comments(self, exception_intro, content_configfile, config_fn, idx, line): return rsplit(line, ';', 1)[0].rstrip()
def _parseLineStripComments(self, exceptionIntro, configContent,
configFile, idx, line):
return rsplit(line, ';', 1)[0].rstrip()
