def _parseTickets(self, cached = True): # Return cached results if requested if cached and self._cache: return self._cache # Call klist and parse results proc = LocalProcess(self._klistExec) self._cache = {} try: for line in proc.stdout.iter(timeout = 10): if line.count('@') and (line.count(':') > 1): issued_expires, principal = rsplit(line, ' ', 1) issued_expires = issued_expires.replace('/', ' ').split() assert(len(issued_expires) % 2 == 0) issued_str = str.join(' ', issued_expires[:int(len(issued_expires) / 2)]) expires_str = str.join(' ', issued_expires[int(len(issued_expires) / 2):]) parseDate = lambda value, format: time.mktime(time.strptime(value, format)) if expires_str.count(' ') == 3: if len(expires_str.split()[2]) == 2: expires = parseDate(expires_str, '%m %d %y %H:%M:%S') else: expires = parseDate(expires_str, '%m %d %Y %H:%M:%S') elif expires_str.count(' ') == 2: # year information is missing currentYear = int(time.strftime('%Y')) expires = parseDate(expires_str + ' %d' % currentYear, '%b %d %H:%M:%S %Y') issued = parseDate(issued_str + ' %d' % currentYear, '%b %d %H:%M:%S %Y') if expires < issued: # wraparound at new year expires = parseDate(expires_str + ' %d' % (currentYear + 1), '%b %d %H:%M:%S %Y') self._cache.setdefault('tickets', {})[principal] = expires elif line.count(':') == 1: key, value = lmap(str.strip, line.split(':', 1)) self._cache[key.lower()] = value except Exception: raise AccessTokenError('Unable to parse kerberos ticket information!') proc.status_raise(timeout = 0) return self._cache
def _parseFile(self, iterator): block = None for idx, line in enumerate(iterator): try: # Found start of block: line = line.strip() if line.startswith(';'): continue elif line.startswith('['): if block: yield self._finishBlock(block) block = self._createBlock(line) elif line != '': tmp = lmap(str.strip, utils.QM('[' in line, line.split(' = ', 1), rsplit(line, '=', 1))) if len(tmp) != 2: raise DatasetError('Malformed entry in dataset file:\n%s' % line) key, value = tmp handlerInfo = self._handleEntry.get(key.lower(), None) if handlerInfo: (prop, parser, msg) = handlerInfo block[prop] = try_apply(value, parser, msg) else: block[DataProvider.FileList].append(self._parseEntry(block, key, value)) except Exception: raise DatasetError('Unable to parse %s:%d\n\t%s' % (repr(self._filename), idx, repr(line))) if block: yield self._finishBlock(block)
def parseLine(idx, line): exceptionText = 'Unable to parse config file %s:%d\n\t%r' % (configFile, idx, line) try: line = rsplit(line, ';', 1)[0].rstrip() except Exception: raise ConfigError(exceptionText + '\nUnable to strip comments!') exceptionText = 'Unable to parse config file %s:%d\n\t%r' % (configFile, idx, line) if not line.strip() or line.lstrip().startswith('#'): # skip empty lines or comment lines return elif line[0].isspace(): try: self._currentValue += '\n' + line.strip() self._currentLines += [idx] except Exception: raise ConfigError(exceptionText + '\nInvalid indentation!') elif line.startswith('['): if self._currentOption: storeOption() try: self._currentSection = line[1:line.index(']')].strip() parseLine(idx, line[line.index(']') + 1:].strip()) except Exception: raise ConfigError(exceptionText + '\nUnable to parse config section!') elif '=' in line: if self._currentOption: storeOption() try: (self._currentOption, self._currentValue) = map(str.strip, line.split('=', 1)) self._currentLines = [idx] except Exception: raise ConfigError(exceptionText + '\nUnable to parse config option!') else: raise ConfigError(exceptionText + '\nPlease use "key = value" syntax or indent values!')
def _create_blocks(self, iterable): block = None for idx, line in enumerate(iterable): try: # Found start of block: line = line.strip() if line.startswith(';'): continue elif line.startswith('['): if block: yield self._finish_block(block) block = self._create_block(line) elif line != '': if '[' in line: # metadata on this line -> enforce whitespace '/path/file = ...' tmp = lmap(str.strip, line.split(' = ', 1)) else: # loose whitespace convention (allow: '/path/file_var=B_test=1') tmp = lmap(str.strip, rsplit(line, '=', 1)) if len(tmp) != 2: raise DatasetError( 'Malformed entry in dataset file:\n%s' % line) self._fill_block(block, *tmp) except Exception: raise DatasetError('Unable to parse %s:%d\n\t%s' % (repr(self._filename), idx, repr(line))) if block: yield self._finish_block(block)
def _parseLine(self, exceptionIntro, configContent, configFile, idx, line): exceptionIntroLineInfo = exceptionIntro + ':%d\n\t%r' % (idx, line) try: line = rsplit(line, ';', 1)[0].rstrip() except Exception: raise ConfigError(exceptionIntroLineInfo + '\nUnable to strip comments!') exceptionIntroLineInfo = exceptionIntro + ':%d\n\t%r' % (idx, line) # removed comment if line.lstrip().startswith(';') or line.lstrip().startswith('#') or not line.strip(): return # skip empty lines or comment lines elif line[0].isspace(): try: self._currentValue += '\n' + line.strip() self._currentIndices += [idx] except Exception: raise ConfigError(exceptionIntroLineInfo + '\nInvalid indentation!') elif line.startswith('['): if self._currentOption: self._storeOption(exceptionIntroLineInfo, configContent, configFile) try: self._currentSection = line[1:line.index(']')].strip() self._parseLine(exceptionIntro, configContent, configFile, idx, line[line.index(']') + 1:].strip()) except Exception: raise ConfigError(exceptionIntroLineInfo + '\nUnable to parse config section!') elif '=' in line: if self._currentOption: self._storeOption(exceptionIntroLineInfo, configContent, configFile) try: (self._currentOption, self._currentValue) = lmap(str.strip, line.split('=', 1)) self._currentIndices = [idx] except Exception: raise ConfigError(exceptionIntroLineInfo + '\nUnable to parse config option!') else: raise ConfigError(exceptionIntroLineInfo + '\nPlease use "key = value" syntax or indent values!')
def registerObject(cls, searchPath = [], base = None): if not base: base = cls if not hasattr(base, 'moduleMap'): (base.moduleMap, base.moduleMapDynamic, base.modPaths) = ({}, {}, []) splitUpFun = lambda x: rsplit(base.__module__, ".", x)[0] base.modPaths = utils.uniqueListRL(searchPath + base.modPaths + map(splitUpFun, range(base.__module__.count(".") + 1)))
def _parseFile(self, iterator): block = None for idx, line in enumerate(iterator): try: # Found start of block: line = line.strip() if line.startswith(';'): continue elif line.startswith('['): if block: yield self._finishBlock(block) block = self._createBlock(line) elif line != '': tmp = lmap( str.strip, utils.QM('[' in line, line.split(' = ', 1), rsplit(line, '=', 1))) if len(tmp) != 2: raise DatasetError( 'Malformed entry in dataset file:\n%s' % line) key, value = tmp handlerInfo = self._handleEntry.get(key.lower(), None) if handlerInfo: (prop, parser, msg) = handlerInfo block[prop] = try_apply(value, parser, msg) else: block[DataProvider.FileList].append( self._parseEntry(block, key, value)) except Exception: raise DatasetError('Unable to parse %s:%d\n\t%s' % (repr(self._filename), idx, repr(line))) if block: yield self._finishBlock(block)
def _iter_possible_parents(self, current_thread_name): # yield activities in current and parent threads stack = list(Activity.running_by_thread_name.get(current_thread_name, [])) stack.reverse() # in reverse order of creation for item in stack: yield item if '-' in current_thread_name: for item in self._iter_possible_parents(rsplit(current_thread_name, '-', 1)[0]): yield item
def getBlocksInternal(self): def doFilter(block): if self._filter: name = "/%s#%s#" % (block[DataProvider.Dataset], block.get(DataProvider.BlockName, "")) return self._filter in name return True (blockinfo, commonMetadata) = (None, []) for line in open(self._filename, "rb"): # Found start of block: line = line.strip() if line.startswith(";"): continue elif line.startswith("["): if blockinfo and doFilter(blockinfo): yield blockinfo blockinfo = {DataProvider.Locations: None, DataProvider.FileList: []} blockname = line.lstrip("[").rstrip("]").split("#") if len(blockname) > 0: blockinfo[DataProvider.Dataset] = blockname[0] if len(blockname) > 1: blockinfo[DataProvider.BlockName] = blockname[1] commonprefix = self._forcePrefix commonMetadata = [] elif line != "": tmp = map(str.strip, utils.QM("[" in line, line.split(" = ", 1), rsplit(line, "=", 1))) if len(tmp) != 2: raise ConfigError("Malformed entry in dataset file:\n%s" % line) key, value = tmp if key.lower() == "nickname": blockinfo[DataProvider.Nickname] = value elif key.lower() == "id": blockinfo[DataProvider.DatasetID] = int(value) elif key.lower() == "events": blockinfo[DataProvider.NEntries] = int(value) elif key.lower() == "metadata": blockinfo[DataProvider.Metadata] = eval(value) elif key.lower() == "metadata common": commonMetadata = eval(value) elif key.lower() == "se list": blockinfo[DataProvider.Locations] = utils.parseList(value) elif key.lower() == "prefix": if not self._forcePrefix: commonprefix = value else: if commonprefix: key = "%s/%s" % (commonprefix, key) value = value.split(" ", 1) data = {DataProvider.URL: key, DataProvider.NEntries: int(value[0])} if commonMetadata: data[DataProvider.Metadata] = commonMetadata if len(value) > 1: data[DataProvider.Metadata] = data.get(DataProvider.Metadata, []) + eval(value[1]) blockinfo[DataProvider.FileList].append(data) else: if blockinfo and doFilter(blockinfo): yield blockinfo
def getBlocksInternal(self): def doFilter(block): if self._filter: name = '/%s#%s#' % (block[DataProvider.Dataset], block.get(DataProvider.BlockName, '')) return self._filter in name return True (blockinfo, commonMetadata) = (None, []) for line in open(self._filename, 'rb'): # Found start of block: line = line.strip() if line.startswith(';'): continue elif line.startswith('['): if blockinfo and doFilter(blockinfo): yield blockinfo blockinfo = { DataProvider.Locations: None, DataProvider.FileList: [] } blockname = line.lstrip('[').rstrip(']').split('#') if len(blockname) > 0: blockinfo[DataProvider.Dataset] = blockname[0] if len(blockname) > 1: blockinfo[DataProvider.BlockName] = blockname[1] commonprefix = self._forcePrefix commonMetadata = [] elif line != '': tmp = map(str.strip, QM('[' in line, line.split(' = ', 1), rsplit(line, '=', 1))) if len(tmp) != 2: raise ConfigError('Malformed entry in dataset file:\n%s' % line) key, value = tmp if key.lower() == 'nickname': blockinfo[DataProvider.Nickname] = value elif key.lower() == 'id': blockinfo[DataProvider.DatasetID] = int(value) elif key.lower() == 'events': blockinfo[DataProvider.NEntries] = int(value) elif key.lower() == 'metadata': blockinfo[DataProvider.Metadata] = eval(value) elif key.lower() == 'metadata common': commonMetadata = eval(value) elif key.lower() == 'se list': blockinfo[DataProvider.Locations] = utils.parseList(value) elif key.lower() == 'prefix': if not self._forcePrefix: commonprefix = value else: if commonprefix: key = '%s/%s' % (commonprefix, key) value = value.split(' ', 1) data = { DataProvider.URL: key, DataProvider.NEntries: int(value[0]) } if commonMetadata: data[DataProvider.Metadata] = commonMetadata if len(value) > 1: data[DataProvider.Metadata] = data.get(DataProvider.Metadata, []) + eval(value[1]) blockinfo[DataProvider.FileList].append(data) else: if blockinfo and doFilter(blockinfo): yield blockinfo
def _iter_possible_parents(self, current_thread_name): # yield activities in current and parent threads stack = list( Activity.running_by_thread_name.get(current_thread_name, [])) stack.reverse() # in reverse order of creation for item in stack: yield item if '-' in current_thread_name: for item in self._iter_possible_parents( rsplit(current_thread_name, '-', 1)[0]): yield item
def _parse_tickets(self, cached=True): # Return cached results if requested if cached and self._cache: return self._cache # Call klist and parse results proc = LocalProcess(self._klist_exec) self._cache = {} try: for line in proc.stdout.iter(timeout=10): if line.count('@') and (line.count(':') > 1): issued_expires, principal = rsplit(line, ' ', 1) issued_expires = issued_expires.replace('/', ' ').split() assert len(issued_expires) % 2 == 0 issued_str = str.join( ' ', issued_expires[:int(len(issued_expires) / 2)]) expires_str = str.join( ' ', issued_expires[int(len(issued_expires) / 2):]) if expires_str.count(' ') == 3: if len(expires_str.split()[2]) == 2: expires = _parse_date(expires_str, '%m %d %y %H:%M:%S') elif len(expires_str.split()[2]) == 4: expires = _parse_date(expires_str, '%m %d %Y %H:%M:%S') # here else: # On NAF, we get an entirely different format now: Sep 2 12:31:34 2021 expires = _parse_date(expires_str, '%b %d %H:%M:%S %Y') elif expires_str.count( ' ') == 2: # year information is missing cur_year = int(time.strftime('%Y')) expires = _parse_date(expires_str + ' %d' % cur_year, '%b %d %H:%M:%S %Y') issued = _parse_date(issued_str + ' %d' % cur_year, '%b %d %H:%M:%S %Y') if expires < issued: # wraparound at new year expires = _parse_date( expires_str + ' %d' % (cur_year + 1), '%b %d %H:%M:%S %Y') self._cache.setdefault('tickets', {})[principal] = expires elif line.count(':') == 1: (key, value) = lmap(str.strip, line.split(':', 1)) self._cache[key.lower()] = value except Exception: raise AccessTokenError( 'Unable to parse kerberos ticket information!') proc.status_raise(timeout=0) return self._cache
def _create_blocks(self, iterable): block = None for idx, line in enumerate(iterable): try: # Found start of block: line = line.strip() if line.startswith(';'): continue elif line.startswith('['): if block: yield self._finish_block(block) block = self._create_block(line) elif line != '': if '[' in line: # metadata on this line -> enforce whitespace '/path/file = ...' tmp = lmap(str.strip, line.split(' = ', 1)) else: # loose whitespace convention (allow: '/path/file_var=B_test=1') tmp = lmap(str.strip, rsplit(line, '=', 1)) if len(tmp) != 2: raise DatasetError('Malformed entry in dataset file:\n%s' % line) self._fill_block(block, *tmp) except Exception: raise DatasetError('Unable to parse %s:%d\n\t%s' % (repr(self._filename), idx, repr(line))) if block: yield self._finish_block(block)
def _parse_line_strip_comments(self, exception_intro, content_configfile, config_fn, idx, line): return rsplit(line, ';', 1)[0].rstrip()
def get_file_name(fn): # Return file name without extension return rsplit(os.path.basename(str(fn)).lstrip('.'), '.', 1)[0]
def _parseLineStripComments(self, exceptionIntro, configContent, configFile, idx, line): return rsplit(line, ';', 1)[0].rstrip()