def _processCfg(self, tar, cfg):
cfgSummary = {}
cfgContent = bytes2str(tar.extractfile('%s/config' % cfg).read())
cfgHashResult = bytes2str(tar.extractfile('%s/hash' % cfg).read()).splitlines()
cfgHash = cfgHashResult[-1].strip()
cfgSummary = {'CMSSW_CONFIG_FILE': cfg, 'CMSSW_CONFIG_HASH': cfgHash}
cfgSummary['CMSSW_CONFIG_CONTENT'] = self._cfgStore.setdefault(cfgSummary[self._mergeKey], cfgContent)
# Read global tag from config file - first from hash file, then from config file
if cfgHash not in self._gtStore:
gtLines = lfilter(lambda x: x.startswith('globaltag:'), cfgHashResult)
if gtLines:
self._gtStore[cfgHash] = gtLines[-1].split(':')[1].strip()
if cfgHash not in self._gtStore:
try:
cfgContentEnv = utils.execWrapper(cfgContent)
self._gtStore[cfgHash] = cfgContentEnv['process'].GlobalTag.globaltag.value()
except Exception:
self._gtStore[cfgHash] = 'unknown:All'
cfgSummary['CMSSW_GLOBALTAG'] = self._gtStore[cfgHash]
# Get annotation from config content
def searchConfigFile(key, regex, default):
try:
tmp = re.compile(regex).search(cfgContent.group(1).strip('\"\' '))
except Exception:
tmp = None
if tmp:
cfgSummary[key] = tmp
else:
cfgSummary[key] = default
searchConfigFile('CMSSW_ANNOTATION', r'.*annotation.*=.*cms.untracked.string.*\((.*)\)', None)
searchConfigFile('CMSSW_DATATIER', r'.*dataTier.*=.*cms.untracked.string.*\((.*)\)', 'USER')
cfgReport = xml.dom.minidom.parseString(bytes2str(tar.extractfile('%s/report.xml' % cfg).read()))
evRead = sum(imap(lambda x: int(readTag(x, 'EventsRead')), cfgReport.getElementsByTagName('InputFile')))
return (cfgSummary, cfgReport, evRead)
def _setup_terminal(self): attr = termios.tcgetattr(self._fd_parent_terminal) attr[1] = attr[1] & ~termios.ONLCR # disable \n -> \r\n attr[3] = attr[3] & ~termios.ECHO # disable terminal echo attr[3] = attr[3] | termios.ICANON # enable canonical mode attr[3] = attr[3] | termios.ISIG # enable signals self.stdin.EOF = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VEOF]) self.stdin.EOL = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VEOL]) self.stdin.INTR = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VINTR]) termios.tcsetattr(self._fd_parent_terminal, termios.TCSANOW, attr)
def _setup_terminal(self): attr = termios.tcgetattr(self._fd_parent_terminal) attr[1] = attr[1] & ~termios.ONLCR # disable \n -> \r\n attr[3] = attr[3] & ~termios.ECHO # disable terminal echo attr[3] = attr[3] | termios.ICANON # enable canonical mode attr[3] = attr[3] | termios.ISIG # enable signals self.stdin.EOF = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VEOF]) self.stdin.EOL = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VEOL]) self.stdin.INTR = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VINTR]) termios.tcsetattr(self._fd_parent_terminal, termios.TCSANOW, attr)
def getEntries(self, path, metadata, events, seList, objStore):
jobNum = metadata['GC_JOBNUM']
cmsRunLog = os.path.join(path, 'cmssw.dbs.tar.gz')
if os.path.exists(cmsRunLog):
tar = tarfile.open(cmsRunLog, 'r')
# Collect infos about transferred files
fileSummaryMap = {}
try:
for rawdata in imap(lambda value: bytes2str(value).split(),
tar.extractfile('files').readlines()):
fileSummaryMap[rawdata[2]] = {
'SE_OUTPUT_HASH_CRC32': rawdata[0],
'SE_OUTPUT_SIZE': int(rawdata[1])
}
objStore['CMSSW_FILES'] = fileSummaryMap
except Exception:
raise DatasetError(
'Could not read CMSSW file infos for job %d!' % jobNum)
# Collect infos about CMSSW processing steps
cfgSummaryMap = {}
self._processSteps(jobNum, tar, cfgSummaryMap, fileSummaryMap)
for cfg in cfgSummaryMap:
metadata.setdefault('CMSSW_CONFIG_JOBHASH', []).append(
cfgSummaryMap[cfg]['CMSSW_CONFIG_HASH'])
objStore.update({
'CMSSW_CONFIG': cfgSummaryMap,
'CMSSW_FILES': fileSummaryMap
})
tar.close()
yield (path, metadata, events, seList, objStore)
def request(self, mode, url, headers, params=None, data=None, cert=None):
request_fun = {
RestSession.GET: lambda: 'GET',
RestSession.PUT: lambda: 'PUT',
RestSession.POST: lambda: 'POST',
RestSession.DELETE: lambda: 'DELETE'
}[mode]
if params:
url += '?%s' % urlencode(params)
if data:
data = str2bytes(data)
request = Request(url=url, data=data, headers=headers)
request.get_method = request_fun
if cert:
class HTTPSClientAuthHandler(HTTPSHandler):
def https_open(self, req):
return self.do_open(self.getConnection, req)
def getConnection(self, host, timeout=None):
return HTTPSConnection(host, key_file=cert, cert_file=cert)
opener = build_opener(HTTPSClientAuthHandler())
else:
opener = build_opener()
return bytes2str(opener.open(request).read())
def _iter_datasource_items(self, item, metadata_dict, entries, location_list, obj_dict):
jobnum = metadata_dict['GC_JOBNUM']
cms_log_fn = os.path.join(item, 'cmssw.dbs.tar.gz')
if os.path.exists(cms_log_fn):
tar = tarfile.open(cms_log_fn, 'r')
# Collect infos about transferred files
file_summary_map = {}
try:
file_info_str_list = tar.extractfile('files').readlines()
for rawdata in imap(lambda value: bytes2str(value).split(), file_info_str_list):
file_summary_map[rawdata[2]] = {
'SE_OUTPUT_HASH_CRC32': rawdata[0],
'SE_OUTPUT_SIZE': int(rawdata[1])
}
obj_dict['CMSSW_FILES'] = file_summary_map
except Exception:
raise DatasetError('Could not read CMSSW file infos for job %d!' % jobnum)
# Collect infos about CMSSW processing steps
config_summary_map = {}
self._process_steps(jobnum, tar, config_summary_map, file_summary_map)
for cfg in config_summary_map:
job_hash_list = metadata_dict.setdefault('CMSSW_CONFIG_JOBHASH', [])
job_hash_list.append(config_summary_map[cfg]['CMSSW_CONFIG_HASH'])
obj_dict.update({'CMSSW_CONFIG': config_summary_map, 'CMSSW_FILES': file_summary_map})
tar.close()
yield (item, metadata_dict, entries, location_list, obj_dict)
def readToBuffer():
while True:
try:
tmp = bytes2str(os.read(fd, 32 * 1024))
except OSError:
tmp = ''
if not tmp:
break
buffer.put(tmp)
def readToBuffer(): while True: try: tmp = bytes2str(os.read(fd, 32*1024)) except OSError: tmp = '' if not tmp: break buffer.put(tmp)
def _read_to_buffer(): while True: try: tmp = bytes2str(os.read(fd_read, 32 * 1024)) except OSError: break if not tmp: break buffer.put(tmp)
def _read_to_buffer():
while True:
try:
tmp = bytes2str(os.read(fd_read, 32 * 1024))
except OSError:
break
if not tmp:
break
buffer.put(tmp)
def request(self, mode, url, headers, params=None, data=None, cert=None):
request_fun = {RestSession.GET: lambda: 'GET', RestSession.PUT: lambda: 'PUT',
RestSession.POST: lambda: 'POST', RestSession.DELETE: lambda: 'DELETE'}[mode]
if params:
url += '?%s' % Urllib2Session.urlencode(params)
if data:
data = str2bytes(data)
request = Urllib2Session.Request(url=url, data=data, headers=headers)
request.get_method = request_fun
return bytes2str(self._get_opener(cert).open(request).read())
def request(self, mode, url, headers, params=None, data=None, cert=None):
request_fun = {
RestSession.GET: lambda: 'GET',
RestSession.PUT: lambda: 'PUT',
RestSession.POST: lambda: 'POST',
RestSession.DELETE: lambda: 'DELETE'
}[mode]
if params:
url += '?%s' % Urllib2Session.urlencode(params)
if data:
data = str2bytes(data)
request = Urllib2Session.Request(url=url, data=data, headers=headers)
request.get_method = request_fun
return bytes2str(self._get_opener(cert).open(request).read())
def _process_config(self, tar, cfg):
config_summary = {}
config_content = bytes2str(tar.extractfile('%s/config' % cfg).read())
config_hash_result = bytes2str(tar.extractfile('%s/hash' % cfg).read()).splitlines()
config_hash = config_hash_result[-1].strip()
config_summary = {'CMSSW_CONFIG_FILE': cfg, 'CMSSW_CONFIG_HASH': config_hash}
config_summary['CMSSW_CONFIG_CONTENT'] = self._stored_config.setdefault(
config_summary[self._merge_key], config_content)
# Read global tag from config file - first from hash file, then from config file
if config_hash not in self._stored_globaltag:
global_tag_lines = lfilter(lambda x: x.startswith('globaltag:'), config_hash_result)
if global_tag_lines:
self._stored_globaltag[config_hash] = global_tag_lines[-1].split(':')[1].strip()
if config_hash not in self._stored_globaltag:
try:
config_content_env = exec_wrapper(config_content)
self._stored_globaltag[config_hash] = config_content_env['process'].GlobalTag.globaltag.value()
except Exception:
clear_current_exception()
self._stored_globaltag[config_hash] = 'unknown:All'
config_summary['CMSSW_GLOBALTAG'] = self._stored_globaltag[config_hash]
# Get annotation from config content
def _search_config_file(key, regex, default):
try:
config_summary[key] = regex.search(config_content).group(1).strip('\"\' ') or default
except Exception:
clear_current_exception()
config_summary[key] = default
_search_config_file('CMSSW_ANNOTATION', self._regex_annotation, None)
_search_config_file('CMSSW_DATATIER', self._regex_datatier, 'USER')
config_report = xml.dom.minidom.parseString(
bytes2str(tar.extractfile('%s/report.xml' % cfg).read()))
events_read = sum(imap(lambda x: int(_read_tag(x, 'EventsRead')),
config_report.getElementsByTagName('InputFile')))
return (config_summary, config_report, events_read)
def _process_steps(self, jobnum, tar, config_summary_map, file_summary_map):
cmssw_version = bytes2str(tar.extractfile('version').read()).strip()
for cfg in ifilter(lambda x: ('/' not in x) and (x not in ['version', 'files']), tar.getnames()):
try:
(config_summary, config_report, events_read) = self._process_config(tar, cfg)
config_summary['CMSSW_VERSION'] = cmssw_version
config_summary_map[cfg] = config_summary
except Exception:
raise DatasetError('Could not read config infos about %s in job %d' % (cfg, jobnum))
for output_file_node in config_report.getElementsByTagName('File'):
(file_summary, pfn) = self._process_output_file(config_report, output_file_node)
file_summary['CMSSW_EVENTS_READ'] = events_read
file_summary['CMSSW_CONFIG_FILE'] = cfg
file_summary_map.setdefault(pfn, {}).update(file_summary)
def _request(self, request_fun, url, api, headers, params=None, data=None):
url = self._get_url(url, api)
if params:
url += '?%s' % urlencode(params)
request_headers = self._get_headers(headers)
if data:
data = str2bytes(data)
request = Request(url=url, data=data, headers=request_headers)
request.get_method = request_fun
if self._cert:
cert_handler = self._https_handler()
opener = build_opener(cert_handler)
else:
opener = build_opener()
return bytes2str(opener.open(request).read())
def _request(self, request_fun, url, api, headers, params = None, data = None): url = self._get_url(url, api) if params: url += '?%s' % urlencode(params) request_headers = self._get_headers(headers) if data: data = str2bytes(data) request = Request(url = url, data = data, headers = request_headers) request.get_method = request_fun if self._cert: cert_handler = self._https_handler() opener = build_opener(cert_handler) else: opener = build_opener() return bytes2str(opener.open(request).read())
def _display_logfile(dn, fn):
full_fn = os.path.join(dn, fn)
if os.path.exists(full_fn):
try:
if fn.endswith('.gz'):
fp = gzip.open(full_fn)
content = bytes2str(fp.read())
fp.close()
else:
content = SafeFile(full_fn).read_close()
self._log.error(fn + '\n' + content + '-' * 50)
except Exception:
self._log.exception('Unable to display %s', fn)
clear_current_exception()
else:
self._log.error('Log file does not exist: %s', fn)
def _display_logfile(dn, fn):
full_fn = os.path.join(dn, fn)
if os.path.exists(full_fn):
try:
if fn.endswith('.gz'):
fp = gzip.open(full_fn)
content = bytes2str(fp.read())
fp.close()
else:
content = SafeFile(full_fn).read_close()
self._log.error(fn + '\n' + content + '-' * 50)
except Exception:
self._log.exception('Unable to display %s', fn)
clear_current_exception()
else:
self._log.error('Log file does not exist: %s', fn)
def request(self, mode, url, headers, params = None, data = None, cert = None):
request_fun = {RestSession.GET: lambda: 'GET', RestSession.PUT: lambda: 'PUT',
RestSession.POST: lambda: 'POST', RestSession.DELETE: lambda: 'DELETE'}[mode]
if params:
url += '?%s' % urlencode(params)
if data:
data = str2bytes(data)
request = Request(url = url, data = data, headers = headers)
request.get_method = request_fun
if cert:
class HTTPSClientAuthHandler(HTTPSHandler):
def https_open(self, req):
return self.do_open(self.getConnection, req)
def getConnection(self, host, timeout = None):
return HTTPSConnection(host, key_file = cert, cert_file = cert)
opener = build_opener(HTTPSClientAuthHandler())
else:
opener = build_opener()
return bytes2str(opener.open(request).read())
def getEntries(self, path, metadata, events, seList, objStore):
jobNum = metadata['GC_JOBNUM']
cmsRunLog = os.path.join(path, 'cmssw.dbs.tar.gz')
if os.path.exists(cmsRunLog):
tar = tarfile.open(cmsRunLog, 'r')
# Collect infos about transferred files
fileSummaryMap = {}
try:
for rawdata in imap(lambda value: bytes2str(value).split(), tar.extractfile('files').readlines()):
fileSummaryMap[rawdata[2]] = {'SE_OUTPUT_HASH_CRC32': rawdata[0], 'SE_OUTPUT_SIZE': int(rawdata[1])}
objStore['CMSSW_FILES'] = fileSummaryMap
except Exception:
raise DatasetError('Could not read CMSSW file infos for job %d!' % jobNum)
# Collect infos about CMSSW processing steps
cfgSummaryMap = {}
self._processSteps(jobNum, tar, cfgSummaryMap, fileSummaryMap)
for cfg in cfgSummaryMap:
metadata.setdefault('CMSSW_CONFIG_JOBHASH', []).append(cfgSummaryMap[cfg]['CMSSW_CONFIG_HASH'])
objStore.update({'CMSSW_CONFIG': cfgSummaryMap, 'CMSSW_FILES': fileSummaryMap})
tar.close()
yield (path, metadata, events, seList, objStore)
def readline(self): return bytes2str(self._fp.readline())
def readline(self):
return bytes2str(self._fp.readline())
