def _processCfg(self, tar, cfg): cfgSummary = {} cfgContent = bytes2str(tar.extractfile('%s/config' % cfg).read()) cfgHashResult = bytes2str(tar.extractfile('%s/hash' % cfg).read()).splitlines() cfgHash = cfgHashResult[-1].strip() cfgSummary = {'CMSSW_CONFIG_FILE': cfg, 'CMSSW_CONFIG_HASH': cfgHash} cfgSummary['CMSSW_CONFIG_CONTENT'] = self._cfgStore.setdefault(cfgSummary[self._mergeKey], cfgContent) # Read global tag from config file - first from hash file, then from config file if cfgHash not in self._gtStore: gtLines = lfilter(lambda x: x.startswith('globaltag:'), cfgHashResult) if gtLines: self._gtStore[cfgHash] = gtLines[-1].split(':')[1].strip() if cfgHash not in self._gtStore: try: cfgContentEnv = utils.execWrapper(cfgContent) self._gtStore[cfgHash] = cfgContentEnv['process'].GlobalTag.globaltag.value() except Exception: self._gtStore[cfgHash] = 'unknown:All' cfgSummary['CMSSW_GLOBALTAG'] = self._gtStore[cfgHash] # Get annotation from config content def searchConfigFile(key, regex, default): try: tmp = re.compile(regex).search(cfgContent.group(1).strip('\"\' ')) except Exception: tmp = None if tmp: cfgSummary[key] = tmp else: cfgSummary[key] = default searchConfigFile('CMSSW_ANNOTATION', r'.*annotation.*=.*cms.untracked.string.*\((.*)\)', None) searchConfigFile('CMSSW_DATATIER', r'.*dataTier.*=.*cms.untracked.string.*\((.*)\)', 'USER') cfgReport = xml.dom.minidom.parseString(bytes2str(tar.extractfile('%s/report.xml' % cfg).read())) evRead = sum(imap(lambda x: int(readTag(x, 'EventsRead')), cfgReport.getElementsByTagName('InputFile'))) return (cfgSummary, cfgReport, evRead)
def _setup_terminal(self): attr = termios.tcgetattr(self._fd_parent_terminal) attr[1] = attr[1] & ~termios.ONLCR # disable \n -> \r\n attr[3] = attr[3] & ~termios.ECHO # disable terminal echo attr[3] = attr[3] | termios.ICANON # enable canonical mode attr[3] = attr[3] | termios.ISIG # enable signals self.stdin.EOF = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VEOF]) self.stdin.EOL = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VEOL]) self.stdin.INTR = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VINTR]) termios.tcsetattr(self._fd_parent_terminal, termios.TCSANOW, attr)
def _setup_terminal(self): attr = termios.tcgetattr(self._fd_parent_terminal) attr[1] = attr[1] & ~termios.ONLCR # disable \n -> \r\n attr[3] = attr[3] & ~termios.ECHO # disable terminal echo attr[3] = attr[3] | termios.ICANON # enable canonical mode attr[3] = attr[3] | termios.ISIG # enable signals self.stdin.EOF = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VEOF]) self.stdin.EOL = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VEOL]) self.stdin.INTR = bytes2str(termios.tcgetattr(self._fd_parent_terminal)[6][termios.VINTR]) termios.tcsetattr(self._fd_parent_terminal, termios.TCSANOW, attr)
def getEntries(self, path, metadata, events, seList, objStore): jobNum = metadata['GC_JOBNUM'] cmsRunLog = os.path.join(path, 'cmssw.dbs.tar.gz') if os.path.exists(cmsRunLog): tar = tarfile.open(cmsRunLog, 'r') # Collect infos about transferred files fileSummaryMap = {} try: for rawdata in imap(lambda value: bytes2str(value).split(), tar.extractfile('files').readlines()): fileSummaryMap[rawdata[2]] = { 'SE_OUTPUT_HASH_CRC32': rawdata[0], 'SE_OUTPUT_SIZE': int(rawdata[1]) } objStore['CMSSW_FILES'] = fileSummaryMap except Exception: raise DatasetError( 'Could not read CMSSW file infos for job %d!' % jobNum) # Collect infos about CMSSW processing steps cfgSummaryMap = {} self._processSteps(jobNum, tar, cfgSummaryMap, fileSummaryMap) for cfg in cfgSummaryMap: metadata.setdefault('CMSSW_CONFIG_JOBHASH', []).append( cfgSummaryMap[cfg]['CMSSW_CONFIG_HASH']) objStore.update({ 'CMSSW_CONFIG': cfgSummaryMap, 'CMSSW_FILES': fileSummaryMap }) tar.close() yield (path, metadata, events, seList, objStore)
def request(self, mode, url, headers, params=None, data=None, cert=None): request_fun = { RestSession.GET: lambda: 'GET', RestSession.PUT: lambda: 'PUT', RestSession.POST: lambda: 'POST', RestSession.DELETE: lambda: 'DELETE' }[mode] if params: url += '?%s' % urlencode(params) if data: data = str2bytes(data) request = Request(url=url, data=data, headers=headers) request.get_method = request_fun if cert: class HTTPSClientAuthHandler(HTTPSHandler): def https_open(self, req): return self.do_open(self.getConnection, req) def getConnection(self, host, timeout=None): return HTTPSConnection(host, key_file=cert, cert_file=cert) opener = build_opener(HTTPSClientAuthHandler()) else: opener = build_opener() return bytes2str(opener.open(request).read())
def _iter_datasource_items(self, item, metadata_dict, entries, location_list, obj_dict): jobnum = metadata_dict['GC_JOBNUM'] cms_log_fn = os.path.join(item, 'cmssw.dbs.tar.gz') if os.path.exists(cms_log_fn): tar = tarfile.open(cms_log_fn, 'r') # Collect infos about transferred files file_summary_map = {} try: file_info_str_list = tar.extractfile('files').readlines() for rawdata in imap(lambda value: bytes2str(value).split(), file_info_str_list): file_summary_map[rawdata[2]] = { 'SE_OUTPUT_HASH_CRC32': rawdata[0], 'SE_OUTPUT_SIZE': int(rawdata[1]) } obj_dict['CMSSW_FILES'] = file_summary_map except Exception: raise DatasetError('Could not read CMSSW file infos for job %d!' % jobnum) # Collect infos about CMSSW processing steps config_summary_map = {} self._process_steps(jobnum, tar, config_summary_map, file_summary_map) for cfg in config_summary_map: job_hash_list = metadata_dict.setdefault('CMSSW_CONFIG_JOBHASH', []) job_hash_list.append(config_summary_map[cfg]['CMSSW_CONFIG_HASH']) obj_dict.update({'CMSSW_CONFIG': config_summary_map, 'CMSSW_FILES': file_summary_map}) tar.close() yield (item, metadata_dict, entries, location_list, obj_dict)
def readToBuffer(): while True: try: tmp = bytes2str(os.read(fd, 32 * 1024)) except OSError: tmp = '' if not tmp: break buffer.put(tmp)
def readToBuffer(): while True: try: tmp = bytes2str(os.read(fd, 32*1024)) except OSError: tmp = '' if not tmp: break buffer.put(tmp)
def _read_to_buffer(): while True: try: tmp = bytes2str(os.read(fd_read, 32 * 1024)) except OSError: break if not tmp: break buffer.put(tmp)
def _read_to_buffer(): while True: try: tmp = bytes2str(os.read(fd_read, 32 * 1024)) except OSError: break if not tmp: break buffer.put(tmp)
def request(self, mode, url, headers, params=None, data=None, cert=None): request_fun = {RestSession.GET: lambda: 'GET', RestSession.PUT: lambda: 'PUT', RestSession.POST: lambda: 'POST', RestSession.DELETE: lambda: 'DELETE'}[mode] if params: url += '?%s' % Urllib2Session.urlencode(params) if data: data = str2bytes(data) request = Urllib2Session.Request(url=url, data=data, headers=headers) request.get_method = request_fun return bytes2str(self._get_opener(cert).open(request).read())
def request(self, mode, url, headers, params=None, data=None, cert=None): request_fun = { RestSession.GET: lambda: 'GET', RestSession.PUT: lambda: 'PUT', RestSession.POST: lambda: 'POST', RestSession.DELETE: lambda: 'DELETE' }[mode] if params: url += '?%s' % Urllib2Session.urlencode(params) if data: data = str2bytes(data) request = Urllib2Session.Request(url=url, data=data, headers=headers) request.get_method = request_fun return bytes2str(self._get_opener(cert).open(request).read())
def _process_config(self, tar, cfg): config_summary = {} config_content = bytes2str(tar.extractfile('%s/config' % cfg).read()) config_hash_result = bytes2str(tar.extractfile('%s/hash' % cfg).read()).splitlines() config_hash = config_hash_result[-1].strip() config_summary = {'CMSSW_CONFIG_FILE': cfg, 'CMSSW_CONFIG_HASH': config_hash} config_summary['CMSSW_CONFIG_CONTENT'] = self._stored_config.setdefault( config_summary[self._merge_key], config_content) # Read global tag from config file - first from hash file, then from config file if config_hash not in self._stored_globaltag: global_tag_lines = lfilter(lambda x: x.startswith('globaltag:'), config_hash_result) if global_tag_lines: self._stored_globaltag[config_hash] = global_tag_lines[-1].split(':')[1].strip() if config_hash not in self._stored_globaltag: try: config_content_env = exec_wrapper(config_content) self._stored_globaltag[config_hash] = config_content_env['process'].GlobalTag.globaltag.value() except Exception: clear_current_exception() self._stored_globaltag[config_hash] = 'unknown:All' config_summary['CMSSW_GLOBALTAG'] = self._stored_globaltag[config_hash] # Get annotation from config content def _search_config_file(key, regex, default): try: config_summary[key] = regex.search(config_content).group(1).strip('\"\' ') or default except Exception: clear_current_exception() config_summary[key] = default _search_config_file('CMSSW_ANNOTATION', self._regex_annotation, None) _search_config_file('CMSSW_DATATIER', self._regex_datatier, 'USER') config_report = xml.dom.minidom.parseString( bytes2str(tar.extractfile('%s/report.xml' % cfg).read())) events_read = sum(imap(lambda x: int(_read_tag(x, 'EventsRead')), config_report.getElementsByTagName('InputFile'))) return (config_summary, config_report, events_read)
def _process_steps(self, jobnum, tar, config_summary_map, file_summary_map): cmssw_version = bytes2str(tar.extractfile('version').read()).strip() for cfg in ifilter(lambda x: ('/' not in x) and (x not in ['version', 'files']), tar.getnames()): try: (config_summary, config_report, events_read) = self._process_config(tar, cfg) config_summary['CMSSW_VERSION'] = cmssw_version config_summary_map[cfg] = config_summary except Exception: raise DatasetError('Could not read config infos about %s in job %d' % (cfg, jobnum)) for output_file_node in config_report.getElementsByTagName('File'): (file_summary, pfn) = self._process_output_file(config_report, output_file_node) file_summary['CMSSW_EVENTS_READ'] = events_read file_summary['CMSSW_CONFIG_FILE'] = cfg file_summary_map.setdefault(pfn, {}).update(file_summary)
def _request(self, request_fun, url, api, headers, params=None, data=None): url = self._get_url(url, api) if params: url += '?%s' % urlencode(params) request_headers = self._get_headers(headers) if data: data = str2bytes(data) request = Request(url=url, data=data, headers=request_headers) request.get_method = request_fun if self._cert: cert_handler = self._https_handler() opener = build_opener(cert_handler) else: opener = build_opener() return bytes2str(opener.open(request).read())
def _request(self, request_fun, url, api, headers, params = None, data = None): url = self._get_url(url, api) if params: url += '?%s' % urlencode(params) request_headers = self._get_headers(headers) if data: data = str2bytes(data) request = Request(url = url, data = data, headers = request_headers) request.get_method = request_fun if self._cert: cert_handler = self._https_handler() opener = build_opener(cert_handler) else: opener = build_opener() return bytes2str(opener.open(request).read())
def _display_logfile(dn, fn): full_fn = os.path.join(dn, fn) if os.path.exists(full_fn): try: if fn.endswith('.gz'): fp = gzip.open(full_fn) content = bytes2str(fp.read()) fp.close() else: content = SafeFile(full_fn).read_close() self._log.error(fn + '\n' + content + '-' * 50) except Exception: self._log.exception('Unable to display %s', fn) clear_current_exception() else: self._log.error('Log file does not exist: %s', fn)
def _display_logfile(dn, fn): full_fn = os.path.join(dn, fn) if os.path.exists(full_fn): try: if fn.endswith('.gz'): fp = gzip.open(full_fn) content = bytes2str(fp.read()) fp.close() else: content = SafeFile(full_fn).read_close() self._log.error(fn + '\n' + content + '-' * 50) except Exception: self._log.exception('Unable to display %s', fn) clear_current_exception() else: self._log.error('Log file does not exist: %s', fn)
def request(self, mode, url, headers, params = None, data = None, cert = None): request_fun = {RestSession.GET: lambda: 'GET', RestSession.PUT: lambda: 'PUT', RestSession.POST: lambda: 'POST', RestSession.DELETE: lambda: 'DELETE'}[mode] if params: url += '?%s' % urlencode(params) if data: data = str2bytes(data) request = Request(url = url, data = data, headers = headers) request.get_method = request_fun if cert: class HTTPSClientAuthHandler(HTTPSHandler): def https_open(self, req): return self.do_open(self.getConnection, req) def getConnection(self, host, timeout = None): return HTTPSConnection(host, key_file = cert, cert_file = cert) opener = build_opener(HTTPSClientAuthHandler()) else: opener = build_opener() return bytes2str(opener.open(request).read())
def getEntries(self, path, metadata, events, seList, objStore): jobNum = metadata['GC_JOBNUM'] cmsRunLog = os.path.join(path, 'cmssw.dbs.tar.gz') if os.path.exists(cmsRunLog): tar = tarfile.open(cmsRunLog, 'r') # Collect infos about transferred files fileSummaryMap = {} try: for rawdata in imap(lambda value: bytes2str(value).split(), tar.extractfile('files').readlines()): fileSummaryMap[rawdata[2]] = {'SE_OUTPUT_HASH_CRC32': rawdata[0], 'SE_OUTPUT_SIZE': int(rawdata[1])} objStore['CMSSW_FILES'] = fileSummaryMap except Exception: raise DatasetError('Could not read CMSSW file infos for job %d!' % jobNum) # Collect infos about CMSSW processing steps cfgSummaryMap = {} self._processSteps(jobNum, tar, cfgSummaryMap, fileSummaryMap) for cfg in cfgSummaryMap: metadata.setdefault('CMSSW_CONFIG_JOBHASH', []).append(cfgSummaryMap[cfg]['CMSSW_CONFIG_HASH']) objStore.update({'CMSSW_CONFIG': cfgSummaryMap, 'CMSSW_FILES': fileSummaryMap}) tar.close() yield (path, metadata, events, seList, objStore)
def readline(self): return bytes2str(self._fp.readline())
def readline(self): return bytes2str(self._fp.readline())