def parse(self, line): ''' Parses single line from accounting log file. Example line of accounting log file: "timestamp=2012-05-20 23:59:47" "userDN=/O=GermanGrid/OU=UniWuppertal/CN=Torsten Harenberg" "userFQAN=/atlas/Role=production/Capability=NULL" "ceID=cream-2-fzk.gridka.de:8443/cream-pbs-atlasXL" "jobID=CREAM410741480" "lrmsID=9575064.lrms1" "localUser=11999" Line was split, if you want to rejoin use ' ' as a joiner. ''' data = {} rc = {} record = BlahdRecord() # split file and remove parts which contain only space (like ' ') parts = [x.split('=',1) for x in [y for y in self.LINE_EXPR.split(line) if len(y) > 1]] # Simple mapping between keys in a log file and a table's columns mapping = { 'TimeStamp' : lambda x: 'T'.join(x['timestamp'].split()) + 'Z', 'GlobalUserName' : lambda x: x['userDN'], 'FQAN' : lambda x: x['userFQAN'], 'VO' : lambda x: parse_fqan(x['userFQAN'])[2], 'VOGroup' : lambda x: parse_fqan(x['userFQAN'])[1], 'VORole' : lambda x: parse_fqan(x['userFQAN'])[0], 'CE' : lambda x: x['ceID'], 'GlobalJobId' : lambda x: x['jobID'], 'LrmsId' : lambda x: x['lrmsID'], 'Site' : lambda x: self.site_name, 'ValidFrom' : lambda x: valid_from(parse_timestamp(x['timestamp'])), 'ValidUntil' : lambda x: valid_until(parse_timestamp(x['timestamp'])), 'Processed' : lambda x: Parser.UNPROCESSED} for key, value in parts: # Store only the first value encountered. This is mainly for the # userFQAN field as the first occurence of this is the primary FQAN. if key not in data: data[key] = value for key in mapping: rc[key] = mapping[key](data) record.set_all(rc) return record
def test_valid_from(self): now = datetime.datetime.now() result = now - datetime.timedelta(days=1) self.assertEqual(result, valid_from(now))
def test_valid_from(self): now = datetime.datetime.now() result = now-datetime.timedelta(days=1) self.assertEqual(result, valid_from(now))
def parse(self, line): ''' Parses single line from accounting log file. Example line of accounting log file: "timestamp=2017-02-01 00:03:49; clusterid=381620; CE_JobId=396933.0; owner=lhpilot007; VO=lhcb; userDN=/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=romanov/CN=427293/CN=Vladimir Romanovskiy; userFQAN=/lhcb/Role=pilot/Capability=NULL; [email protected]; request_cpus=1; cputime=3466.000000; syscputime=259.000000; jobduration=4821.575215; walltime+suspensiontime=4823.000000; suspensiontime=0.000000; cputmult=1.1864; pmem=1684532; vmem=944; disk=38543; ExitCode=0; ExitSignal=undefined; LastStatus=4; JobStatus=3; startdate=1485899007; enddate=1485903829" Line was split, if you want to rejoin use ' ' as a joiner. ''' data = {} rc = {} LINE_EXPR = re.compile(r'\"|\"_\"') # This is basically for the FQAN parts = [ x.split('=', 1) for x in [y for y in LINE_EXPR.split(line) if len(y) > 1] ] for item in line.split("; "): key, value = item.split('=', 1) data[key] = value mapping = { 'TimeStamp': lambda x: 'T'.join(x['timestamp'].split()) + 'Z', 'GlobalUserName': lambda x: x['userDN'], 'FQAN': lambda x: x['userFQAN'], 'VO': lambda x: x['VO'], 'VOGroup': lambda x: x['userFQAN'].split("/")[1], 'VORole': lambda x: x['userFQAN'].split("/")[2], 'CE': lambda x: self.machine_name + ":" + "9619" + "/" + self. machine_name + "-" + "condor", 'GlobalJobId': lambda x: x['CE_JobId'] + "_" + self.machine_name, 'LrmsId': lambda x: x['clusterid'] + "_" + self.machine_name, 'Site': lambda x: self.site_name, 'ValidFrom': lambda x: valid_from(parse_timestamp(x['timestamp'])), 'ValidUntil': lambda x: valid_until(parse_timestamp(x['timestamp'])), 'Processed': lambda x: Parser.UNPROCESSED } for key, value in parts: # Store only the first value encountered. This is mainly for the # userFQAN field as the first occurence of this is the primary FQAN. if key not in data: data[key] = value for key in mapping: rc[key] = mapping[key](data) record = HTCondorCERecord() record.set_all(rc) return record