def parse(self, line):
'''
Parses single line from accounting log file.
Example line of accounting log file:
"timestamp=2012-05-20 23:59:47" "userDN=/O=GermanGrid/OU=UniWuppertal/CN=Torsten Harenberg"
"userFQAN=/atlas/Role=production/Capability=NULL" "ceID=cream-2-fzk.gridka.de:8443/cream-pbs-atlasXL"
"jobID=CREAM410741480" "lrmsID=9575064.lrms1" "localUser=11999"
Line was split, if you want to rejoin use ' ' as a joiner.
'''
data = {}
rc = {}
record = BlahdRecord()
# split file and remove parts which contain only space (like ' ')
parts = [x.split('=',1) for x in [y for y in self.LINE_EXPR.split(line) if len(y) > 1]]
# Simple mapping between keys in a log file and a table's columns
mapping = {
'TimeStamp' : lambda x: 'T'.join(x['timestamp'].split()) + 'Z',
'GlobalUserName' : lambda x: x['userDN'],
'FQAN' : lambda x: x['userFQAN'],
'VO' : lambda x: parse_fqan(x['userFQAN'])[2],
'VOGroup' : lambda x: parse_fqan(x['userFQAN'])[1],
'VORole' : lambda x: parse_fqan(x['userFQAN'])[0],
'CE' : lambda x: x['ceID'],
'GlobalJobId' : lambda x: x['jobID'],
'LrmsId' : lambda x: x['lrmsID'],
'Site' : lambda x: self.site_name,
'ValidFrom' : lambda x: valid_from(parse_timestamp(x['timestamp'])),
'ValidUntil' : lambda x: valid_until(parse_timestamp(x['timestamp'])),
'Processed' : lambda x: Parser.UNPROCESSED}
for key, value in parts:
# Store only the first value encountered. This is mainly for the
# userFQAN field as the first occurence of this is the primary FQAN.
if key not in data:
data[key] = value
for key in mapping:
rc[key] = mapping[key](data)
record.set_all(rc)
return record
def test_valid_from(self):
now = datetime.datetime.now()
result = now - datetime.timedelta(days=1)
self.assertEqual(result, valid_from(now))
def test_valid_from(self):
now = datetime.datetime.now()
result = now-datetime.timedelta(days=1)
self.assertEqual(result, valid_from(now))
def parse(self, line):
'''
Parses single line from accounting log file.
Example line of accounting log file:
"timestamp=2017-02-01 00:03:49; clusterid=381620; CE_JobId=396933.0; owner=lhpilot007; VO=lhcb;
userDN=/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=romanov/CN=427293/CN=Vladimir Romanovskiy;
userFQAN=/lhcb/Role=pilot/Capability=NULL; [email protected]; request_cpus=1;
cputime=3466.000000; syscputime=259.000000; jobduration=4821.575215; walltime+suspensiontime=4823.000000;
suspensiontime=0.000000; cputmult=1.1864; pmem=1684532; vmem=944; disk=38543; ExitCode=0;
ExitSignal=undefined; LastStatus=4; JobStatus=3; startdate=1485899007; enddate=1485903829"
Line was split, if you want to rejoin use ' ' as a joiner.
'''
data = {}
rc = {}
LINE_EXPR = re.compile(r'\"|\"_\"')
# This is basically for the FQAN
parts = [
x.split('=', 1)
for x in [y for y in LINE_EXPR.split(line) if len(y) > 1]
]
for item in line.split("; "):
key, value = item.split('=', 1)
data[key] = value
mapping = {
'TimeStamp':
lambda x: 'T'.join(x['timestamp'].split()) + 'Z',
'GlobalUserName':
lambda x: x['userDN'],
'FQAN':
lambda x: x['userFQAN'],
'VO':
lambda x: x['VO'],
'VOGroup':
lambda x: x['userFQAN'].split("/")[1],
'VORole':
lambda x: x['userFQAN'].split("/")[2],
'CE':
lambda x: self.machine_name + ":" + "9619" + "/" + self.
machine_name + "-" + "condor",
'GlobalJobId':
lambda x: x['CE_JobId'] + "_" + self.machine_name,
'LrmsId':
lambda x: x['clusterid'] + "_" + self.machine_name,
'Site':
lambda x: self.site_name,
'ValidFrom':
lambda x: valid_from(parse_timestamp(x['timestamp'])),
'ValidUntil':
lambda x: valid_until(parse_timestamp(x['timestamp'])),
'Processed':
lambda x: Parser.UNPROCESSED
}
for key, value in parts:
# Store only the first value encountered. This is mainly for the
# userFQAN field as the first occurence of this is the primary FQAN.
if key not in data:
data[key] = value
for key in mapping:
rc[key] = mapping[key](data)
record = HTCondorCERecord()
record.set_all(rc)
return record
