def process_lumis_with(ikey, gen):
"Helper function to process lumis with given key from provided generator"
odict = {}
for row in gen:
if "error" in row:
yield row
continue
lfn, run, lumi = row
if ikey == "file":
key = lfn
elif ikey == "run":
key = run
elif ikey == "file_run" or "block_run":
key = (lfn, run) # here lfn refers either to lfn or block
if isinstance(lumi, list):
for ilumi in lumi:
odict.setdefault(key, []).append(ilumi)
else:
odict.setdefault(key, []).append(ilumi)
for key, lumi_list in odict.iteritems():
lumi_list.sort()
lumis = convert2ranges(lumi_list)
if ikey == "file":
yield {"file": {"name": key}, "lumi": {"number": lumis}}
elif ikey == "run":
yield {"run": {"run_number": key}, "lumi": {"number": lumis}}
elif ikey == "file_run":
lfn, run = key
yield {"run": {"run_number": run}, "lumi": {"number": lumis}, "file": {"name": lfn}}
elif ikey == "block_run":
blk, run = key
yield {"run": {"run_number": run}, "lumi": {"number": lumis}, "block": {"name": blk}}
def process_lumis_with(ikey, gen):
"Helper function to process lumis with given key from provided generator"
odict = {}
for row in gen:
if 'error' in row:
yield row
continue
lfn, run, lumi = row
if ikey == 'file':
key = lfn
elif ikey == 'run':
key = run
elif ikey == 'file_run' or 'block_run':
key = (lfn, run) # here lfn refers either to lfn or block
if isinstance(lumi, list):
for ilumi in lumi:
odict.setdefault(key, []).append(ilumi)
else:
odict.setdefault(key, []).append(ilumi)
for key, lumi_list in odict.items():
lumi_list.sort()
lumis = convert2ranges(lumi_list)
if ikey == 'file':
yield {'file':{'name':key}, 'lumi':{'number':lumis}}
elif ikey == 'run':
yield {'run':{'run_number':key}, 'lumi':{'number':lumis}}
elif ikey == 'file_run':
lfn, run = key
yield {'run':{'run_number':run}, 'lumi':{'number':lumis},
'file':{'name': lfn}}
elif ikey == 'block_run':
blk, run = key
yield {'run':{'run_number':run}, 'lumi':{'number':lumis},
'block':{'name': blk}}
def collect_lumis(records):
"Helper function to collect lumi records for same runs"
rec = {}
for row in records:
if not rec:
rec.update(row)
continue
if row['lumi']['run_number'] == rec['lumi']['run_number']:
lumis = rec['lumi']
lumis['number'] += row['lumi']['number']
rec['lumi'] = lumis
else:
rec['lumi']['number'] = convert2ranges(rec['lumi']['number'])
yield rec
rec = {}
if rec:
rec['lumi']['number'] = convert2ranges(rec['lumi']['number'])
yield rec
def collect_lumis(records):
"Helper function to collect lumi records for same runs"
rec = {}
for row in records:
if not rec:
rec.update(row)
continue
if row['lumi']['run_number'] == rec['lumi']['run_number']:
lumis = rec['lumi']
lumis['number'] += row['lumi']['number']
rec['lumi'] = lumis
else:
rec['lumi']['number'] = convert2ranges(rec['lumi']['number'])
yield rec
rec = {}
if rec:
rec['lumi']['number'] = convert2ranges(rec['lumi']['number'])
yield rec
def test_convert2ranges(self):
"Test convert2ranges function"
rows = [5, 1, 1, 1, 2, 2, 3, 3, 3, 4, 8, 9, 1]
expect = [[1,5], [8,9]]
result = [r for r in convert2ranges(rows)]
self.assertEqual(result, expect)
def parser_helper(self, query, dformat, source, api):
"""
DBS3 data-service parser helper, it is used by parser method.
"""
if api in ['site4dataset', 'site4block']:
gen = json_parser(source, self.logger)
else:
gen = DASAbstractService.parser(self, query, dformat, source, api)
if api in ['site4dataset', 'site4block']:
sites = set()
for rec in gen:
if isinstance(rec, list):
for row in rec:
orig_site = row['origin_site_name']
if orig_site not in sites:
sites.add(orig_site)
else:
orig_site = rec.get('origin_site_name', None)
if orig_site and orig_site not in sites:
sites.add(orig_site)
for site in sites:
yield {'site': {'name': site}}
elif api == 'datasets' or api == 'dataset_info' or api == 'datasetlist':
for row in gen:
row['name'] = row['dataset']
del row['dataset']
yield {'dataset':row}
elif api == 'filesummaries':
name = query.mongo_query['spec']['dataset.name']
for row in gen:
row['dataset']['name'] = name
yield row
elif api == 'summary4dataset_run' or api == 'summary4block_run':
spec = query.mongo_query.get('spec', {})
dataset = spec.get('dataset.name', '')
block = spec.get('block.name', '')
run = spec.get('run.run_number', 0)
if isinstance(run, dict): # we got a run range
if '$in' in run:
run = run['$in']
elif '$lte' in run:
run = range(run['$gte'], run['$lte'])
for row in gen:
if run:
row.update({"run": run})
if dataset:
row.update({"dataset": dataset})
if block:
row.update({"block": block})
yield row
elif api == 'releaseversions':
for row in gen:
values = row['release']['release_version']
for val in values:
yield dict(release=dict(name=val))
elif api == 'datasetaccesstypes':
for row in gen:
values = row['status']['dataset_access_type']
for val in values:
yield dict(status=dict(name=val))
elif api == 'blockorigin':
for row in gen:
yield row
elif api == 'blockparents':
for row in gen:
try:
del row['parent']['this_block_name']
except:
pass
yield row
elif api == 'fileparents':
for row in gen:
parent = row['parent']
for val in parent['parent_logical_file_name']:
yield dict(name=val)
elif api == 'runs_via_dataset' or api == 'runs':
for row in gen:
values = row.get('run', {}).get('run_num', 'N/A')
if isinstance(values, list):
for val in values:
yield dict(run_number=val)
else:
yield dict(run_number=values)
elif api == 'filechildren':
for row in gen:
parent = row['child']
for val in parent['child_logical_file_name']:
yield dict(name=val)
elif api == 'files' or api == 'files_via_dataset' or \
api == 'files_via_block':
status = 'VALID'
for row in gen:
if 'spec' in query.mongo_query:
if 'status.name' in query.mongo_query['spec']:
status = query.mongo_query['spec']['status.name']
try:
file_status = row['file']['is_file_valid']
except KeyError:
file_status = 0 # file status is unknown
if status == '*': # any file
pass
elif status == 'INVALID': # filter out valid files
if int(file_status) == 1:# valid status
row = None
else: # filter out invalid files
if int(file_status) == 0:# invalid status
row = None
if row:
yield row
elif api == 'filelumis' or api == 'filelumis4block':
for row in gen:
if 'lumi' in row:
if 'lumi_section_num' in row['lumi']:
val = row['lumi']['lumi_section_num']
row['lumi']['lumi_section_num'] = convert2ranges(val)
yield row
else:
yield row
else:
for row in gen:
yield row
def parse_run_dict(rdict):
"Parser input run dict and normalize lumi lists"
for key, val in rdict.items():
rdict[key] = convert2ranges(val)
def parser_helper(self, query, dformat, source, api):
"""
DBS3 data-service parser helper, it is used by parser method.
"""
if api == "site4dataset":
gen = json_parser(source, self.logger)
else:
gen = DASAbstractService.parser(self, query, dformat, source, api)
if api == "site4dataset":
sites = set()
for rec in gen:
if isinstance(rec, list):
for row in rec:
orig_site = row["origin_site_name"]
if orig_site not in sites:
sites.add(orig_site)
else:
orig_site = rec.get("origin_site_name", None)
if orig_site and orig_site not in sites:
sites.add(orig_site)
for site in sites:
yield {"site": {"name": site}}
elif api == "datasets" or api == "dataset_info":
for row in gen:
row["name"] = row["dataset"]
del row["dataset"]
yield {"dataset": row}
elif api == "filesummaries":
name = query.mongo_query["spec"]["dataset.name"]
for row in gen:
row["dataset"]["name"] = name
yield row
elif api == "summary4dataset_run" or api == "summary4block_run":
spec = query.mongo_query.get("spec", {})
dataset = spec.get("dataset.name", "")
block = spec.get("block.name", "")
run = spec.get("run.run_number", 0)
if isinstance(run, dict): # we got a run range
if "$in" in run:
run = run["$in"]
elif "$lte" in run:
run = range(run["$gte"], run["$lte"])
for row in gen:
if run:
row.update({"run": run})
if dataset:
row.update({"dataset": dataset})
if block:
row.update({"block": block})
yield row
elif api == "blockorigin":
for row in gen:
yield row
elif api == "blockparents":
for row in gen:
try:
del row["parent"]["this_block_name"]
except:
pass
yield row
elif api == "fileparents":
for row in gen:
parent = row["parent"]
for val in parent["parent_logical_file_name"]:
yield dict(name=val)
elif api == "runs_via_dataset" or api == "runs":
for row in gen:
values = row["run"]["run_num"]
if isinstance(values, list):
for val in values:
yield dict(run_number=val)
else:
yield dict(run_number=values)
elif api == "filechildren":
for row in gen:
parent = row["child"]
for val in parent["child_logical_file_name"]:
yield dict(name=val)
elif api == "files" or api == "files_via_dataset" or api == "files_via_block":
status = "VALID"
for row in gen:
if "spec" in query.mongo_query:
if "status.name" in query.mongo_query["spec"]:
status = query.mongo_query["spec"]["status.name"]
file_status = row["file"]["is_file_valid"]
if status == "INVALID": # filter out valid files
if int(file_status) == 1: # valid status
row = None
else: # filter out invalid files
if int(file_status) == 0: # invalid status
row = None
if row:
yield row
elif api == "filelumis" or api == "filelumis4block":
for row in gen:
if "lumi" in row:
if "lumi_section_num" in row["lumi"]:
val = row["lumi"]["lumi_section_num"]
row["lumi"]["lumi_section_num"] = convert2ranges(val)
yield row
else:
yield row
else:
for row in gen:
yield row
def test_convert2ranges(self):
"Test convert2ranges function"
rows = [5, 1, 1, 1, 2, 2, 3, 3, 3, 4, 8, 9, 1]
expect = [[1, 5], [8, 9]]
result = [r for r in convert2ranges(rows)]
self.assertEqual(result, expect)
def adjust_values(func, gen, links, pkey):
"""
Helper function to adjust values in UI.
It groups values for identical key, make links for provided mapped function,
represent "Number of" keys as integers and represents size values in GB format.
The mapped function is the one from das_mapping_db which convert
UI key into triplet of das key, das access key and link, see
das_mapping_db:daskey_from_presentation
"""
rdict = {}
uidict = {}
for uikey, value, uilink, uidesc, uiexamples in [k for k, _g in groupby(gen)]:
val = quote(value)
if uikey in rdict:
existing_val = rdict[uikey]
if not isinstance(existing_val, list):
existing_val = [existing_val]
if val not in existing_val:
rdict[uikey] = existing_val + [val]
else:
rdict[uikey] = val
uidict[uikey] = (uilink, uidesc, uiexamples)
page = ""
to_show = []
green = 'style="color:green"'
red = 'style="color:red"'
for key, val in rdict.items():
uilink, _uidesc, _uiexamples = uidict[key]
if uilink and val:
if not isinstance(val, list):
val = [val]
values = []
for elem in val:
for ilink in uilink:
dasquery = ilink['query'] % elem
val = '<a href="/das/request?input=%s">%s</a>' \
% (dasquery, elem)
values.append(val)
to_show.append((key, ', '.join(values)))
continue
lookup = func(key)
if key.lower() == 'reason' or key.lower() == 'qhash' or key.lower() == 'hints':
continue
if key.lower() == 'error':
key = '<span %s>WARNING</span>' % red
val = json.dumps(val) + ', click on show link to get more info<br/>'
if lookup:
if key.find('Member') != -1 and val:
link = '/das/request?input=user%3D'
if isinstance(val, list):
val = ['<a href="%s%s">%s</a>' \
% (link, quote(v), quote(v)) for v in val]
elif isinstance(val, basestring):
val = '<a href="%s%s">%s</a>' \
% (link, quote(val), quote(val))
if key.find('Config urls') != -1 and val:
if isinstance(val, dict):
urls = []
for rtype, rurls in val.items():
for vdx in range(len(rurls)):
urls.append('<a href="%s">%s-config-%d</a>' % (rurls[vdx], rtype, vdx+1))
value = ', '.join(urls)
else:
value = '<a href="%s">config</a>' % val
key = tooltip_helper(key)
elif isinstance(val, list):
value = ', '.join([str(v) for v in val])
try:
length = len(set(val))
except TypeError: # happens when val list contains a dict
length = len(val)
if length > 1 and \
(key.lower().find('number') != -1 or \
key.lower().find('size') != -1):
if key not in ['Luminosity number', 'Run number']:
value = '<span %s>%s</span>' % (red, value)
elif key.lower().find('size') != -1 and val:
value = size_format(val)
elif key.find('Number of ') != -1 and val and int(val) != 0:
value = int(val)
elif key.find('Run number') != -1 and val:
value = int(val)
elif key.find('Lumi') != -1 and val:
value = int(val)
elif key.find('Site type') != -1 and val:
if isinstance(val, basestring) and val.lower() == 'disk':
value = val
else:
value = '<span %s><b>TAPE</b></span> <b>no user access</b>' % red
elif key.find('Tag') != -1 and val:
if isinstance(val, basestring) and val.lower() == 'unknown':
value = '<span %s>%s</span>' % (red, val)
else:
value = val
elif key.find('Creation time') != -1 and val:
try:
value = time.strftime('%d/%b/%Y %H:%M:%S GMT', \
time.gmtime(val))
except:
value = val
else:
value = val
if isinstance(value, list) and isinstance(value[0], str):
value = ', '.join(value)
if key == 'Open' or key == 'Custodial':
if value == 'n':
value = '<span %s>%s</span>' % (green, value)
else:
value = '<span %s>%s</span>' % (red, value)
if key.lower().find('presence') != -1 or \
key.lower().find('completion') != -1:
if not value:
continue
else:
if value == '100.00%':
value = '<span %s>100%%</span>' % green
else:
value = '<span %s>%s</span>' % (red, value)
key = tooltip_helper(key)
to_show.append((key, value))
else:
if key == 'result' and isinstance(val, dict) and \
'value' in val: # result of aggregation function
if 'key' in rdict and rdict['key'].find('.size') != -1:
val = size_format(val['value'])
elif isinstance(val['value'], float):
val = '%.2f' % val['value']
else:
val = val['value']
to_show.append((key, val))
if to_show:
to_show = list(set(to_show))
page += '<br />'
tdict = {}
for key, val in to_show:
tdict[key] = val
result_keys = ['function', 'result', 'key']
if set(tdict.keys()) & set(result_keys) == set(result_keys):
page += '%s(%s)=%s' \
% (tdict['function'], tdict['key'], tdict['result'])
elif sorted(tdict.keys()) == sorted(['Luminosity number', 'Run number']):
page += 'Run number: %s, Luminosity ranges: %s' \
% (tdict['Run number'], convert2ranges(rdict['Luminosity number']))
elif sorted(tdict.keys()) == sorted(['Events', 'Luminosity number', 'Run number']):
page += 'Run number: %s, Luminosity ranges: %s' \
% (tdict['Run number'], convert2ranges(rdict['Luminosity number']))
page += lumi_evts(rdict)
elif sorted(tdict.keys()) == sorted(['Luminosity number']):
page += 'Luminosity ranges: %s' \
% (convert2ranges(rdict['Luminosity number']))
elif sorted(tdict.keys()) == sorted(['Events', 'Luminosity number']):
page += 'Luminosity ranges: %s' \
% (convert2ranges(rdict['Luminosity number']))
page += lumi_evts(rdict)
else:
rlist = ["%s: %s" \
% (k[0].capitalize()+k[1:], v) for k, v in to_show]
rlist.sort()
page += ', '.join(rlist)
page = page.replace('<br/>,', '<br/>')
if links:
page += '<br />' + ', '.join(links)
return page