def assign(self, **kwds):
"""assign page"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'assignment-approved'})
docs = []
attrs = [
'RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus'
]
data = self.reqmgr.getRequestByStatus(statusList=[kwds['status']])
for val in data.values():
docs.append(request_attr(val, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
misc_json = {
'RequestPriority': 5000,
'CMSSWVersion': releases(),
'ScramArch': architectures(),
'SiteWhitelist': SITE_CACHE.getData(),
'SiteBlacklist': SITE_CACHE.getData(),
'SubscriptionPriority': ['Low', 'Normal', 'High'],
'CustodialSubType': ['Move', 'Replica'],
'NonCustodialSubType': ['Move', 'Replica'],
'MinMergeSize': 2147483648,
'MaxMergeSize': 4294967296,
'MaxMergeEvents': 50000,
'MaxRSS': 20411724,
'MaxVSize': 20411724,
'SoftTimeout': 129600,
'GracePeriod': 300,
'BlockCloseMaxWaitTime': 66400,
'BlockCloseMaxFiles': 500,
'BlockCloseMaxEvents': 250000000,
'BlockCloseMaxSize': 5000000000000,
'AcquisitionEra': '',
'ProcessingVersion': 1,
'ProcessingString': '',
'MergedLFNBase': lfn_bases(),
'UnmergedLFNBase': lfn_unmerged_bases(),
'Dashboard': dashboardActivities(),
'Team': self.getTeams()
}
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('assign',
sort=sortby,
filter_sort_table=filter_sort,
sites=SITE_CACHE.getData(),
site_white_list=site_white_list(),
site_black_list=site_black_list(),
user=user(),
user_dn=user_dn(),
requests=toString(docs),
misc_table=json2table(
misc_json, web_ui_names(),
"all_attributes"),
misc_json=json2form(misc_json,
indent=2,
keep_first_value=True))
return self.abs_page('assign', content)
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {"RequestType": {"default": "Express"},
"ConfigCacheID": {"optional": True, "null": True},
"Scenario": {"optional": False, "attr": "procScenario"},
"RecoCMSSWVersion": {"validate": lambda x: x in releases(),
"optional": False, "attr": "recoFrameworkVersion"},
"RecoScramArch": {"validate": lambda x: all([y in architectures() for y in x]),
"optional": False, "type": makeNonEmptyList},
"GlobalTag": {"optional": False},
"GlobalTagTransaction": {"optional": False},
"ProcessingString": {"default": "", "validate": procstringT0},
"StreamName": {"optional": False},
"SpecialDataset": {"optional": False},
"AlcaHarvestTimeout": {"type": int, "optional": False},
"AlcaHarvestDir": {"optional": False, "null": True},
"AlcaSkims": {"type": makeList, "optional": False},
"DQMSequences": {"type": makeList, "attr": "dqmSequences", "optional": False},
"Outputs": {"type": makeList, "optional": False},
"MaxInputRate": {"type": int, "optional": False},
"MaxInputEvents": {"type": int, "optional": False},
"MaxInputSize": {"type": int, "optional": False},
"MaxInputFiles": {"type": int, "optional": False},
"MaxLatency": {"type": int, "optional": False},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def assign(self, **kwds):
"""assign page"""
if not kwds:
kwds = {}
if 'status' not in kwds:
kwds.update({'status': 'assignment-approved'})
docs = []
attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus']
dataResult = self.reqmgr.getRequestByStatus(statusList=[kwds['status']])
for data in dataResult:
for val in data.values():
docs.append(request_attr(val, attrs))
sortby = kwds.get('sort', 'status')
docs = [r for r in sort(docs, sortby)]
misc_json = {'RequestPriority': 5000,
'CMSSWVersion': releases(),
'ScramArch': architectures(),
'SiteWhitelist': SITE_CACHE.getData(),
'SiteBlacklist': SITE_CACHE.getData(),
'SubscriptionPriority': ['Low', 'Normal', 'High'],
'CustodialSubType': ['Move', 'Replica'],
'NonCustodialSubType': ['Move', 'Replica'],
'MinMergeSize': 2147483648,
'MaxMergeSize': 4294967296,
'MaxMergeEvents': 50000,
'MaxRSS': 20411724,
'MaxVSize': 20411724,
'SoftTimeout': 129600,
'GracePeriod': 300,
'BlockCloseMaxWaitTime': 66400,
'BlockCloseMaxFiles': 500,
'BlockCloseMaxEvents': 250000000,
'BlockCloseMaxSize': 5000000000000,
'AcquisitionEra': '',
'ProcessingVersion': 1,
'ProcessingString': '',
'MergedLFNBase': lfn_bases(),
'UnmergedLFNBase': lfn_unmerged_bases(),
'Dashboard': dashboardActivities(),
'Team': self.getTeams()}
filter_sort = self.templatepage('filter_sort')
content = self.templatepage('assign', sort=sortby,
filter_sort_table=filter_sort,
sites=SITE_CACHE.getData(),
site_white_list=site_white_list(),
site_black_list=site_black_list(),
user=user(), user_dn=user_dn(), requests=toString(docs),
misc_table=json2table(misc_json, web_ui_names(), "all_attributes"),
misc_json=json2form(misc_json, indent=2, keep_first_value=True))
return self.abs_page('assign', content)
def getPropValueMap():
"""
Return all possible values for some assignment arguments
"""
prop_value_map = {'CMSSWVersion': releases(),
'SiteWhitelist': SITE_CACHE.getData(),
'SiteBlacklist': SITE_CACHE.getData(),
'SubscriptionPriority': ['Low', 'Normal', 'High'],
'CustodialSites': PNN_CACHE.getData(),
'CustodialSubType': ['Move', 'Replica'],
'NonCustodialSites': PNN_CACHE.getData(),
'NonCustodialSubType': ['Move', 'Replica'],
'AutoApproveSubscriptionSites': PNN_CACHE.getData(),
'MergedLFNBase': lfn_bases(),
'UnmergedLFNBase': lfn_unmerged_bases(),
'TrustPUSitelists': [True, False],
'TrustSitelists': [True, False],
'Dashboard': dashboardActivities()}
return prop_value_map
def create_json_template_spec(specArgs):
template = {}
for key, prop in specArgs.items():
if key == "RequestorDN":
# this will be automatically collected so skip it.
continue
if key == "CMSSWVersion":
# get if from tag collector
value = releases()
elif key == "ScramArch":
value = architectures()
elif prop.get("optional", True):
# if optional need to always have default value
value = prop["default"]
else:
value = "REPLACE-%s" % key
template[key] = value
return template
def create_json_template_spec(specArgs):
template = {}
for key, prop in specArgs.items():
if key == "RequestorDN":
# this will be automatically collected so skip it.
continue
if key == "CMSSWVersion":
# get if from tag collector
value = releases()
elif key == "ScramArch":
value = architectures()
elif prop.get("optional", True):
# if optional need to always have default value
value = prop["default"]
else:
value = "REPLACE-%s" % key
template[key] = value
return template
def getWorkloadCreateArgs():
baseArgs = StdBase.getWorkloadCreateArgs()
specArgs = {
"RequestType": {
"default": "Express"
},
"ConfigCacheID": {
"optional": True,
"null": True
},
"Scenario": {
"optional": False,
"attr": "procScenario"
},
"RecoCMSSWVersion": {
"validate": lambda x: x in releases(),
"optional": False,
"attr": "recoFrameworkVersion"
},
"RecoScramArch": {
"validate": lambda x: all([y in architectures() for y in x]),
"optional": False,
"type": makeNonEmptyList
},
"GlobalTag": {
"optional": False
},
"GlobalTagTransaction": {
"optional": False
},
"ProcessingString": {
"default": "",
"validate": procstringT0
},
"StreamName": {
"optional": False
},
"SpecialDataset": {
"optional": False
},
"AlcaHarvestTimeout": {
"type": int,
"optional": False
},
"AlcaHarvestDir": {
"optional": False,
"null": True
},
"AlcaSkims": {
"type": makeList,
"optional": False
},
"DQMSequences": {
"type": makeList,
"attr": "dqmSequences",
"optional": False
},
"Outputs": {
"type": makeList,
"optional": False
},
"MaxInputRate": {
"type": int,
"optional": False
},
"MaxInputEvents": {
"type": int,
"optional": False
},
"MaxInputSize": {
"type": int,
"optional": False
},
"MaxInputFiles": {
"type": int,
"optional": False
},
"MaxLatency": {
"type": int,
"optional": False
},
}
baseArgs.update(specArgs)
StdBase.setDefaultArgumentsProperty(baseArgs)
return baseArgs
def fetch(self, rid):
"Fetch document for given id"
rid = rid.replace('request-', '')
doc = self.reqmgr.getRequestByNames(rid)
transitions = []
tst = time.time()
# get request tasks
tasks = self.reqmgr.getRequestTasks(rid)
if len(doc) == 1:
try:
doc = doc[rid]
except:
pass
name = doc.get('RequestName', 'NA')
title = 'Request %s' % name
status = doc.get('RequestStatus', '')
transitions = REQUEST_STATE_TRANSITION.get(status, [])
if status in transitions:
transitions.remove(status)
visible_attrs = get_modifiable_properties(status)
filterout_attrs = get_protected_properties()
#extend filterout list with "RequestStatus" since it is passed separately
filterout_attrs.append("RequestStatus")
if visible_attrs == "all_attributes":
filteredDoc = doc
for prop in filterout_attrs:
if prop in filteredDoc:
del filteredDoc[prop]
else:
filteredDoc = {}
for prop in visible_attrs:
filteredDoc[prop] = doc.get(prop, "")
listPNNs = PNN_CACHE.getData()
prop_value_map = {
'CMSSWVersion': releases(),
'SiteWhitelist': SITE_CACHE.getData(),
'SiteBlacklist': SITE_CACHE.getData(),
'SubscriptionPriority': ['Low', 'Normal', 'High'],
'CustodialSites': listPNNs,
'CustodialSubType': ['Move', 'Replica'],
'NonCustodialSites': listPNNs,
'NonCustodialSubType': ['Move', 'Replica'],
'AutoApproveSubscriptionSites': listPNNs,
'MergedLFNBase': lfn_bases(),
'UnmergedLFNBase': lfn_unmerged_bases(),
'TrustPUSitelists': [True, False],
'TrustSitelists': [True, False],
'Dashboard': dashboardActivities(),
'Team': self.getTeams()
}
selected = {}
for prop in prop_value_map:
if prop in filteredDoc:
filteredDoc[prop], selected[prop] = reorder_list(
prop_value_map[prop], filteredDoc[prop])
content = self.templatepage(
'doc',
title=title,
status=status,
name=name,
rid=rid,
tasks=json2form(tasks, indent=2, keep_first_value=False),
table=json2table(filteredDoc, web_ui_names(), visible_attrs,
selected),
jsondata=json2form(doc, indent=2, keep_first_value=False),
doc=json.dumps(doc),
time=time,
transitions=transitions,
ts=tst,
user=user(),
userdn=user_dn())
elif len(doc) > 1:
jsondata = [pprint.pformat(d) for d in doc]
content = self.templatepage('doc',
title='Series of docs: %s' % rid,
table="",
jsondata=jsondata,
time=time,
transitions=transitions,
ts=tst,
user=user(),
userdn=user_dn())
else:
doc = 'No request found for name=%s' % rid
return self.abs_page('request', content)
def fetch(self, rid):
"Fetch document for given id"
rid = rid.replace('request-', '')
doc = self.reqmgr.getRequestByNames(rid)
transitions = []
tst = time.time()
# get request tasks
tasks = self.reqmgr.getRequestTasks(rid)
if len(doc) == 1:
try:
doc = doc[rid]
except:
pass
name = doc.get('RequestName', 'NA')
title = 'Request %s' % name
status = doc.get('RequestStatus', '')
transitions = REQUEST_STATE_TRANSITION.get(status, [])
if status in transitions:
transitions.remove(status)
visible_attrs = get_modifiable_properties(status)
filterout_attrs = get_protected_properties()
#extend filterout list with "RequestStatus" since it is passed separately
filterout_attrs.append("RequestStatus")
if visible_attrs == "all_attributes":
filteredDoc = doc
for prop in filterout_attrs:
if prop in filteredDoc:
del filteredDoc[prop]
else:
filteredDoc = {}
for prop in visible_attrs:
filteredDoc[prop] = doc.get(prop, "")
listPNNs = PNN_CACHE.getData()
prop_value_map = {'CMSSWVersion': releases(),
'SiteWhitelist': SITE_CACHE.getData(),
'SiteBlacklist': SITE_CACHE.getData(),
'SubscriptionPriority': ['Low', 'Normal', 'High'],
'CustodialSites': listPNNs,
'CustodialSubType': ['Move', 'Replica'],
'NonCustodialSites': listPNNs,
'NonCustodialSubType': ['Move', 'Replica'],
'AutoApproveSubscriptionSites': listPNNs,
'MergedLFNBase': lfn_bases(),
'UnmergedLFNBase': lfn_unmerged_bases(),
'TrustPUSitelists': [True, False],
'TrustSitelists': [True, False],
'Dashboard': dashboardActivities(),
'Team': self.getTeams()}
selected = {}
for prop in prop_value_map:
if prop in filteredDoc:
filteredDoc[prop], selected[prop] = reorder_list(prop_value_map[prop], filteredDoc[prop])
content = self.templatepage('doc', title=title, status=status, name=name, rid=rid,
tasks=json2form(tasks, indent=2, keep_first_value=False),
table=json2table(filteredDoc, web_ui_names(), visible_attrs, selected),
jsondata=json2form(doc, indent=2, keep_first_value=False),
doc=json.dumps(doc), time=time,
transitions=transitions, ts=tst, user=user(), userdn=user_dn())
elif len(doc) > 1:
jsondata = [pprint.pformat(d) for d in doc]
content = self.templatepage('doc', title='Series of docs: %s' % rid,
table="", jsondata=jsondata, time=time,
transitions=transitions, ts=tst, user=user(), userdn=user_dn())
else:
doc = 'No request found for name=%s' % rid
return self.abs_page('request', content)