Example #1
0
 def assign(self, **kwds):
     """assign page"""
     if not kwds:
         kwds = {}
     if 'status' not in kwds:
         kwds.update({'status': 'assignment-approved'})
     docs = []
     attrs = [
         'RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus'
     ]
     data = self.reqmgr.getRequestByStatus(statusList=[kwds['status']])
     for val in data.values():
         docs.append(request_attr(val, attrs))
     sortby = kwds.get('sort', 'status')
     docs = [r for r in sort(docs, sortby)]
     misc_json = {
         'RequestPriority': 5000,
         'CMSSWVersion': releases(),
         'ScramArch': architectures(),
         'SiteWhitelist': SITE_CACHE.getData(),
         'SiteBlacklist': SITE_CACHE.getData(),
         'SubscriptionPriority': ['Low', 'Normal', 'High'],
         'CustodialSubType': ['Move', 'Replica'],
         'NonCustodialSubType': ['Move', 'Replica'],
         'MinMergeSize': 2147483648,
         'MaxMergeSize': 4294967296,
         'MaxMergeEvents': 50000,
         'MaxRSS': 20411724,
         'MaxVSize': 20411724,
         'SoftTimeout': 129600,
         'GracePeriod': 300,
         'BlockCloseMaxWaitTime': 66400,
         'BlockCloseMaxFiles': 500,
         'BlockCloseMaxEvents': 250000000,
         'BlockCloseMaxSize': 5000000000000,
         'AcquisitionEra': '',
         'ProcessingVersion': 1,
         'ProcessingString': '',
         'MergedLFNBase': lfn_bases(),
         'UnmergedLFNBase': lfn_unmerged_bases(),
         'Dashboard': dashboardActivities(),
         'Team': self.getTeams()
     }
     filter_sort = self.templatepage('filter_sort')
     content = self.templatepage('assign',
                                 sort=sortby,
                                 filter_sort_table=filter_sort,
                                 sites=SITE_CACHE.getData(),
                                 site_white_list=site_white_list(),
                                 site_black_list=site_black_list(),
                                 user=user(),
                                 user_dn=user_dn(),
                                 requests=toString(docs),
                                 misc_table=json2table(
                                     misc_json, web_ui_names(),
                                     "all_attributes"),
                                 misc_json=json2form(misc_json,
                                                     indent=2,
                                                     keep_first_value=True))
     return self.abs_page('assign', content)
Example #2
0
    def getWorkloadCreateArgs():
        baseArgs = StdBase.getWorkloadCreateArgs()
        specArgs = {"RequestType": {"default": "Express"},
                    "ConfigCacheID": {"optional": True, "null": True},
                    "Scenario": {"optional": False, "attr": "procScenario"},
                    "RecoCMSSWVersion": {"validate": lambda x: x in releases(),
                                         "optional": False, "attr": "recoFrameworkVersion"},
                    "RecoScramArch": {"validate": lambda x: all([y in architectures() for y in x]),
                                      "optional": False, "type": makeNonEmptyList},
                    "GlobalTag": {"optional": False},
                    "GlobalTagTransaction": {"optional": False},
                    "ProcessingString": {"default": "", "validate": procstringT0},
                    "StreamName": {"optional": False},
                    "SpecialDataset": {"optional": False},
                    "AlcaHarvestTimeout": {"type": int, "optional": False},
                    "AlcaHarvestDir": {"optional": False, "null": True},
                    "AlcaSkims": {"type": makeList, "optional": False},
                    "DQMSequences": {"type": makeList, "attr": "dqmSequences", "optional": False},
                    "Outputs": {"type": makeList, "optional": False},
                    "MaxInputRate": {"type": int, "optional": False},
                    "MaxInputEvents": {"type": int, "optional": False},
                    "MaxInputSize": {"type": int, "optional": False},
                    "MaxInputFiles": {"type": int, "optional": False},
                    "MaxLatency": {"type": int, "optional": False},

                    }
        baseArgs.update(specArgs)
        StdBase.setDefaultArgumentsProperty(baseArgs)
        return baseArgs
Example #3
0
 def assign(self, **kwds):
     """assign page"""
     if not kwds:
         kwds = {}
     if 'status' not in kwds:
         kwds.update({'status': 'assignment-approved'})
     docs = []
     attrs = ['RequestName', 'RequestDate', 'Group', 'Requestor', 'RequestStatus']
     dataResult = self.reqmgr.getRequestByStatus(statusList=[kwds['status']])
     for data in dataResult:
         for val in data.values():
             docs.append(request_attr(val, attrs))
     sortby = kwds.get('sort', 'status')
     docs = [r for r in sort(docs, sortby)]
     misc_json = {'RequestPriority': 5000,
                  'CMSSWVersion': releases(),
                  'ScramArch': architectures(),
                  'SiteWhitelist': SITE_CACHE.getData(),
                  'SiteBlacklist': SITE_CACHE.getData(),
                  'SubscriptionPriority': ['Low', 'Normal', 'High'],
                  'CustodialSubType': ['Move', 'Replica'],
                  'NonCustodialSubType': ['Move', 'Replica'],
                  'MinMergeSize': 2147483648,
                  'MaxMergeSize': 4294967296,
                  'MaxMergeEvents': 50000,
                  'MaxRSS': 20411724,
                  'MaxVSize': 20411724,
                  'SoftTimeout': 129600,
                  'GracePeriod': 300,
                  'BlockCloseMaxWaitTime': 66400,
                  'BlockCloseMaxFiles': 500,
                  'BlockCloseMaxEvents': 250000000,
                  'BlockCloseMaxSize': 5000000000000,
                  'AcquisitionEra': '',
                  'ProcessingVersion': 1,
                  'ProcessingString': '',
                  'MergedLFNBase': lfn_bases(),
                  'UnmergedLFNBase': lfn_unmerged_bases(),
                  'Dashboard': dashboardActivities(),
                  'Team': self.getTeams()}
     filter_sort = self.templatepage('filter_sort')
     content = self.templatepage('assign', sort=sortby,
                                 filter_sort_table=filter_sort,
                                 sites=SITE_CACHE.getData(),
                                 site_white_list=site_white_list(),
                                 site_black_list=site_black_list(),
                                 user=user(), user_dn=user_dn(), requests=toString(docs),
                                 misc_table=json2table(misc_json, web_ui_names(), "all_attributes"),
                                 misc_json=json2form(misc_json, indent=2, keep_first_value=True))
     return self.abs_page('assign', content)
Example #4
0
def getPropValueMap():
    """
    Return all possible values for some assignment arguments
    """
    prop_value_map = {'CMSSWVersion': releases(),
                      'SiteWhitelist': SITE_CACHE.getData(),
                      'SiteBlacklist': SITE_CACHE.getData(),
                      'SubscriptionPriority': ['Low', 'Normal', 'High'],
                      'CustodialSites': PNN_CACHE.getData(),
                      'CustodialSubType': ['Move', 'Replica'],
                      'NonCustodialSites': PNN_CACHE.getData(),
                      'NonCustodialSubType': ['Move', 'Replica'],
                      'AutoApproveSubscriptionSites': PNN_CACHE.getData(),
                      'MergedLFNBase': lfn_bases(),
                      'UnmergedLFNBase': lfn_unmerged_bases(),
                      'TrustPUSitelists': [True, False],
                      'TrustSitelists': [True, False],
                      'Dashboard': dashboardActivities()}
    return prop_value_map
Example #5
0
def create_json_template_spec(specArgs):
    template = {}
    for key, prop in specArgs.items():
        
        if key == "RequestorDN":
            # this will be automatically collected so skip it.
            continue
        
        if key == "CMSSWVersion":
            # get if from tag collector
            value = releases()
        elif key == "ScramArch":
            value = architectures()
        elif prop.get("optional", True):
            # if optional need to always have default value
            value = prop["default"]
        else:
            value = "REPLACE-%s" % key
        template[key] = value
    return template
Example #6
0
def create_json_template_spec(specArgs):
    template = {}
    for key, prop in specArgs.items():

        if key == "RequestorDN":
            # this will be automatically collected so skip it.
            continue

        if key == "CMSSWVersion":
            # get if from tag collector
            value = releases()
        elif key == "ScramArch":
            value = architectures()
        elif prop.get("optional", True):
            # if optional need to always have default value
            value = prop["default"]
        else:
            value = "REPLACE-%s" % key
        template[key] = value
    return template
Example #7
0
 def getWorkloadCreateArgs():
     baseArgs = StdBase.getWorkloadCreateArgs()
     specArgs = {
         "RequestType": {
             "default": "Express"
         },
         "ConfigCacheID": {
             "optional": True,
             "null": True
         },
         "Scenario": {
             "optional": False,
             "attr": "procScenario"
         },
         "RecoCMSSWVersion": {
             "validate": lambda x: x in releases(),
             "optional": False,
             "attr": "recoFrameworkVersion"
         },
         "RecoScramArch": {
             "validate": lambda x: all([y in architectures() for y in x]),
             "optional": False,
             "type": makeNonEmptyList
         },
         "GlobalTag": {
             "optional": False
         },
         "GlobalTagTransaction": {
             "optional": False
         },
         "ProcessingString": {
             "default": "",
             "validate": procstringT0
         },
         "StreamName": {
             "optional": False
         },
         "SpecialDataset": {
             "optional": False
         },
         "AlcaHarvestTimeout": {
             "type": int,
             "optional": False
         },
         "AlcaHarvestDir": {
             "optional": False,
             "null": True
         },
         "AlcaSkims": {
             "type": makeList,
             "optional": False
         },
         "DQMSequences": {
             "type": makeList,
             "attr": "dqmSequences",
             "optional": False
         },
         "Outputs": {
             "type": makeList,
             "optional": False
         },
         "MaxInputRate": {
             "type": int,
             "optional": False
         },
         "MaxInputEvents": {
             "type": int,
             "optional": False
         },
         "MaxInputSize": {
             "type": int,
             "optional": False
         },
         "MaxInputFiles": {
             "type": int,
             "optional": False
         },
         "MaxLatency": {
             "type": int,
             "optional": False
         },
     }
     baseArgs.update(specArgs)
     StdBase.setDefaultArgumentsProperty(baseArgs)
     return baseArgs
Example #8
0
    def fetch(self, rid):
        "Fetch document for given id"
        rid = rid.replace('request-', '')
        doc = self.reqmgr.getRequestByNames(rid)
        transitions = []
        tst = time.time()
        # get request tasks
        tasks = self.reqmgr.getRequestTasks(rid)
        if len(doc) == 1:
            try:
                doc = doc[rid]
            except:
                pass
            name = doc.get('RequestName', 'NA')
            title = 'Request %s' % name
            status = doc.get('RequestStatus', '')
            transitions = REQUEST_STATE_TRANSITION.get(status, [])
            if status in transitions:
                transitions.remove(status)
            visible_attrs = get_modifiable_properties(status)
            filterout_attrs = get_protected_properties()

            #extend filterout list with "RequestStatus" since it is passed separately
            filterout_attrs.append("RequestStatus")

            if visible_attrs == "all_attributes":
                filteredDoc = doc
                for prop in filterout_attrs:
                    if prop in filteredDoc:
                        del filteredDoc[prop]
            else:
                filteredDoc = {}
                for prop in visible_attrs:
                    filteredDoc[prop] = doc.get(prop, "")

            listPNNs = PNN_CACHE.getData()
            prop_value_map = {
                'CMSSWVersion': releases(),
                'SiteWhitelist': SITE_CACHE.getData(),
                'SiteBlacklist': SITE_CACHE.getData(),
                'SubscriptionPriority': ['Low', 'Normal', 'High'],
                'CustodialSites': listPNNs,
                'CustodialSubType': ['Move', 'Replica'],
                'NonCustodialSites': listPNNs,
                'NonCustodialSubType': ['Move', 'Replica'],
                'AutoApproveSubscriptionSites': listPNNs,
                'MergedLFNBase': lfn_bases(),
                'UnmergedLFNBase': lfn_unmerged_bases(),
                'TrustPUSitelists': [True, False],
                'TrustSitelists': [True, False],
                'Dashboard': dashboardActivities(),
                'Team': self.getTeams()
            }

            selected = {}
            for prop in prop_value_map:
                if prop in filteredDoc:
                    filteredDoc[prop], selected[prop] = reorder_list(
                        prop_value_map[prop], filteredDoc[prop])

            content = self.templatepage(
                'doc',
                title=title,
                status=status,
                name=name,
                rid=rid,
                tasks=json2form(tasks, indent=2, keep_first_value=False),
                table=json2table(filteredDoc, web_ui_names(), visible_attrs,
                                 selected),
                jsondata=json2form(doc, indent=2, keep_first_value=False),
                doc=json.dumps(doc),
                time=time,
                transitions=transitions,
                ts=tst,
                user=user(),
                userdn=user_dn())
        elif len(doc) > 1:
            jsondata = [pprint.pformat(d) for d in doc]
            content = self.templatepage('doc',
                                        title='Series of docs: %s' % rid,
                                        table="",
                                        jsondata=jsondata,
                                        time=time,
                                        transitions=transitions,
                                        ts=tst,
                                        user=user(),
                                        userdn=user_dn())
        else:
            doc = 'No request found for name=%s' % rid
        return self.abs_page('request', content)
Example #9
0
    def fetch(self, rid):
        "Fetch document for given id"
        rid = rid.replace('request-', '')
        doc = self.reqmgr.getRequestByNames(rid)
        transitions = []
        tst = time.time()
        # get request tasks
        tasks = self.reqmgr.getRequestTasks(rid)
        if len(doc) == 1:
            try:
                doc = doc[rid]
            except:
                pass
            name = doc.get('RequestName', 'NA')
            title = 'Request %s' % name
            status = doc.get('RequestStatus', '')
            transitions = REQUEST_STATE_TRANSITION.get(status, [])
            if status in transitions:
                transitions.remove(status)
            visible_attrs = get_modifiable_properties(status)
            filterout_attrs = get_protected_properties()
            
            #extend filterout list with "RequestStatus" since it is passed separately
            filterout_attrs.append("RequestStatus")
            
            if visible_attrs == "all_attributes":
                filteredDoc = doc
                for prop in filterout_attrs:
                    if prop in filteredDoc:
                        del filteredDoc[prop]                    
            else:
                filteredDoc = {}
                for prop in visible_attrs:
                    filteredDoc[prop] = doc.get(prop, "")

            listPNNs = PNN_CACHE.getData()
            prop_value_map = {'CMSSWVersion': releases(),
                              'SiteWhitelist': SITE_CACHE.getData(),
                              'SiteBlacklist': SITE_CACHE.getData(),
                              'SubscriptionPriority': ['Low', 'Normal', 'High'],
                              'CustodialSites': listPNNs,
                              'CustodialSubType': ['Move', 'Replica'],
                              'NonCustodialSites': listPNNs,
                              'NonCustodialSubType': ['Move', 'Replica'],
                              'AutoApproveSubscriptionSites': listPNNs,
                              'MergedLFNBase': lfn_bases(),
                              'UnmergedLFNBase': lfn_unmerged_bases(),
                              'TrustPUSitelists': [True, False],
                              'TrustSitelists': [True, False],                    
                              'Dashboard': dashboardActivities(),
                              'Team': self.getTeams()}
            
            selected = {}
            for prop in prop_value_map:
                if prop in filteredDoc:
                    filteredDoc[prop], selected[prop] = reorder_list(prop_value_map[prop], filteredDoc[prop])
                    
            content = self.templatepage('doc', title=title, status=status, name=name, rid=rid,
                                        tasks=json2form(tasks, indent=2, keep_first_value=False),
                                        table=json2table(filteredDoc, web_ui_names(), visible_attrs, selected),
                                        jsondata=json2form(doc, indent=2, keep_first_value=False),
                                        doc=json.dumps(doc), time=time,
                                        transitions=transitions, ts=tst, user=user(), userdn=user_dn())
        elif len(doc) > 1:
            jsondata = [pprint.pformat(d) for d in doc]
            content = self.templatepage('doc', title='Series of docs: %s' % rid,
                                        table="", jsondata=jsondata, time=time,
                                        transitions=transitions, ts=tst, user=user(), userdn=user_dn())
        else:
            doc = 'No request found for name=%s' % rid
        return self.abs_page('request', content)