Пример #1
0
    if "RunWhitelist" in schema:
        helper.setRunWhitelist(schema["RunWhitelist"])
    # can't save Request object directly, because it makes it hard to retrieve the _rev
    metadata = {}
    metadata.update(request)
    # Add the output datasets if necessary
    for ds in helper.listOutputDatasets():
        if not ds in request['OutputDatasets']:
            request['OutputDatasets'].append(ds)
    # don't want to JSONify the whole workflow
    del metadata['WorkloadSpec']
    workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
    request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
    try:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        CheckIn.checkIn(request, kwargs['RequestType'], wmstatSvc)
    except CheckIn.RequestCheckInError, ex:
        msg = ex._message
        raise HTTPError(400, "Error in Request check-in: %s" % msg)
    return request

def requestDetails(requestName):
    """ Adds details from the Couch document as well as the database """
    WMCore.Lexicon.identifier(requestName)
    request = GetRequest.getRequestDetails(requestName)
    helper = loadWorkload(request)
    schema = helper.data.request.schema.dictionary_whole_tree_()
    # take the stuff from the DB preferentially
    schema.update(request)
    task = helper.getTopLevelTask()[0]
    schema['Site Whitelist']  = task.siteWhitelist()
Пример #2
0
    # Add the output datasets if necessary
    # for some bizarre reason OutpuDatasets is list of lists, when cloning
    # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
    # #3743
    if not clone:
        for ds in helper.listOutputDatasets():
            if ds not in request['OutputDatasets']:
                request['OutputDatasets'].append(ds)

    # don't want to JSONify the whole workflow
    del metadata['WorkloadSpec']
    workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
    request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
    try:
        CheckIn.checkIn(request, reqSchema['RequestType'])
    except CheckIn.RequestCheckInError, ex:
        msg = ex._message
        raise HTTPError(400, "Error in Request check-in: %s" % msg)

    try:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        wmstatSvc.insertRequest(request)
    except Exception as ex:
        webApi.error("Could not update WMStats, reason: %s" % ex)
        raise HTTPError(400,
                        "Creating request failed, could not update WMStats.")

    return request

Пример #3
0
 
 # Add the output datasets if necessary
 # for some bizarre reason OutpuDatasets is list of lists, when cloning
 # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
 # #3743
 if not clone:
     for ds in helper.listOutputDatasets():
         if ds not in request['OutputDatasets']:
             request['OutputDatasets'].append(ds)
             
 # don't want to JSONify the whole workflow
 del metadata['WorkloadSpec']
 workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
 request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
 try:
     CheckIn.checkIn(request, reqSchema['RequestType'])
 except CheckIn.RequestCheckInError, ex:
     raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
     
 # Inconsistent request parameters between Oracle and Couch (#4380, #4388)
 # metadata above is what is saved into couch to represent a request document.
 # Number of request arguments on a corresponding couch document
 # is not set, has default null/None values, update those accordingly now.
 # It's a mess to have two mutually inconsistent database backends.
 # Not easy to handle this earlier since couch is stored first and
 # some parameters are worked out later when storing into Oracle.
 reqDetails = requestDetails(request["RequestName"])
 # couchdb request parameters which are null at the injection time and remain so
 paramsToUpdate = ["RequestStatus",
                   "RequestSizeFiles",
                   "AcquisitionEra",
Пример #4
0
def buildWorkloadAndCheckIn(webApi, reqSchema, couchUrl, couchDB, wmstatUrl, clone=False):
    """
    If clone is True, the function is called on a cloned request in which
    case no modification of the reqSchema shall happen and should be checked in
    as is.
    
    """
    try:
        request = buildWorkloadForRequest(typename = reqSchema["RequestType"], 
                                          schema = reqSchema)
    except WMSpecFactoryException as ex:
        logging.error(traceback.format_exc())
        raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())
    
    helper = WMWorkloadHelper(request['WorkloadSpec'])
    
    # update request as well for wmstats update
    # there is a better way to do this (passing helper to request but make sure all the information is there) 
    request["Campaign"] = helper.getCampaign()

    # Add the output datasets if necessary
    # for some bizarre reason OutpuDatasets is list of lists, when cloning
    # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
    # #3743
    if not clone:
        for ds in helper.listOutputDatasets():
            if ds not in request['OutputDatasets']:
                request['OutputDatasets'].append(ds)
    #TODO: need to update output dataset by Task for task chain requests
    # can't save Request object directly, because it makes it hard to retrieve the _rev
    metadata = {}
    metadata.update(request)            
    # don't want to JSONify the whole workflow
    del metadata['WorkloadSpec']
    workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
    request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
    try:
        CheckIn.checkIn(request, reqSchema['RequestType'])
    except CheckIn.RequestCheckInError as ex:
        raise HTTPError(400, "Error in Request check-in: %s" % str(ex))
        
    # Inconsistent request parameters between Oracle and Couch (#4380, #4388)
    # metadata above is what is saved into couch to represent a request document.
    # Number of request arguments on a corresponding couch document
    # is not set, has default null/None values, update those accordingly now.
    # It's a mess to have two mutually inconsistent database backends.
    # Not easy to handle this earlier since couch is stored first and
    # some parameters are worked out later when storing into Oracle.
    reqDetails = requestDetails(request["RequestName"])
    # couchdb request parameters which are null at the injection time and remain so
    paramsToUpdate = ["RequestStatus",
                      "RequestSizeFiles",
                      "AcquisitionEra",
                      "RequestWorkflow",
                      "RequestType",
                      "RequestStatus",
                      "RequestPriority",
                      "Requestor",
                      "Group",
                      "SizePerEvent",
                      "PrepID",
                      "RequestNumEvents",
                      "ProcessingString",
                      "ProcessingVersion",
                      ]
    
    couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
    fields = {}
    for key in paramsToUpdate:
        fields[key] = reqDetails[key]
    couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields=fields, useBody=True) 
        
    try:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        wmstatSvc.insertRequest(request)
    except Exception as ex:
        webApi.error("Could not update WMStats, reason: %s" % ex)
        raise HTTPError(400, "Creating request failed, could not update WMStats.")

    return request
Пример #5
0
def makeRequest(kwargs, couchUrl, couchDB):
    logging.info(kwargs)
    """ Handles the submission of requests """
    # make sure no extra spaces snuck in
    for k, v in kwargs.iteritems():
        if isinstance(v, str):
            kwargs[k] = v.strip()
    maker = retrieveRequestMaker(kwargs["RequestType"])
    schema = maker.newSchema()
    schema.update(kwargs)
    currentTime = time.strftime('%y%m%d_%H%M%S',
                             time.localtime(time.time()))
    requestString = schema.get('RequestString', "")
    if requestString != "":
        schema['RequestName'] = "%s_%s_%s" % (
        schema['Requestor'], requestString, currentTime)
    else:
        schema['RequestName'] = "%s_%s" % (schema['Requestor'], currentTime)
    schema["Campaign"] = kwargs.get("Campaign", "")
    if 'Scenario' in kwargs and 'ProdConfigCacheID' in kwargs:
        # Use input mode to delete the unused one
        inputMode = kwargs['inputMode']
        inputValues = {'scenario':'Scenario',
                       'couchDB':'ProdConfigCacheID'}
        for n, v in inputValues.iteritems():
            if n != inputMode:
                schema[v] = ""

    if kwargs.has_key("InputDataset"):
        schema["InputDatasets"] = [kwargs["InputDataset"]]
    if kwargs.has_key("FilterEfficiency"):
        kwargs["FilterEfficiency"] = float(kwargs["FilterEfficiency"])
    skimNumber = 1
    # a list of dictionaries
    schema["SkimConfigs"] = []
    while kwargs.has_key("SkimName%s" % skimNumber):
        d = {}
        d["SkimName"] = kwargs["SkimName%s" % skimNumber]
        d["SkimInput"] = kwargs["SkimInput%s" % skimNumber]
        d["Scenario"] = kwargs["Scenario"]

        if kwargs.get("Skim%sConfigCacheID" % skimNumber, None) != None:
            d["ConfigCacheID"] = kwargs["Skim%sConfigCacheID" % skimNumber]

        schema["SkimConfigs"].append(d)
        skimNumber += 1

    if kwargs.has_key("DataPileup") or kwargs.has_key("MCPileup"):
        schema["PileupConfig"] = {}
        if kwargs.has_key("DataPileup") and kwargs["DataPileup"] != "":
            schema["PileupConfig"]["data"] = [kwargs["DataPileup"]]
        if kwargs.has_key("MCPileup") and kwargs["MCPileup"] != "":
            schema["PileupConfig"]["mc"] = [kwargs["MCPileup"]]

    for runlist in ["RunWhitelist", "RunBlacklist"]:
        if runlist in kwargs:
            schema[runlist] = parseRunList(kwargs[runlist])
    for blocklist in ["BlockWhitelist", "BlockBlacklist"]:
        if blocklist in kwargs:
            schema[blocklist] = parseBlockList(kwargs[blocklist])
    validate(schema)
    request = maker(schema)
    helper = WMWorkloadHelper(request['WorkflowSpec'])
    helper.setCampaign(schema["Campaign"])
    helper.setRunWhitelist(schema["RunWhitelist"])
    # can't save Request object directly, because it makes it hard to retrieve the _rev
    metadata = {}
    metadata.update(request)
    # don't want to JSONify the whole workflow
    del metadata['WorkflowSpec']
    workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
    request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
    CheckIn.checkIn(request)
    return request
Пример #6
0
        helper.setCustodialSite(siteName = schema['SiteWhitelist'][0])
    if "RunWhitelist" in schema:
        helper.setRunWhitelist(schema["RunWhitelist"])
    # can't save Request object directly, because it makes it hard to retrieve the _rev
    metadata = {}
    metadata.update(request)
    # Add the output datasets if necessary
    for ds in helper.listOutputDatasets():
        if not ds in request['OutputDatasets']:
            request['OutputDatasets'].append(ds)
    # don't want to JSONify the whole workflow
    del metadata['WorkloadSpec']
    workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
    request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
    try:
        CheckIn.checkIn(request)
    except CheckIn.RequestCheckInError, ex:
        msg = ex._message
        raise HTTPError(400, "Error in Request check-in: %s" % msg)
    return request

def requestDetails(requestName):
    """ Adds details from the Couch document as well as the database """
    WMCore.Lexicon.identifier(requestName)
    request = GetRequest.getRequestDetails(requestName)
    helper = loadWorkload(request)
    schema = helper.data.request.schema.dictionary_()
    # take the stuff from the DB preferentially
    schema.update(request)
    task = helper.getTopLevelTask()[0]
    schema['Site Whitelist']  = task.siteWhitelist()
Пример #7
0
        helper.setCustodialSite(siteName = schema['SiteWhitelist'][0])
    if "RunWhitelist" in schema:
        helper.setRunWhitelist(schema["RunWhitelist"])
    # can't save Request object directly, because it makes it hard to retrieve the _rev
    metadata = {}
    metadata.update(request)
    # Add the output datasets if necessary
    for ds in helper.listOutputDatasets():
        if not ds in request['OutputDatasets']:
            request['OutputDatasets'].append(ds)
    # don't want to JSONify the whole workflow
    del metadata['WorkloadSpec']
    workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
    request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
    try:
        CheckIn.checkIn(request, requestType = kwargs['RequestType'])
    except CheckIn.RequestCheckInError, ex:
        msg = ex._message
        raise HTTPError(400, "Error in Request check-in: %s" % msg)
    return request

def requestDetails(requestName):
    """ Adds details from the Couch document as well as the database """
    WMCore.Lexicon.identifier(requestName)
    request = GetRequest.getRequestDetails(requestName)
    helper = loadWorkload(request)
    schema = helper.data.request.schema.dictionary_()
    # take the stuff from the DB preferentially
    schema.update(request)
    task = helper.getTopLevelTask()[0]
    schema['Site Whitelist']  = task.siteWhitelist()
Пример #8
0
def buildWorkloadAndCheckIn(webApi,
                            reqSchema,
                            couchUrl,
                            couchDB,
                            wmstatUrl,
                            clone=False):
    """
    If clone is True, the function is called on a cloned request in which
    case no modification of the reqSchema shall happen and should be checked in
    as is.
    
    """
    try:
        request = buildWorkloadForRequest(typename=reqSchema["RequestType"],
                                          schema=reqSchema)
    except WMSpecFactoryException as ex:
        logging.error(traceback.format_exc())
        raise HTTPError(400, "Error in Workload Validation: %s" % ex.message())

    helper = WMWorkloadHelper(request['WorkloadSpec'])

    # update request as well for wmstats update
    # there is a better way to do this (passing helper to request but make sure all the information is there)
    request["Campaign"] = helper.getCampaign()

    # Add the output datasets if necessary
    # for some bizarre reason OutpuDatasets is list of lists, when cloning
    # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
    # #3743
    if not clone:
        for ds in helper.listOutputDatasets():
            if ds not in request['OutputDatasets']:
                request['OutputDatasets'].append(ds)
    #TODO: need to update output dataset by Task for task chain requests
    # can't save Request object directly, because it makes it hard to retrieve the _rev
    metadata = {}
    metadata.update(request)
    # don't want to JSONify the whole workflow
    del metadata['WorkloadSpec']
    workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
    request['RequestWorkflow'] = removePasswordFromUrl(workloadUrl)
    try:
        CheckIn.checkIn(request, reqSchema['RequestType'])
    except CheckIn.RequestCheckInError as ex:
        raise HTTPError(400, "Error in Request check-in: %s" % str(ex))

    # Inconsistent request parameters between Oracle and Couch (#4380, #4388)
    # metadata above is what is saved into couch to represent a request document.
    # Number of request arguments on a corresponding couch document
    # is not set, has default null/None values, update those accordingly now.
    # It's a mess to have two mutually inconsistent database backends.
    # Not easy to handle this earlier since couch is stored first and
    # some parameters are worked out later when storing into Oracle.
    reqDetails = requestDetails(request["RequestName"])
    # couchdb request parameters which are null at the injection time and remain so
    paramsToUpdate = [
        "RequestStatus",
        "RequestSizeFiles",
        "AcquisitionEra",
        "RequestWorkflow",
        "RequestType",
        "RequestStatus",
        "RequestPriority",
        "Requestor",
        "Group",
        "SizePerEvent",
        "PrepID",
        "RequestNumEvents",
        "ProcessingString",
        "ProcessingVersion",
    ]

    couchDb = Database(reqDetails["CouchWorkloadDBName"],
                       reqDetails["CouchURL"])
    fields = {}
    for key in paramsToUpdate:
        fields[key] = reqDetails[key]
    couchDb.updateDocument(request["RequestName"],
                           "ReqMgr",
                           "updaterequest",
                           fields=fields,
                           useBody=True)

    try:
        wmstatSvc = WMStatsWriter(wmstatUrl)
        wmstatSvc.insertRequest(request)
    except Exception as ex:
        webApi.error("Could not update WMStats, reason: %s" % ex)
        raise HTTPError(400,
                        "Creating request failed, could not update WMStats.")

    return request
Пример #9
0
    # Add the output datasets if necessary
    # for some bizarre reason OutpuDatasets is list of lists, when cloning
    # [['/MinimumBias/WMAgentCommissioning10-v2/RECO'], ['/MinimumBias/WMAgentCommissioning10-v2/ALCARECO']]
    # #3743
    if not clone:
        for ds in helper.listOutputDatasets():
            if ds not in request["OutputDatasets"]:
                request["OutputDatasets"].append(ds)

    # don't want to JSONify the whole workflow
    del metadata["WorkloadSpec"]
    workloadUrl = helper.saveCouch(couchUrl, couchDB, metadata=metadata)
    request["RequestWorkflow"] = removePasswordFromUrl(workloadUrl)
    try:
        CheckIn.checkIn(request, reqSchema["RequestType"])
    except CheckIn.RequestCheckInError, ex:
        msg = ex._message
        raise HTTPError(400, "Error in Request check-in: %s" % msg)

    # Inconsistent request parameters between Oracle and Couch (#4380, #4388)
    # metadata above is what is saved into couch to represent a request document.
    # Number of request arguments on a corresponding couch document
    # is not set, has default null/None values, update those accordingly now.
    # It's a mess to have two mutually inconsistent database backends.
    # Not easy to handle this earlier since couch is stored first and
    # some parameters are worked out later when storing into Oracle.
    reqDetails = requestDetails(request["RequestName"])
    # couchdb request parameters which are null at the injection time and remain so
    paramsToUpdate = [
        "RequestStatus",