Beispiel #1
0
    def generateFailoverFile(self):
        """ Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper
        """
        reportRequest = None
        result = self.jobReport.generateForwardDISET()
        if not result['OK']:
            self.log.warn(
                "Could not generate Operation for job report with result:\n%s"
                % (result))
        else:
            reportRequest = result['Value']
        if reportRequest:
            self.log.info("Populating request with job report information")
            self.request.addOperation(reportRequest)

        accountingReport = None
        if self.workflow_commons.has_key('AccountingReport'):
            accountingReport = self.workflow_commons['AccountingReport']
        if accountingReport:
            result = accountingReport.commit()
            if not result['OK']:
                self.log.error(
                    "!!! Both accounting and RequestDB are down? !!!")
                return result

        if len(self.request):
            isValid = gRequestValidator.validate(self.request)
            if not isValid['OK']:
                raise RuntimeError("Failover request is not valid: %s" %
                                   isValid['Message'])
            else:
                requestJSON = self.request.toJSON()
                if requestJSON['OK']:
                    self.log.info(
                        "Creating failover request for deferred operations for job %d"
                        % self.jobID)
                    request_string = str(requestJSON['Value'])
                    self.log.debug(request_string)
                    # Write out the request string
                    fname = '%s_%s_request.json' % (self.production_id,
                                                    self.prod_job_id)
                    jsonFile = open(fname, 'w')
                    jsonFile.write(request_string)
                    jsonFile.close()
                    self.log.info(
                        "Created file containing failover request %s" % fname)
                    result = self.request.getDigest()
                    if result['OK']:
                        self.log.info("Digest of the request: %s" %
                                      result['Value'])
                    else:
                        self.log.error(
                            "No digest? That's not sooo important, anyway: %s"
                            % result['Message'])
                else:
                    raise RuntimeError(requestJSON['Message'])
Beispiel #2
0
  def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
    """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
    """
    if ( not owner ) or ( not ownerGroup ):
      res = getProxyInfo( False, False )
      if not res['OK']:
        return res
      proxyInfo = res['Value']
      owner = proxyInfo['username']
      ownerGroup = proxyInfo['group']

    if not ownerDN:
      res = getDNForUsername( owner )
      if not res['OK']:
        return res
      ownerDN = res['Value'][0]

    requestOperation = 'ReplicateAndRegister'
    if transBody:
      try:
        _requestType, requestOperation = transBody.split( ';' )
      except AttributeError:
        pass

    for taskID in sorted( taskDict ):
      paramDict = taskDict[taskID]
      if paramDict['InputData']:
        transID = paramDict['TransformationID']

        oRequest = Request()
        transfer = Operation()
        transfer.Type = requestOperation
        transfer.TargetSE = paramDict['TargetSE']

        if type( paramDict['InputData'] ) == type( [] ):
          files = paramDict['InputData']
        elif type( paramDict['InputData'] ) == type( '' ):
          files = paramDict['InputData'].split( ';' )
        for lfn in files:
          trFile = File()
          trFile.LFN = lfn

          transfer.addFile( trFile )

        oRequest.addOperation( transfer )
        oRequest.RequestName = _requestName( transID, taskID )
        oRequest.OwnerDN = ownerDN
        oRequest.OwnerGroup = ownerGroup

      isValid = gRequestValidator.validate( oRequest )
      if not isValid['OK']:
        return isValid

      taskDict[taskID]['TaskObject'] = oRequest

    return S_OK( taskDict )
Beispiel #3
0
  def commitRequest( self ):
    """ Send request to the Request Management Service
    """
    if self.request.isEmpty():
      return S_OK()

    isValid = gRequestValidator.validate( self.request )
    if not isValid["OK"]:
      return S_ERROR( "Failover request is not valid: %s" % isValid["Message"] )
    else:
      requestClient = ReqClient()
      result = requestClient.putRequest( self.request )
      return result
Beispiel #4
0
    def commitRequest(self):
        """ Send request to the Request Management Service
    """
        if self.request.isEmpty():
            return S_OK()

        isValid = gRequestValidator.validate(self.request)
        if not isValid["OK"]:
            return S_ERROR("Failover request is not valid: %s" %
                           isValid["Message"])
        else:
            requestClient = ReqClient()
            result = requestClient.putRequest(self.request)
            return result
Beispiel #5
0
  def generateFailoverFile( self ):
    """ Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper
    """
    reportRequest = None
    result = self.jobReport.generateForwardDISET()
    if not result['OK']:
      self.log.warn( "Could not generate Operation for job report with result:\n%s" % ( result ) )
    else:
      reportRequest = result['Value']
    if reportRequest:
      self.log.info( "Populating request with job report information" )
      self.request.addOperation( reportRequest )

    accountingReport = None
    if self.workflow_commons.has_key( 'AccountingReport' ):
      accountingReport = self.workflow_commons['AccountingReport']
    if accountingReport:
      result = accountingReport.commit()
      if not result['OK']:
        self.log.error( "!!! Both accounting and RequestDB are down? !!!" )
        return result

    if len( self.request ):
      isValid = gRequestValidator.validate( self.request )
      if not isValid['OK']:
        raise RuntimeError, "Failover request is not valid: %s" % isValid['Message']
      else:
        requestJSON = self.request.toJSON()
        if requestJSON['OK']:
          self.log.info( "Creating failover request for deferred operations for job %d" % self.jobID )
          request_string = str( requestJSON['Value'] )
          self.log.debug( request_string )
          # Write out the request string
          fname = '%d_%d_request.json' % ( self.production_id, self.prod_job_id )
          jsonFile = open( fname, 'w' )
          jsonFile.write( request_string )
          jsonFile.close()
          self.log.info( "Created file containing failover request %s" % fname )
          result = self.request.getDigest()
          if result['OK']:
            self.log.info( "Digest of the request: %s" % result['Value'] )
          else:
            self.log.error( "No digest? That's not sooo important, anyway: %s" % result['Message'] )
        else:
          raise RuntimeError, requestJSON['Message']
Beispiel #6
0
  def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
    """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
    """
    requestOperation = 'ReplicateAndRegister'
    if transBody:
      try:
        _requestType, requestOperation = transBody.split( ';' )
      except AttributeError:
        pass

    for taskID in sorted( taskDict ):
      paramDict = taskDict[taskID]
      if paramDict['InputData']:
        transID = paramDict['TransformationID']

        oRequest = Request()
        transfer = Operation()
        transfer.Type = requestOperation
        transfer.TargetSE = paramDict['TargetSE']

        if type( paramDict['InputData'] ) == type( [] ):
          files = paramDict['InputData']
        elif type( paramDict['InputData'] ) == type( '' ):
          files = paramDict['InputData'].split( ';' )
        for lfn in files:
          trFile = File()
          trFile.LFN = lfn

          transfer.addFile( trFile )

        oRequest.addOperation( transfer )
        oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
        oRequest.OwnerDN = owner
        oRequest.OwnerGroup = ownerGroup

      isValid = gRequestValidator.validate( oRequest )
      if not isValid['OK']:
        return isValid

      taskDict[taskID]['TaskObject'] = oRequest

    return S_OK( taskDict )
    print "Can't get file metadata: %s" % res['Message']
    DIRAC.exit( 1 )
  if res['Value']['Failed']:
    print "Could not get the file metadata of the following, so skipping them:"
    for fFile in res['Value']['Failed']:
      print fFile

  lfnMetadata = res['Value']['Successful']

  for lfn in lfnMetadata:
    rarFile = File()
    rarFile.LFN = lfn
    rarFile.Size = lfnMetadata[lfn]['Size']
    rarFile.Checksum = lfnMetadata[lfn]['Checksum']
    rarFile.GUID = lfnMetadata[lfn]['GUID']
    rarFile.ChecksumType = 'ADLER32'
    oOperation.addFile( rarFile )

  oRequest.addOperation( oOperation )

  isValid = gRequestValidator.validate( oRequest )
  if not isValid['OK']:
    print "Request is not valid: ", isValid['Message']
    DIRAC.exit( 1 )

  result = reqClient.putRequest( oRequest )
  if result['OK']:
    print 'Request %d Submitted' % result['Value']
  else:
    print 'Failed to submit Request: ', result['Message']
    if not res['OK']:
        print "Can't get file metadata: %s" % res['Message']
        DIRAC.exit(1)
    if res['Value']['Failed']:
        print "Could not get the file metadata of the following, so skipping them:"
        for fFile in res['Value']['Failed']:
            print fFile

    lfnMetadata = res['Value']['Successful']

    for lfn in lfnMetadata:
        rarFile = File()
        rarFile.LFN = lfn
        rarFile.Size = lfnMetadata[lfn]['Size']
        rarFile.Checksum = lfnMetadata[lfn]['Checksum']
        rarFile.GUID = lfnMetadata[lfn]['GUID']
        rarFile.ChecksumType = 'ADLER32'
        replicateAndRegister.addFile(rarFile)

    oRequest.addOperation(replicateAndRegister)
    isValid = gRequestValidator.validate(oRequest)
    if not isValid['OK']:
        print "Request is not valid: ", isValid['Message']
        DIRAC.exit(1)

    result = reqClient.putRequest(oRequest)
    if result['OK']:
        print "Request %d submitted successfully" % result['Value']
    else:
        print "Failed to submit Request: ", result['Message']