Exemple #1
0
  def do_getall( self, args ):
    """Get transformation details

       usage: getall [Status] [Status]
    """
    oTrans = Transformation()
    oTrans.getTransformations( transStatus = args.split(), printOutput = True )
Exemple #2
0
    def do_getall(self, args):
        """Get transformation details

       usage: getall [Status] [Status]
    """
        oTrans = Transformation()
        oTrans.getTransformations(transStatus=args.split(), printOutput=True)
Exemple #3
0
  def do_getall( self, args ):
    '''Get transformation details

       usage: getall [Status] [Status]
    '''
    oTrans = Transformation()
    oTrans.setServer( self.serverURL )
    oTrans.getTransformations( transStatus = args.split(), printOutput = True )
Exemple #4
0
  def do_getall( self, args ):
    """Get transformation details

       usage: getall [Status] [Status]
    """
    oTrans = Transformation()
    oTrans.setServer( self.serverURL )
    oTrans.getTransformations( transStatus = string.split( args ), printOutput = True )
Exemple #5
0
    def test_AddFilesGetFilesSetFileStatus(self):
        """ Testing adding, getting and setting file status.

          addFilesToTransformation()
          getTransformationFiles()
          getTransformationStats()
          setFileStatusForTransformation()
          addTaskForTransformation()

        Test adding and files to transformation.
        Test selecting the files for the transformation.
        Test getting the status count of the transformation files.
        Test setting the file status for transformation.
        Test creating a task for the added files and ensure the status is updated correctly.
    """
        oTrans = Transformation(self.transID)
        lfns = ['/test/lfn/file1', '/test/lfn/file2']
        res = oTrans.addFilesToTransformation(lfns)
        self.assert_(res['OK'])
        res = oTrans.getTransformationFiles()
        self.assert_(res['OK'])
        self.assertEqual(sortList(lfns), res['LFNs'])
        self.assertEqual(len(lfns), len(res['Records']))
        self.assertEqual(len(lfns), len(res['Value']))
        fileParams = sortList([
            'LFN', 'TransformationID', 'FileID', 'Status', 'TaskID',
            'TargetSE', 'UsedSE', 'ErrorCount', 'LastUpdate', 'InsertedTime'
        ])
        self.assertEqual(fileParams, sortList(res['ParameterNames']))
        self.assertEqual(res['Records'][0][0], lfns[0])
        self.assertEqual(res['Value'][0]['LFN'], lfns[0])
        self.assertEqual(res['Records'][0][1], self.transID)
        self.assertEqual(res['Value'][0]['TransformationID'], self.transID)
        self.assertEqual(res['Records'][0][3], 'Unused')
        self.assertEqual(res['Value'][0]['Status'], 'Unused')
        res = oTrans.getTransformationStats()
        self.assert_(res['OK'])
        self.assertEqual(res['Value']['Total'], 2)
        self.assertEqual(res['Value']['Unused'], 2)
        res = oTrans.setFileStatusForTransformation('Processed', [lfns[0]])
        self.assert_(res['OK'])
        res = oTrans.getTransformationStats()
        self.assert_(res['OK'])
        self.assertEqual(res['Value']['Total'], 2)
        self.assertEqual(res['Value']['Unused'], 1)
        self.assertEqual(res['Value']['Processed'], 1)
        res = oTrans.setFileStatusForTransformation('Unused', [lfns[0]])
        self.assert_(res['OK'])
        self.assert_(res['Value']['Failed'].has_key(lfns[0]))
        self.assertEqual(res['Value']['Failed'][lfns[0]],
                         'Can not change Processed status')
        res = oTrans.addTaskForTransformation(lfns=[lfns[1]], se='Test')
        self.assert_(res['OK'])
        res = oTrans.getTransformationStats()
        self.assert_(res['OK'])
        self.assertEqual(res['Value']['Total'], 2)
        self.assertEqual(res['Value']['Assigned'], 1)
        self.assertEqual(res['Value']['Processed'], 1)
Exemple #6
0
    def do_getAllByUser(self, args):
        """Get all transformations created by a given user

The first argument is the authorDN or username. The authorDN
is preferred: it need to be inside quotes because contains
white spaces. Only authorDN should be quoted.

When the username is provided instead, 
the authorDN is retrieved from the uploaded proxy,
so that the retrieved transformations are those created by
the user who uploaded that proxy: that user could be different
that the username provided to the function.

       usage: getAllByUser authorDN or username [Status] [Status]
    """
        oTrans = Transformation()
        argss = args.split()
        username = ""
        author = ""
        status = []
        if not len(argss) > 0:
            print self.do_getAllByUser.__doc__
            return

        # if the user didnt quoted the authorDN ends
        if '=' in argss[0] and argss[0][0] not in ["'", '"']:
            print "AuthorDN need to be quoted (just quote that argument)"
            return

        if argss[0][0] in ["'", '"']:  # authorDN given
            author = argss[0]
            status_idx = 1
            for arg in argss[1:]:
                author += ' ' + arg
                status_idx += 1
                if arg[-1] in ["'", '"']:
                    break
            # At this point we should have something like 'author'
            if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']:
                print "AuthorDN need to be quoted (just quote that argument)"
                return
            else:
                author = author[1:-1]  # throw away the quotes
            # the rest are the requested status
            status = argss[status_idx:]
        else:  # username given
            username = argss[0]
            status = argss[1:]

        oTrans.getTransformationsByUser(authorDN=author,
                                        userName=username,
                                        transStatus=status,
                                        printOutput=True)
Exemple #7
0
    def test_DeleteTransformation(self):
        """ Testing of the deletion of a transformation. 

          addTransformation()
          deleteTransformation()
           
        Tests that a transformation can be removed.
        Tests that retrieving a non existant transformation raises an AttributeError.
    """
        oTrans = Transformation(self.transID)
        res = oTrans.deleteTransformation()
        self.assert_(res['OK'])
        self.assertRaises(AttributeError, Transformation, self.transID)
        self.transID = 0
Exemple #8
0
  def do_getAllByUser( self, args ):
    """Get all transformations created by a given user

The first argument is the authorDN or username. The authorDN
is preferred: it need to be inside quotes because contains
white spaces. Only authorDN should be quoted.

When the username is provided instead, 
the authorDN is retrieved from the uploaded proxy,
so that the retrieved transformations are those created by
the user who uploaded that proxy: that user could be different
that the username provided to the function.

       usage: getAllByUser authorDN or username [Status] [Status]
    """
    oTrans = Transformation()
    argss = args.split()
    username = ""
    author = ""
    status = []
    if not len( argss ) > 0:
      print self.do_getAllByUser.__doc__
      return

    # if the user didnt quoted the authorDN ends
    if '=' in argss[0] and argss[0][0] not in ["'", '"']:
      print "AuthorDN need to be quoted (just quote that argument)"
      return

    if argss[0][0] in ["'", '"']: # authorDN given
      author = argss[0]
      status_idx = 1
      for arg in argss[1:]:
        author += ' ' + arg
        status_idx +=1
        if arg[-1] in ["'", '"']:
          break
      # At this point we should have something like 'author'
      if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']:
        print "AuthorDN need to be quoted (just quote that argument)"
        return
      else:
        author = author[1:-1] # throw away the quotes
      # the rest are the requested status
      status = argss[ status_idx: ]
    else: # username given
      username = argss[0]
      status = argss[ 1: ]

    oTrans.getTransformationsByUser( authorDN = author, userName = username, transStatus = status, printOutput = True )
  def test_DeleteTransformation(self):
    """ Testing of the deletion of a transformation. 

          addTransformation()
          deleteTransformation()
           
        Tests that a transformation can be removed.
        Tests that retrieving a non existant transformation raises an AttributeError.
    """
    oTrans = Transformation(self.transID)
    res = oTrans.deleteTransformation()
    self.assert_(res['OK'])
    self.assertRaises(AttributeError,Transformation,self.transID)
    self.transID = 0
  def test_AddFilesGetFilesSetFileStatus(self):
    """ Testing adding, getting and setting file status.

          addFilesToTransformation()
          getTransformationFiles()
          getTransformationStats()
          setFileStatusForTransformation()
          addTaskForTransformation()

        Test adding and files to transformation.
        Test selecting the files for the transformation.
        Test getting the status count of the transformation files.
        Test setting the file status for transformation.
        Test creating a task for the added files and ensure the status is updated correctly.
    """
    oTrans = Transformation(self.transID)
    lfns = ['/test/lfn/file1','/test/lfn/file2']
    res = oTrans.addFilesToTransformation(lfns)
    self.assert_(res['OK'])
    res = oTrans.getTransformationFiles()
    self.assert_(res['OK'])
    self.assertEqual(sortList(lfns),res['LFNs'])
    self.assertEqual(len(lfns),len(res['Records']))
    self.assertEqual(len(lfns),len(res['Value']))
    fileParams = sortList(['LFN', 'TransformationID', 'FileID', 'Status', 'TaskID', 'TargetSE', 'UsedSE', 'ErrorCount', 'LastUpdate', 'InsertedTime'])
    self.assertEqual(fileParams,sortList(res['ParameterNames']))
    self.assertEqual(res['Records'][0][0], lfns[0])
    self.assertEqual(res['Value'][0]['LFN'],lfns[0]) 
    self.assertEqual(res['Records'][0][1], self.transID)
    self.assertEqual(res['Value'][0]['TransformationID'], self.transID)
    self.assertEqual(res['Records'][0][3],'Unused')
    self.assertEqual(res['Value'][0]['Status'],'Unused')
    res = oTrans.getTransformationStats()
    self.assert_(res['OK'])
    self.assertEqual(res['Value']['Total'],2)
    self.assertEqual(res['Value']['Unused'],2)
    res = oTrans.setFileStatusForTransformation('Processed',[lfns[0]])
    self.assert_(res['OK'])
    res = oTrans.getTransformationStats()
    self.assert_(res['OK'])
    self.assertEqual(res['Value']['Total'],2)
    self.assertEqual(res['Value']['Unused'],1)
    self.assertEqual(res['Value']['Processed'],1)
    res = oTrans.setFileStatusForTransformation('Unused',[lfns[0]])
    self.assert_(res['OK'])
    self.assert_(res['Value']['Failed'].has_key(lfns[0]))
    self.assertEqual(res['Value']['Failed'][lfns[0]],'Can not change Processed status')
    res = oTrans.addTaskForTransformation(lfns=[lfns[1]],se='Test')
    self.assert_(res['OK'])
    res = oTrans.getTransformationStats()
    self.assert_(res['OK'])
    self.assertEqual(res['Value']['Total'],2)
    self.assertEqual(res['Value']['Assigned'],1)
    self.assertEqual(res['Value']['Processed'],1)
Exemple #11
0
  def do_summaryTransformations( self, args ):
    """Show the summary for a list of Transformations

    Fields starting with 'F' ('J')  refers to files (jobs).
    Proc. stand for processed.

        Usage: summaryTransformations <ProdID> [<ProdID> ...]
    """
    argss = args.split()
    if not len( argss ) > 0:
      print self.do_summaryTransformations.__doc__
      return

    transid = argss
    oTrans = Transformation()
    oTrans.getSummaryTransformations( transID = transid )
Exemple #12
0
  def setUp( self ):

    self.mockTransClient = Mock()
    self.mockTransClient.setTaskStatusAndWmsID.return_value = {'OK':True}

    self.WMSClientMock = Mock()
    self.jobMonitoringClient = Mock()
    self.mockReqClient = Mock()

    self.jobMock = Mock()
    self.jobMock2 = Mock()
    mockWF = Mock()
    mockPar = Mock()
    mockWF.findParameter.return_value = mockPar
    mockPar.getValue.return_value = 'MySite'

    self.jobMock2.workflow = mockWF
    self.jobMock2.setDestination.return_value = {'OK':True}
    self.jobMock.workflow.return_value = ''
    self.jobMock.return_value = self.jobMock2

    self.taskBase = TaskBase( transClient = self.mockTransClient )
    self.wfTasks = WorkflowTasks( transClient = self.mockTransClient,
                                  submissionClient = self.WMSClientMock,
                                  jobMonitoringClient = self.jobMonitoringClient,
                                  outputDataModule = "mock",
                                  jobClass = self.jobMock )
    self.requestTasks = RequestTasks( transClient = self.mockTransClient,
                                      requestClient = self.mockReqClient
                                      )
    self.tc = TransformationClient()
    self.transformation = Transformation()

    self.maxDiff = None
Exemple #13
0
    def __init__(self):
        """Create a string containing restructured text documentation for all the special tranformation parameters

    :param str module: full path to the module
    :param bool replace: wether to replace the full docstring or not
       if you replace it completely add the automodule commands etc.!
    """
        self.module = MODULE
        self.replace = False
        self.doc_string = ''
        # Fill the docstring addition with what we want to add
        self.doc_string += textwrap.dedent("""
                                       Transformation Parameters
                                       -------------------------

                                       Any parameter with ``ParameterName`` can be set for a transformation with a call
                                       to ``setParameterName(parameterValue)``.

                                       The following parameters have a special meaning
                                       """)
        from DIRAC.TransformationSystem.Client.Transformation import Transformation
        trans = Transformation()
        for paramName in sorted(trans.paramTypes):
            self.doc_string += '\n``%s``:\n    Default value: %r' % (
                paramName, trans.paramValues[paramName])
Exemple #14
0
    def do_summaryTransformations(self, args):
        """Show the summary for a list of Transformations

    Fields starting with 'F' ('J')  refers to files (jobs).
    Proc. stand for processed.

        Usage: summaryTransformations <ProdID> [<ProdID> ...]
    """
        argss = args.split()
        if not len(argss) > 0:
            print self.do_summaryTransformations.__doc__
            return

        transid = argss
        oTrans = Transformation()
        oTrans.getSummaryTransformations(transID=transid)
Exemple #15
0
    def test_SetGetReset(self):
        """ Testing of the set, get and reset methods.

          set*()
          get*()
          setTargetSE()
          setSourceSE()
          getTargetSE()
          getSourceSE()
          reset()
        Ensures that after a reset all parameters are returned to their defaults
    """
        oTrans = Transformation()
        res = oTrans.getParameters()
        self.assert_(res['OK'])
        defaultParams = res['Value'].copy()
        for parameterName, defaultValue in res['Value'].items():
            if type(defaultValue) in types.StringTypes:
                testValue = "'TestValue'"
            else:
                testValue = '99999'
            execString = "res = oTrans.set%s(%s)" % (parameterName, testValue)
            exec(execString)
            self.assert_(res['OK'])
            execString = "res = oTrans.get%s()" % (parameterName)
            exec(execString)
            self.assert_(res['OK'])
            self.assertEqual(res['Value'], eval(testValue))

        # Test that SEs delivered as a space or comma seperated string are resolved...
        stringSEs = 'CERN-USER, CNAF-USER GRIDKA-USER,IN2P3-USER'
        listSEs = stringSEs.replace(',', ' ').split()
        res = oTrans.setTargetSE(stringSEs)
        self.assert_(res['OK'])
        res = oTrans.getTargetSE()
        self.assert_(res['OK'])
        self.assertEqual(res['Value'], listSEs)
        # ...but that lists are correctly handled also
        res = oTrans.setSourceSE(listSEs)
        self.assert_(res['OK'])
        res = oTrans.getSourceSE()
        self.assert_(res['OK'])
        self.assertEqual(res['Value'], listSEs)

        res = oTrans.reset()
        self.assert_(res['OK'])
        res = oTrans.getParameters()
        self.assert_(res['OK'])
        for parameterName, resetValue in res['Value'].items():
            self.assertEqual(resetValue, defaultParams[parameterName])
        self.assertRaises(AttributeError, oTrans.getTargetSE)
        self.assertRaises(AttributeError, oTrans.getSourceSE)
  def test_SetGetReset(self):
    """ Testing of the set, get and reset methods.

          set*()
          get*()
          setTargetSE()
          setSourceSE()
          getTargetSE()
          getSourceSE()
          reset()
        Ensures that after a reset all parameters are returned to their defaults
    """
    oTrans = Transformation()
    res = oTrans.getParameters()
    self.assert_(res['OK'])
    defaultParams = res['Value'].copy()
    for parameterName, defaultValue in res['Value'].items():
      if type(defaultValue) in types.StringTypes:
        testValue = "'TestValue'"
      else:
        testValue = '99999'
      execString = "res = oTrans.set%s(%s)" % (parameterName,testValue)
      exec(execString)
      self.assert_(res['OK'])
      execString = "res = oTrans.get%s()" % (parameterName)
      exec(execString)
      self.assert_(res['OK']) 
      self.assertEqual(res['Value'],eval(testValue))

    # Test that SEs delivered as a space or comma seperated string are resolved...
    stringSEs = 'CERN-USER, CNAF-USER GRIDKA-USER,IN2P3-USER'
    listSEs = stringSEs.replace(',',' ').split()
    res = oTrans.setTargetSE(stringSEs)
    self.assert_(res['OK']) 
    res = oTrans.getTargetSE()
    self.assert_(res['OK'])
    self.assertEqual(res['Value'],listSEs)
    # ...but that lists are correctly handled also
    res = oTrans.setSourceSE(listSEs)
    self.assert_(res['OK']) 
    res = oTrans.getSourceSE()
    self.assert_(res['OK'])
    self.assertEqual(res['Value'],listSEs)

    res = oTrans.reset()
    self.assert_(res['OK'])
    res = oTrans.getParameters()
    self.assert_(res['OK'])
    for parameterName, resetValue in res['Value'].items():
      self.assertEqual(resetValue,defaultParams[parameterName])
    self.assertRaises(AttributeError,oTrans.getTargetSE)
    self.assertRaises(AttributeError,oTrans.getSourceSE)
Exemple #17
0
    def test_getTransformations(self):
        """ Testing the selection of transformations from the database

          getTransformations
         
        This will select all the transformations associated to this test suite and remove them.
    """
        oTrans = Transformation()
        res = oTrans.getTransformations()
        self.assert_(res['OK'])
        parameters = [
            'TransformationID', 'TransformationName', 'Description',
            'LongDescription', 'CreationDate', 'LastUpdate', 'AuthorDN',
            'AuthorGroup', 'Type', 'Plugin', 'AgentType', 'Status', 'FileMask',
            'TransformationGroup', 'GroupSize', 'InheritedFrom', 'Body',
            'MaxNumberOfTasks', 'EventsPerTask'
        ]
        self.assertEqual(sortList(res['ParameterNames']), sortList(parameters))
        self.assertEqual(sortList(res['Value'][0].keys()),
                         sortList(parameters))
        self.assertEqual(len(res['Value']), len(res['Records']))
        ignore = self.transID
        for transDict in res['Value']:
            name = transDict['TransformationName']
            if re.search('TestClientTransformation', name):
                transID = transDict['TransformationID']
                if transID != ignore:
                    oTrans = Transformation(transID)
                    res = oTrans.deleteTransformation()
                    self.assert_(res['OK'])
        self.transID = ignore
Exemple #18
0
  def do_getall( self, args ):
    """Get transformation details

       usage: getall [Status] [Status]
    """
    oTrans = Transformation()
    oTrans.setServer( self.serverURL )
    oTrans.getTransformations( transStatus = string.split( args ), printOutput = True )
Exemple #19
0
    def do_getall(self, args):
        '''Get transformation details

       usage: getall [Status] [Status]
    '''
        oTrans = Transformation()
        oTrans.setServer(self.serverURL)
        oTrans.getTransformations(transStatus=args.split(), printOutput=True)
  def test_getTransformations(self):  
    """ Testing the selection of transformations from the database

          getTransformations
         
        This will select all the transformations associated to this test suite and remove them.
    """
    oTrans = Transformation()
    res = oTrans.getTransformations()
    self.assert_(res['OK'])
    parameters = ['TransformationID', 'TransformationName', 'Description', 'LongDescription', 'CreationDate', 'LastUpdate', 'AuthorDN', 'AuthorGroup', 'Type', 'Plugin', 'AgentType', 'Status', 'FileMask', 'TransformationGroup', 'GroupSize', 'InheritedFrom', 'Body', 'MaxNumberOfTasks', 'EventsPerTask']
    self.assertEqual(sortList(res['ParameterNames']),sortList(parameters))
    self.assertEqual(sortList(res['Value'][0].keys()),sortList(parameters))
    self.assertEqual(len(res['Value']),len(res['Records']))
    ignore = self.transID
    for transDict in res['Value']:
      name = transDict['TransformationName']
      if re.search('TestClientTransformation',name):
        transID = transDict['TransformationID']
        if transID != ignore:
          oTrans = Transformation(transID)
          res = oTrans.deleteTransformation()
          self.assert_(res['OK'])
    self.transID = ignore
Exemple #21
0
 def _createTransformation(self, transName, plugin='Standard'):
     oTrans = Transformation()
     res = oTrans.setTransformationName(transName)
     self.assert_(res['OK'])
     description = 'Test transforamtion description'
     res = oTrans.setDescription(description)
     longDescription = 'Test transformation long description'
     res = oTrans.setLongDescription(longDescription)
     self.assert_(res['OK'])
     res = oTrans.setType('MCSimulation')
     self.assert_(res['OK'])
     res = oTrans.setPlugin(plugin)
     self.assert_(res['OK'])
     res = oTrans.addTransformation()
     self.assert_(res['OK'])
     self.transID = res['Value']
     return res
Exemple #22
0
  def setUp(self):
    self.mockTransClient = mock.MagicMock()
    self.mockTransClient.setTaskStatusAndWmsID.return_value = {'OK': True}

    self.mockReqClient = mock.MagicMock()

    self.taskBase = TaskBase(transClient=self.mockTransClient)
    self.pu = PluginUtilities(transClient=self.mockTransClient)

    self.requestTasks = RequestTasks(transClient=self.mockTransClient,
                                     requestClient=self.mockReqClient,
                                     requestValidator=reqValFake)
    self.transformation = Transformation()

    self.maxDiff = None
Exemple #23
0
 def test_getTransformationLogging(self):
     """ Testing the obtaining of transformation logging information
  
       getTransformationLogging()
 """
     oTrans = Transformation(self.transID)
     res = oTrans.setStatus('Active')
     self.assert_(res['OK'])
     res = oTrans.extendTransformation(100)
     self.assert_(res['OK'])
     res = oTrans.setStatus('Completed')
     self.assert_(res['OK'])
     res = oTrans.cleanTransformation()
     self.assert_(res['OK'])
     res = oTrans.getTransformationLogging()
     self.assert_(res['OK'])
     self.assertEqual(len(res['Value']), 6)
 def _createTransformation(self,transName,plugin='Standard'):
   oTrans = Transformation()
   res = oTrans.setTransformationName(transName)
   self.assert_(res['OK'])
   description = 'Test transforamtion description'
   res = oTrans.setDescription(description)
   longDescription = 'Test transformation long description'
   res = oTrans.setLongDescription(longDescription)
   self.assert_(res['OK'])
   res = oTrans.setType('MCSimulation')
   self.assert_(res['OK'])
   res = oTrans.setPlugin(plugin)
   self.assert_(res['OK'])
   res = oTrans.addTransformation()
   self.assert_(res['OK'])
   self.transID = res['Value']
   return res
    def setUp(self):

        self.mockTransClient = MagicMock()
        self.mockTransClient.setTaskStatusAndWmsID.return_value = {'OK': True}

        self.WMSClientMock = MagicMock()
        self.jobMonitoringClient = MagicMock()
        self.mockReqClient = MagicMock()

        self.jobMock = MagicMock()
        self.jobMock2 = MagicMock()
        mockWF = MagicMock()
        mockPar = MagicMock()
        mockWF.findParameter.return_value = mockPar
        mockPar.getValue.return_value = 'MySite'

        self.jobMock2.workflow = mockWF
        self.jobMock2.setDestination.return_value = {'OK': True}
        self.jobMock.workflow.return_value = ''
        self.jobMock.return_value = self.jobMock2

        self.reqValidatorMock = MagicMock()
        self.reqValidatorMock.validate.return_value = {'OK': True}

        self.taskBase = TaskBase(transClient=self.mockTransClient)
        self.pu = PluginUtilities(transClient=self.mockTransClient)
        self.wfTasks = WorkflowTasks(
            transClient=self.mockTransClient,
            submissionClient=self.WMSClientMock,
            jobMonitoringClient=self.jobMonitoringClient,
            outputDataModule="mock")

        self.requestTasks = RequestTasks(transClient=self.mockTransClient,
                                         requestClient=self.mockReqClient,
                                         requestValidator=reqValFake)
        self.tc = TransformationClient()
        self.transformation = Transformation()

        self.maxDiff = None

        gLogger.setLevel('DEBUG')
 def test_getTransformationLogging(self):
   """ Testing the obtaining of transformation logging information
    
         getTransformationLogging()
   """
   oTrans = Transformation(self.transID)
   res = oTrans.setStatus('Active')
   self.assert_(res['OK'])
   res = oTrans.extendTransformation(100)
   self.assert_(res['OK'])
   res = oTrans.setStatus('Completed')
   self.assert_(res['OK'])
   res = oTrans.cleanTransformation()
   self.assert_(res['OK'])
   res = oTrans.getTransformationLogging()
   self.assert_(res['OK'])
   self.assertEqual(len(res['Value']),6)
Exemple #27
0
def OldSubmitTS(job, infileList):
    """ Create a transformation executing the job workflow  """
    t = Transformation()
    tc = TransformationClient()
    t.setType("DataReprocessing")
    t.setDescription("Runs EvnDisp analysis for array HB9 SCT")
    t.setLongDescription(
        "merge_simtel, evndisp converter and evndisp analysis for HB9 SCT"
    )  # mandatory
    t.setGroupSize(5)
    t.setBody(job.workflow.toXML())

    res = t.addTransformation()  # Transformation is created here

    if not res['OK']:
        print res['Message']
        DIRAC.exit(-1)

    t.setStatus("Active")
    t.setAgentType("Automatic")
    transID = t.getTransformationID()
    print('Adding %s files to transformation' % len(infileList))
    tc.addFilesToTransformation(transID['Value'],
                                infileList)  # Files added here

    return res
Exemple #28
0
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.Interfaces.API.Job import Job

j = Job()
tc = TransformationClient()

t = Transformation()
t.setTransformationName("Un exemple") #<- unique
t.setTransformationGroup("Un groupe") #<- for monitoring
t.setType("MCSimulation")#type must be among known types
t.setDescription("Ceci est un exemple")
t.setLongDescription("C'est un bel exemple")
t.setBody(j.workflow._toXML())
t.setGroupSize(1)
t.setPlugin("Standard")
t.addTransformation() #<-- transformation is created here
t.setStatus("Active") #<-- make it start
t.setAgentType("Automatic") #<-- should be by default
transfid = t.getTransformationID()['Value'] #<- unique
tc.createTransformationInputDataQuery(transfid, 
                                      {'meta1':val1,"meta2":{">":34}})
  if not res['OK']:
    print res['Message']
  res = psl.finalizeProd()
  if not res['OK']:
    print res['Message']
    exit(1)
  # get the metadata for this production to define input for the next step
  meta = psl.getMetadata()


from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient

# Define transformation steps for the replication of the output data
if replicateFiles and meta:
  Trans = Transformation()
  Trans.setTransformationName( 'replicate_%s_%s_%s_%s' % ( process, energy, polarisation, meta['Datatype'] ) )
  description = 'Replicate %s %s %s %s to' % ( process, energy, polarisation, meta['Datatype'] )
  for replicaSRM in replicaSRMs:
    description += ' %s,' % ( replicaSRM )
  description.rstrip( ',' )
  Trans.setDescription( description )
  Trans.setLongDescription( description )
  Trans.setType( 'Replication' )
  Trans.setPlugin( 'Broadcast' )
  Trans.setSourceSE( outputSRM )
  Trans.setTargetSE( replicaSRMs )

  res = Trans.addTransformation()
  if not res['OK']:
    print res
name = prodJob.workflow.getName()
name = name.replace('/', '').replace('\\', '')
prodJob.workflow.toXMLFile(name)

print 'Workflow XML file name is: %s' % name

workflowBody = ''
if os.path.exists(name):
  with open(name, 'r') as fopen:
    workflowBody = fopen.read()
else:
  print 'Could not get workflow body'

# Standard parameters
transformation = Transformation()
transformation.setTransformationName(name)
transformation.setTransformationGroup('Test')
transformation.setDescription(desc)
transformation.setLongDescription(desc)
transformation.setType('Merge')
transformation.setBody(workflowBody)
transformation.setPlugin('Standard')
transformation.setTransformationFamily('Test')
transformation.setGroupSize(2)
transformation.setOutputDirectories(['/dirac/outConfigName/configVersion/LOG/00000000',
                                     '/dirac/outConfigName/configVersion/RAW/00000000',
                                     '/dirac/outConfigName/configVersion/CORE/00000000'])


# Set directory meta data and create a transformation with a meta-data filter
def submitTS(job):
    """ Create a transformation executing the job workflow  """

    ### Temporary fix to initialize JOB_ID #######
    job.workflow.addParameter(
        Parameter("JOB_ID", "000000", "string", "", "", True, False,
                  "Temporary fix"))
    job.workflow.addParameter(
        Parameter("PRODUCTION_ID", "000000", "string", "", "", True, False,
                  "Temporary fix"))
    job.setType('MCSimulation')  ## Used for the JobType plugin

    t = Transformation()
    # t.setTransformationName("Prod3Exemple")  # This must be unique. If not set it's asked in the prompt
    t.setType("MCSimulation")
    t.setDescription("MC Prod3 BaseLine HE test")
    t.setLongDescription("corsika-simtel production")  # mandatory
    t.setBody(job.workflow.toXML())

    res = t.addTransformation()  # Transformation is created here

    if not res['OK']:
        print(res['Message'])
        DIRAC.exit(-1)

    t.setStatus("Active")
    t.setAgentType("Automatic")

    return res
def createDataTransformation(
    flavour,
    targetSE,
    sourceSE,
    metaKey,
    metaValue,
    extraData=None,
    extraname="",
    groupSize=1,
    plugin="Broadcast",
    tGroup=None,
    tBody=None,
    enable=False,
):
    """Creates the replication transformation based on the given parameters.

    :param str flavour: Flavour of replication to create: Replication or Moving
    :param targetSE: Destination for files
    :type targetSE: python:list or str
    :param sourceSE: Origin of files
    :type sourceSE: python:list or str
    :param int metaKey: Meta key to identify input files
    :param int metaValue: Meta value to identify input files
    :param dict metaData: Additional meta data to use to identify input files
    :param str extraname: addition to the transformation name, only needed if the same transformation was already created
    :param int groupSize: number of files per transformation taks
    :param str plugin: plugin to use
    :param str tGroup: transformation group to set
    :param tBody: transformation body to set
    :param bool enable: if true submit the transformation, otherwise dry run
    :returns: S_OK (with the transformation object, if successfully added), S_ERROR
    """
    metadata = {metaKey: metaValue}
    if isinstance(extraData, dict):
        metadata.update(extraData)

    gLogger.debug("Using %r for metadata search" % metadata)

    if isinstance(targetSE, six.string_types):
        targetSE = [targetSE]

    if isinstance(sourceSE, (list, tuple)):
        sourceSE = "%s" % (",".join(sourceSE))

    gLogger.debug("Using plugin: %r" % plugin)

    if flavour not in ("Replication", "Moving"):
        return S_ERROR("Unsupported flavour %s" % flavour)

    transVerb = {"Replication": "Replicate", "Moving": "Move"}[flavour]
    transGroup = {
        "Replication": "Replication",
        "Moving": "Moving"
    }[flavour] if not tGroup else tGroup

    trans = Transformation()
    transName = "%s_%s_%s" % (transVerb, str(metaValue), ",".join(targetSE))
    if extraname:
        transName += "_%s" % extraname

    trans.setTransformationName(transName)
    description = "%s files for %s %s to %s" % (
        transVerb, metaKey, str(metaValue), ",".join(targetSE))
    trans.setDescription(description[:255])
    trans.setLongDescription(description)
    trans.setType("Replication")
    trans.setTransformationGroup(transGroup)
    trans.setGroupSize(groupSize)
    trans.setPlugin(plugin)

    transBody = ({
        "Moving": [
            ("ReplicateAndRegister", {
                "SourceSE": sourceSE,
                "TargetSE": targetSE
            }),
            ("RemoveReplica", {
                "TargetSE": sourceSE
            }),
        ],
        "Replication":
        "",  # empty body
    }[flavour] if tBody is None else tBody)

    trans.setBody(transBody)
    trans.setInputMetaQuery(metadata)

    if sourceSE:
        res = trans.setSourceSE(sourceSE)
        if not res["OK"]:
            return S_ERROR("SourceSE not valid: %s" % res["Message"])
    res = trans.setTargetSE(targetSE)
    if not res["OK"]:
        return S_ERROR("TargetSE not valid: %s" % res["Message"])

    if not enable:
        gLogger.always("Dry run, not creating transformation")
        return S_OK()

    res = trans.addTransformation()
    if not res["OK"]:
        return res
    gLogger.verbose(res)
    trans.setStatus("Active")
    trans.setAgentType("Automatic")

    gLogger.always("Successfully created replication transformation")
    return S_OK(trans)
def submit_trans(job, trans_name, mqJson, group_size, with_file_mask=True):
    """ Create a transformation executing the job workflow
    """
    DIRAC.gLogger.notice('submit_trans : %s' % trans_name)

    # Initialize JOB_ID
    job.workflow.addParameter(
        Parameter("JOB_ID", "000000", "string", "", "", True, False,
                  "Temporary fix"))

    trans = Transformation()
    trans.setTransformationName(trans_name)  # this must be unique
    trans.setType("DataReprocessing")
    trans.setDescription("Prod3 DL1 Data Handler TS")
    trans.setLongDescription("Prod3 DL1 Data Handler conversion")  # mandatory
    trans.setBody(job.workflow.toXML())
    trans.setGroupSize(group_size)
    if with_file_mask:
        trans.setFileMask(mqJson)  # catalog query is defined here
    result = trans.addTransformation()  # transformation is created here
    if not result['OK']:
        return result
    trans.setStatus("Active")
    trans.setAgentType("Automatic")
    trans_id = trans.getTransformationID()
    return trans_id
if not result['OK']:
    gLogger.error("Error: %s" % result['Message'])
    exit(0)
if not result['Value']:
    gLogger.error("Error: no metadata fields defined")
    exit(0)
typeDict = result['Value']['FileMetaFields']
typeDict.update(result['Value']['DirectoryMetaFields'])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
mq = MetaQuery(typeDict=typeDict)
mq.setMetaQuery([metaTransfer])
query = mq.getMetaQuery()
gLogger.notice('Query: {0}'.format(query))

t = Transformation()
tc = TransformationClient()
t.setTransformationName(transformationName)  # Must be unique
t.setTransformationGroup("Transfer")
t.setType(transformationType)
t.setPlugin(plugin)
t.setDescription("Data Transfer")
t.setLongDescription("Data Transfer")  # Mandatory
t.setGroupSize(
    groupSize
)  # Here you specify how many files should be grouped within he same request, e.g. 100

transBody = ''
t.setBody(transBody)

if sourceSE:
 def test_ExtendCleanTransformation(self):
   """ Tests the extension of transformations and the removal of tasks. Also obtain tasks, their status and update their status.
       
         extendTransformation()
         getTransformationTasks()
         getTransformationTaskStats()
         deleteTasks()
         setTaskStatus()
         cleanTransformation()
       
       Tests a transformation can be extended.
       Tests can obtain the the transformation tasks and their statistics.
       Tests the removal of already created tasks.
       Tests can change the status of a task.
       Tests that Cleaning a transformation removes tasks defined for the transformation.
   """
   oTrans = Transformation(self.transID)
   nTasks = 100
   res = oTrans.extendTransformation(nTasks)
   self.assert_(res['OK'])
   taskIDs = res['Value']
   self.assertEqual(len(taskIDs),nTasks)
   res = oTrans.getTransformationTasks()
   self.assert_(res['OK'])
   parameters = ['TargetSE', 'TransformationID', 'LastUpdateTime', 'ExternalID', 'CreationTime', 'TaskID', 'ExternalStatus']
   self.assertEqual(sortList(res['ParameterNames']),sortList(parameters))
   self.assertEqual(sortList(res['Value'][0].keys()),sortList(parameters))
   self.assertEqual(res['Value'][0]['TargetSE'],'Unknown')
   self.assertEqual(res['Value'][0]['TransformationID'],self.transID)
   self.assertEqual(res['Value'][0]['ExternalID'],'0')
   self.assertEqual(res['Value'][0]['TaskID'],1)
   self.assertEqual(res['Value'][0]['ExternalStatus'],'Created')
   self.assertEqual(res['Records'][0][0],1)
   self.assertEqual(res['Records'][0][1],self.transID)
   self.assertEqual(res['Records'][0][2],'Created')
   self.assertEqual(res['Records'][0][3],'0')
   self.assertEqual(res['Records'][0][4],'Unknown')
   res = oTrans.getTransformationTaskStats()
   self.assert_(res['OK'])
   self.assertEqual(res['Value']['Created'],100)
   res = oTrans.deleteTasks(11,100)
   self.assert_(res['OK'])
   res = oTrans.getTransformationTaskStats()
   self.assert_(res['OK'])
   self.assertEqual(res['Value']['Created'],10)
   res = oTrans.setTaskStatus(1, 'Done')
   self.assert_(res['OK'])
   res = oTrans.getTransformationTaskStats()
   self.assert_(res['OK'])
   self.assertEqual(res['Value']['Created'],10)
   self.assertEqual(res['Value']['Done'],1)
   res = oTrans.cleanTransformation()
   self.assert_(res['OK'])
   res = oTrans.getStatus()
   self.assert_(res['OK'])
   self.assertEqual(res['Value'],'Cleaned')
   res = oTrans.getTransformationTasks()
   self.assert_(res['OK'])
   self.assertFalse(res['Value'])
   self.assertFalse(res['Records'])
Exemple #36
0
def createMovingTransformation(targetSE,
                               sourceSE,
                               prodID,
                               datatype,
                               extraname='',
                               forceMoving=False):
    """Creates the replication transformation based on the given parameters

  
  :param targetSE: Destination for files
  :type targetSE: python:list or str
  :param str sourceSE: Origin of files. Files will be removed from this SE
  :param int prodID: Production ID of files to be moved
  :param str datatype: DataType of files to be moved
  :param str extraname: addition to the transformation name, only needed if the same transformation was already created
  :param bool forceMoving: Move always, even if GEN/SIM files don't have descendents
  :returns: S_OK, S_ERROR
  """

    metadata = {"Datatype": datatype, "ProdID": prodID}

    if isinstance(targetSE, basestring):
        targetSE = [targetSE]

    trans = Transformation()
    transName = 'Move_%s_%s_%s' % (datatype, str(prodID), ",".join(targetSE))
    if extraname:
        transName += "_%s" % extraname

    trans.setTransformationName(transName)
    description = 'Move files for prodID %s to %s' % (str(prodID),
                                                      ",".join(targetSE))
    trans.setDescription(description)
    trans.setLongDescription(description)
    trans.setType('Replication')
    trans.setGroup('Moving')
    if datatype in ('GEN', 'SIM') and not forceMoving:
        trans.setPlugin('BroadcastProcessed')
    else:
        trans.setPlugin('Broadcast')

    transBody = [
        ("ReplicateAndRegister", {
            "SourceSE": sourceSE,
            "TargetSE": targetSE
        }),
        ("RemoveReplica", {
            "TargetSE": sourceSE
        }),
    ]

    trans.setBody(transBody)

    res = trans.setSourceSE(sourceSE)
    if not res['OK']:
        return S_ERROR("SourceSE not valid: %s" % res['Message'])
    res = trans.setTargetSE(targetSE)
    if not res['OK']:
        return S_ERROR("TargetSE not valid: %s" % res['Message'])

    res = trans.addTransformation()
    if not res['OK']:
        gLogger.error("Failed to create Transformation", res['Message'])
        return res
    gLogger.verbose(res)
    trans.setStatus('Active')
    trans.setAgentType('Automatic')
    currtrans = trans.getTransformationID()['Value']
    client = TransformationClient()
    res = client.createTransformationInputDataQuery(currtrans, metadata)
    if res['OK']:
        gLogger.always("Successfully created replication transformation")
        return S_OK()
    else:
        gLogger.error("Failure during replication creation", res['Message'])
        return S_ERROR("Failed to create transformation:%s " % res['Message'])
Exemple #37
0
  exit(1)

print("Example file: %s" % lfns[0])

answer = raw_input('Proceed and submit replication? (Y/N): ')
if not answer.lower() in ('y', 'yes'):
  print("Canceled")
  exit(1)

trc = TransformationClient()
res = trc.getTransformationStats(name_of_replication)
if res['OK']:
  print("Replication with name %s already exists! Cannot proceed." % name_of_replication)
  exit(1)
  
Trans = Transformation()
Trans.setTransformationName(name_of_replication)
Trans.setDescription(description)
Trans.setLongDescription(description)
Trans.setType('Replication')
Trans.setPlugin('Broadcast')
#Trans.setFileMask(fmask)
Trans.setSourceSE(source)
Trans.setTargetSE(destination)
res = Trans.addTransformation()
if not res['OK']:
  print("Failed to add Replication: %s" % res['Message'])
  exit(1)
Trans.setStatus("Active")
Trans.setAgentType("Automatic")
currtrans = Trans.getTransformationID()['Value']
def createDataTransformation(flavour, targetSE, sourceSE,
                             metaKey, metaValue,
                             extraData=None, extraname='',
                             groupSize=1,
                             plugin='Broadcast',
                             tGroup=None,
                             tBody=None,
                             enable=False,
                             ):
  """Creates the replication transformation based on the given parameters.

  :param str flavour: Flavour of replication to create: Replication or Moving
  :param targetSE: Destination for files
  :type targetSE: python:list or str
  :param str sourceSE: Origin of files.
  :param int metaKey: Meta key to identify input files
  :param int metaValue: Meta value to identify input files
  :param dict metaData: Additional meta data to use to identify input files
  :param str extraname: addition to the transformation name, only needed if the same transformation was already created
  :param int groupSize: number of files per transformation taks
  :param str plugin: plugin to use
  :param str tGroup: transformation group to set
  :param tBody: transformation body to set
  :param bool enable: if true submit the transformation, otherwise dry run
  :returns: S_OK (with the transformation object, if successfully added), S_ERROR
  """
  metadata = {metaKey: metaValue}
  if isinstance(extraData, dict):
    metadata.update(extraData)

  gLogger.debug("Using %r for metadata search" % metadata)

  if isinstance(targetSE, basestring):
    targetSE = [targetSE]

  if flavour not in ('Replication', 'Moving'):
    return S_ERROR('Unsupported flavour %s' % flavour)

  transVerb = {'Replication': 'Replicate', 'Moving': 'Move'}[flavour]
  transGroup = {'Replication': 'Replication', 'Moving': 'Moving'}[flavour] if not tGroup else tGroup

  trans = Transformation()
  transName = '%s_%s_%s' % (transVerb, str(metaValue), ",".join(targetSE))
  if extraname:
    transName += "_%s" % extraname

  trans.setTransformationName(transName)
  description = '%s files for %s %s to %s' % (transVerb, metaKey, str(metaValue), ",".join(targetSE))
  trans.setDescription(description)
  trans.setLongDescription(description)
  trans.setType('Replication')
  trans.setTransformationGroup(transGroup)
  trans.setGroupSize(groupSize)
  trans.setPlugin(plugin)

  transBody = {'Moving': [("ReplicateAndRegister", {"SourceSE": sourceSE, "TargetSE": targetSE}),
                          ("RemoveReplica", {"TargetSE": sourceSE})],
               'Replication': '',  # empty body
               }[flavour] if tBody is None else tBody

  trans.setBody(transBody)

  if sourceSE:
    res = trans.setSourceSE(sourceSE)
    if not res['OK']:
      return S_ERROR("SourceSE not valid: %s" % res['Message'])
  res = trans.setTargetSE(targetSE)
  if not res['OK']:
    return S_ERROR("TargetSE not valid: %s" % res['Message'])

  if not enable:
    gLogger.always("Dry run, not creating transformation")
    return S_OK()

  res = trans.addTransformation()
  if not res['OK']:
    return res
  gLogger.verbose(res)
  trans.setStatus('Active')
  trans.setAgentType('Automatic')
  currtrans = trans.getTransformationID()['Value']
  client = TransformationClient()
  res = client.createTransformationInputDataQuery(currtrans, metadata)
  if not res['OK']:
    return res

  gLogger.always("Successfully created replication transformation")
  return S_OK(trans)
def submitTS( job, infileList ):
  """ Create a transformation executing the job workflow  """
  t = Transformation()
  tc = TransformationClient()
  t.setType( "SimtelMerging" )
  t.setDescription( "Runs merge_simtel for array 3HB89" )
  t.setLongDescription( "Merging array 3HB89 analysis" )  # mandatory
  t.setGroupSize(5)
  t.setBody ( job.workflow.toXML() )

  res = t.addTransformation()  # Transformation is created here

  if not res['OK']:
    print res['Message']
    DIRAC.exit( -1 )

  t.setStatus( "Active" )
  t.setAgentType( "Automatic" )
  transID = t.getTransformationID()
  print('Adding %s files to transformation'%len(infileList))
  tc.addFilesToTransformation( transID['Value'], infileList )  # Files added here

  return res
Exemple #40
0
def submit_trans(job, trans_name, input_meta_query, group_size):
    """ Create a transformation executing the job workflow
    """
    DIRAC.gLogger.notice('submit_trans : %s' % trans_name)

    # Initialize JOB_ID
    job.workflow.addParameter(
        Parameter("JOB_ID", "000000", "string", "", "", True, False,
                  "Temporary fix"))

    trans = Transformation()
    trans.setTransformationName(trans_name)  # this must be unique
    trans.setType("DataReprocessing")
    trans.setDescription("Prod5 EventDisplay TS")
    trans.setLongDescription("Prod5 EventDisplay processing")  # mandatory
    trans.setBody(job.workflow.toXML())
    trans.setGroupSize(group_size)
    trans.setInputMetaQuery(input_meta_query)
    result = trans.addTransformation()  # transformation is created here
    if not result['OK']:
        return result
    trans.setStatus("Active")
    trans.setAgentType("Automatic")
    trans_id = trans.getTransformationID()
    return trans_id
 def tearDown(self):
   if self.transID:
     oTrans = Transformation(self.transID)
     oTrans.deleteTransformation()
Exemple #42
0
def submit_trans(job, trans_name):
    """ Create a transformation executing the job workflow
    """
    DIRAC.gLogger.notice('submit_trans : %s' % trans_name)

    # Initialize JOB_ID
    job.workflow.addParameter(Parameter("JOB_ID", "000000", "string", "", "",
                                        True, False, "Temporary fix"))

    trans = Transformation()
    trans.setTransformationName(trans_name)  # this must be unique
    trans.setType("MCSimulation")
    trans.setDescription("Prod5 MC Pipe NSB TS")
    trans.setLongDescription("Prod5 simulation pipeline")  # mandatory
    trans.setBody(job.workflow.toXML())
    result = trans.addTransformation()  # transformation is created here
    if not result['OK']:
        return result
    trans.setStatus("Active")
    trans.setAgentType("Automatic")
    trans_id = trans.getTransformationID()
    return trans_id
def submit_trans(job, infileList, trans_name, group_size):
    """ Create a transformation executing the job workflow
    """
    DIRAC.gLogger.notice('submit_trans : %s' % trans_name)

    # Initialize JOB_ID
    job.workflow.addParameter(
        Parameter("JOB_ID", "000000", "string", "", "", True, False,
                  "Temporary fix"))

    trans = Transformation()
    trans.setTransformationName(trans_name)  # this must be unique
    trans.setType("DataReprocessing")
    trans.setDescription("Simtel TS example")
    trans.setLongDescription("Simtel tel_sim")  # mandatory
    trans.setBody(job.workflow.toXML())
    trans.setGroupSize(group_size)
    res = trans.addTransformation()  # transformation is created here
    if not res['OK']:
        return res
    trans.setStatus("Active")
    trans.setAgentType("Automatic")
    # add 10*group_size files to transformation (to have the first 10 jobs)
    trans_id = trans.getTransformationID()
    trans_client = TransformationClient()
    res = trans_client.addFilesToTransformation(trans_id['Value'],
                                                infileList[:10 * group_size])
    return res
def submit_trans(job, input_meta_query, group_size):
    """ Create a transformation executing the job workflow  """

    #DIRAC.gLogger.notice('submit_trans : %s' % trans_name)

    # Initialize JOB_ID
    job.workflow.addParameter(
        Parameter("JOB_ID", "000000", "string", "", "", True, False,
                  "Temporary fix"))

    t = Transformation()
    t.setType("DataReprocessing")
    t.setDescription("EvnDisplay MQ example")
    t.setLongDescription("EvnDisplay calib_imgreco")  # mandatory
    t.setBody(job.workflow.toXML())
    t.setGroupSize(group_size)
    t.setInputMetaQuery(input_meta_query)
    res = t.addTransformation()  # transformation is created here
    if not res['OK']:
        return res

    #t.setStatus("Active")
    t.setAgentType("Automatic")
    trans_id = t.getTransformationID()

    return trans_id
Exemple #45
0
def submitTS():

    ########################################
    # Modify here with your dirac username
    owner = 'user02'
    ########################################

    ########################################
    # Job description
    ########################################
    job = Job()
    job.setName('merge mandelbrot')
    job.setOutputSandbox(['*log'])

    ## define the job workflow in 3 steps
    # job step1: setup software
    job.setExecutable('git clone https://github.com/bregeon/mandel4ts.git')
    # job step2: run mandelbrot merge
    job.setExecutable('./mandel4ts/merge_data.py')

    outputPath = os.path.join('/vo.france-grilles.fr/user', owner[0], owner,
                              'mandelbrot/images/merged')
    outputPattern = 'data_merged*txt'
    outputSE = 'DIRAC-USER'
    nb_input_files = 7
    outputMetadata = json.dumps({
        "application": "mandelbrot",
        "image_format": "ascii",
        "image_width": 7680,
        "image_height": 200 * nb_input_files,
        "owner": owner
    })

    # job step3: upload data and set metadata
    job.setExecutable('./mandel4ts/dirac-add-files.py',
                      arguments="%s '%s' %s '%s'" %
                      (outputPath, outputPattern, outputSE, outputMetadata))

    ########################################
    # Transformation definition
    ########################################
    t = Transformation()

    t.setTransformationName(owner + '_step2')
    t.setType("DataReprocessing")
    t.setDescription("Merge mandelbrot images production")
    t.setLongDescription("Merge mandelbrot images production")
    t.setGroupSize(nb_input_files)  # group input files
    # set the job workflow to the transformation
    t.setBody(job.workflow.toXML())

    # define input data by metadata query
    inputMetaquery = json.dumps({
        "application": "mandelbrot",
        "image_format": "ascii",
        "image_width": 7680,
        "image_height": 200,
        "owner": owner
    })
    t.setFileMask(inputMetaquery)

    ########################################
    # Transformation submission
    ########################################
    res = t.addTransformation()

    if not res['OK']:
        print(res['Message'])
        DIRAC.exit(-1)

    t.setStatus("Active")
    t.setAgentType("Automatic")

    return res
Exemple #46
0
def submitTS():

  ########################################
  # Modify here with your dirac username 
  owner = 'user02'
  ########################################

  
  ########################################
  # Job description
  ########################################
  job = Job()
  job.setName('build mandelbrot')
  job.setOutputSandbox( ['*log'] )
  job.setType('DataReprocessing')

  ## define the job workflow in 3 steps
  # job step1: setup software  
  job.setExecutable('git clone https://github.com/bregeon/mandel4ts.git')
  # job step2: run mandelbrot build image
  job.setExecutable('./mandel4ts/build_merged_img.py')

  outputPath = os.path.join('/vo.france-grilles.fr/user',owner[0],owner,'mandelbrot/images/final')
  outputPattern = 'merged_image.bmp'
  outputSE = 'DIRAC-USER'
  outputMetadata = json.dumps( {"application":"mandelbrot","image_format":"bmp", "image_width":7680, "image_height":4200, "owner":owner} )

  # job step3: upload data and set metadata
  job.setExecutable( './mandel4ts/dirac-add-files.py', arguments = "%s '%s' %s '%s'" % (outputPath, outputPattern, outputSE, outputMetadata ) )
  
  # job step4: mark input files as done with the FailoverRequest (and a few other things)
  job.setExecutable('/bin/ls -l', modulesList=['Script', 'FailoverRequest'])

  ########################################
  # Transformation definition
  ########################################
  t = Transformation()

  t.setTransformationName( owner+'_step3' )
  t.setType( "DataReprocessing" ) 
  t.setDescription( "Merge mandelbrot images production" )
  t.setLongDescription( "Merge mandelbrot images production" )
  t.setGroupSize( 3 ) # group input files
  # set the job workflow to the transformation
  t.setBody ( job.workflow.toXML() )

  # define input data by metadata query
  inputMetaquery = {"application":"mandelbrot","image_format":"ascii", "image_width":7680, "image_height":1400, "owner":owner} 
  t.setInputMetaQuery(inputMetaquery) 

  ########################################
  # Transformation submission
  ########################################
  res = t.addTransformation() 

  if not res['OK']:
    print(res['Message'])
    DIRAC.exit( -1 )

  t.setStatus( "Active" )
  t.setAgentType( "Automatic" )
  
  return res
def DataReplicaRemovalTSExample(args=None):

    from DIRAC.TransformationSystem.Client.Transformation import Transformation
    from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient

    if (len(args) != 1):
        Script.gLogger.notice('Wrong number of arguments')
        Script.showHelp()

    infile = args[0]
    f = open(infile, 'r')

    infileList = []
    for line in f:
        infile = line.strip()
        if line != "\n":
            infileList.append(infile)

    t = Transformation()
    tc = TransformationClient()

    t.setType("Removal")

    t.setPlugin("Broadcast")  # Mandatory for ReplicaRemoval
    t.setSourceSE(['CC-IN2P3-Tape'
                   ])  # A list of SE where at least 1 SE is the valid one
    t.setTargetSE([
        'CEA-Disk', 'LAPP-Disk', 'DESY-ZN-Disk', 'CC-IN2P3-Disk',
        'CYF-STORM-Disk'
    ])  # A list of SE where at least 1 SE is the valid one
    t.setDescription("AarProd2 ReplicaRemoval")
    t.setLongDescription("AarProd2 ReplicaRemoval")  # Mandatory

    t.setGroupSize(
        100
    )  # Here you specify how many files should be grouped within the same request, e.g. 100
    t.setBody("Removal;RemoveReplica"
              )  # Mandatory (the default is a ReplicateAndRegister operation)

    t.addTransformation()  # Transformation is created here
    t.setStatus("Active")
    t.setAgentType("Automatic")

    transID = t.getTransformationID()
    tc.addFilesToTransformation(transID['Value'],
                                infileList)  # Files are added here
Exemple #48
0
def submitTS(job, transName, mqJson):
    """ Create a transformation executing the job workflow
    This is using a file mask so that files be added on the fly.
    """
    DIRAC.gLogger.notice('submitTS')

    # Initialize JOB_ID
    job.workflow.addParameter(
        Parameter("JOB_ID", "000000", "string", "", "", True, False,
                  "Temporary fix"))

    t = Transformation()
    t.setTransformationName(transName)  # this must be unique
    t.setType("DataReprocessing")
    t.setDescription("EvnDisp3MSCW example")
    t.setLongDescription("EvnDisplay stereo reconstruction")  # mandatory
    t.setBody(job.workflow.toXML())
    t.setGroupSize(100)
    t.setFileMask(mqJson)  # catalog query is defined here
    t.addTransformation()  # transformation is created here
    t.setStatus("Active")
    t.setAgentType("Automatic")

    return
def _createReplication( targetSE, sourceSE, prodID, datatype, extraname=''):
  """Creates the replication transformation based on the given parameters"""

  from DIRAC.TransformationSystem.Client.Transformation import Transformation
  from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
  metadata = {"Datatype":datatype, "ProdID":prodID}
  
  trans = Transformation()
  transName = 'replicate_%s_%s' % ( str(prodID), ",".join(targetSE) )
  if extraname:
    transName += "_%s" % extraname

  trans.setTransformationName( transName )
  description = 'Replicate files for prodID %s to %s' % ( str(prodID), ",".join(targetSE) )
  trans.setDescription( description )
  trans.setLongDescription( description )
  trans.setType( 'Replication' )
  trans.setPlugin( 'Broadcast' )
  res = trans.setSourceSE( sourceSE )
  if not res['OK']:
    exit(1)
  res = trans.setTargetSE( targetSE )
  if not res['OK']:
    exit(1)

  res = trans.addTransformation()
  if not res['OK']:
    gLogger.error(res['Message'])
    exit(1)
  gLogger.verbose(res)
  trans.setStatus( 'Active' )
  trans.setAgentType( 'Automatic' )
  currtrans = trans.getTransformationID()['Value']
  client = TransformationClient()
  res = client.createTransformationInputDataQuery( currtrans, metadata )
  if res['OK']:
    gLogger.always("Successfully created replication transformation")
    return S_OK()
  else:
    gLogger.error("Failure during replication creation", res['Message'])
    return S_ERROR("Failed to create transformation")