Beispiel #1
0
def _getFileInfo(lfn):
    """ Retrieve the file info
  """
    from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
    from DIRAC.Core.Utilities import DEncode
    from DIRAC import gLogger
    fc = FileCatalogClient()
    lumi = 0
    nbevts = 0
    res = fc.getFileUserMetadata(lfn)
    if not res['OK']:
        gLogger.error("Failed to get metadata of %s" % lfn)
        return (0, 0, {})
    if 'Luminosity' in res['Value']:
        lumi += float(res['Value']['Luminosity'])
    addinfo = {}
    if 'AdditionalInfo' in res['Value']:
        addinfo = res['Value']['AdditionalInfo']
        if addinfo.count("{"):
            addinfo = eval(addinfo)
        else:
            addinfo = DEncode.decode(addinfo)[0]
    if "NumberOfEvents" in res['Value'].keys():
        nbevts += int(res['Value']['NumberOfEvents'])
    return (float(lumi), int(nbevts), addinfo)
Beispiel #2
0
    def initialize(self):
        """agent initialisation

        reading and setting config opts

        :param self: self reference
        """
        # # shifter proxy
        # See cleanContent method: this proxy will be used ALSO when the file catalog used
        # is the DIRAC File Catalog (DFC).
        # This is possible because of unset of the "UseServerCertificate" option
        self.shifterProxy = self.am_getOption("shifterProxy",
                                              self.shifterProxy)

        # # transformations types
        self.dataProcTTypes = Operations().getValue(
            "Transformations/DataProcessing", self.dataProcTTypes)
        self.dataManipTTypes = Operations().getValue(
            "Transformations/DataManipulation", self.dataManipTTypes)
        agentTSTypes = self.am_getOption("TransformationTypes", [])
        if agentTSTypes:
            self.transformationTypes = sorted(agentTSTypes)
        else:
            self.transformationTypes = sorted(self.dataProcTTypes +
                                              self.dataManipTTypes)
        self.log.info("Will consider the following transformation types: %s" %
                      str(self.transformationTypes))
        # # directory locations
        self.directoryLocations = sorted(
            self.am_getOption("DirectoryLocations", self.directoryLocations))
        self.log.info(
            "Will search for directories in the following locations: %s" %
            str(self.directoryLocations))
        # # transformation metadata
        self.transfidmeta = self.am_getOption("TransfIDMeta",
                                              self.transfidmeta)
        self.log.info("Will use %s as metadata tag name for TransformationID" %
                      self.transfidmeta)
        # # archive periof in days
        self.archiveAfter = self.am_getOption("ArchiveAfter",
                                              self.archiveAfter)  # days
        self.log.info("Will archive Completed transformations after %d days" %
                      self.archiveAfter)
        # # transformation log SEs
        self.logSE = Operations().getValue("/LogStorage/LogSE", self.logSE)
        self.log.info("Will remove logs found on storage element: %s" %
                      self.logSE)

        # # transformation client
        self.transClient = TransformationClient()
        # # wms client
        self.wmsClient = WMSClient()
        # # request client
        self.reqClient = ReqClient()
        # # file catalog client
        self.metadataClient = FileCatalogClient()
        # # job monitoring client
        self.jobMonitoringClient = JobMonitoringClient()

        return S_OK()
Beispiel #3
0
  def __init__( self, *args, **kwargs ):
    ''' c'tor
    '''
    AgentModule.__init__( self, *args, **kwargs )
    # # replica manager
    self.replicaManager = ReplicaManager()
    # # transformation client
    self.transClient = TransformationClient()
    # # wms client
    self.wmsClient = WMSClient()
    # # request client
    self.requestClient = RequestClient()
    # # file catalog clinet
    self.metadataClient = FileCatalogClient()

    # # placeholders for CS options

    # # transformations types
    self.transformationTypes = None
    # # directory locations
    self.directoryLocations = None
    # # transformation metadata
    self.transfidmeta = None
    # # archive periof in days
    self.archiveAfter = None
    # # active SEs
    self.activeStorages = None
    # # transformation log SEs
    self.logSE = None
    # # enable/disable execution
    self.enableFlag = None
Beispiel #4
0
 def __init__(self, cpuTime=432000):
     """ Constructor
 
 Keyword arguments:
 cpuTime -- max cpu time allowed for the job
 """
     Job.__init__(self)
     self.setCPUTime(cpuTime)
     # defaults
     self.setName('Evndisplay_Analysis')
     self.package = 'evndisplay'
     self.program_category = 'analysis'  #'calibimgreco'
     self.version = 'prod3_d20150831b'
     self.prefix = 'CTA.prod3S'
     self.layout_list = '3HB1 3HB2 3HB3 3HD1 3HD2 3HI1'
     self.calibration_file = 'prod3.peds.20150820.dst.root'
     self.reconstructionparameter = 'EVNDISP.prod3.reconstruction.runparameter.NN'
     self.NNcleaninginputcard = 'EVNDISP.NNcleaning.dat'
     self.basepath = '/vo.cta.in2p3.fr/MC/PROD3/'
     self.outputpattern = './*evndisp.tar.gz'
     self.fcc = FileCatalogClient()
     self.metadata = collections.OrderedDict()
     self.filemetadata = {}
     self.catalogs = json.dumps(['DIRACFileCatalog', 'TSCatalog'])
     self.jobGroupID = 1
  def _getProdInfoFromIDs(self):
    """get the processName, energy and eventsPerJob from the MetaData catalog

    :raises: AttributeError if some of the information cannot be found
    :returns: None
    """
    if not self.prodIDs:
      raise AttributeError("No prodIDs defined")

    self.eventsPerJobs = []
    self.processes = []
    self.energies = []
    from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
    from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
    trc = TransformationClient()
    fc = FileCatalogClient()
    for prodID in self.prodIDs:
      gLogger.notice("Getting information for %s" % prodID)
      tRes = trc.getTransformation(str(prodID))
      if not tRes['OK']:
        raise AttributeError("No prodInfo found for %s" % prodID)
      self.eventsPerJobs.append(int(tRes['Value']['EventsPerTask']))
      lfnRes = fc.findFilesByMetadata({'ProdID': prodID})
      if not lfnRes['OK'] or not lfnRes['Value']:
        raise AttributeError("Could not find files for %s: %s " % (prodID, lfnRes.get('Message', lfnRes.get('Value'))))
      path = os.path.dirname(lfnRes['Value'][0])
      fileRes = fc.getDirectoryUserMetadata(path)
      self.processes.append(fileRes['Value']['EvtType'])
      self.energies.append(fileRes['Value']['Energy'])
      gLogger.notice("Found (Evts,Type,Energy): %s %s %s " %
                     (self.eventsPerJobs[-1], self.processes[-1], self.energies[-1]))
    def initialize(self):
        """ agent's initalisation

        :param self: self reference
        """
        gLogger.info('Initialize')
        self.dataManager = DataManager()
        self.am_setOption('shifterProxy', 'DataManager')
        self.CopyToSE = (self.am_getOption("CopyToSE", 'PNNL-PIC-SRM-SE'))
        self.SEDataDirPath = (self.am_getOption("SEDataDirPath",
                                                '/project8/dirac/calib/'))
        self.LocalDataDirPath = (self.am_getOption("LocalDataDirPath",
                                                   '/data_claude/'))
        self.DIRACCfgSEPath = 'Resources/StorageElements'
        self.dryRun = bool(self.am_getOption("DryRun", True))
        self.fc = FileCatalogClient()

        gLogger.info("DryRun: " + str(self.dryRun))
        gLogger.info("CopyToSE: " + str(self.CopyToSE))

        ### This defines which calibration directories to consider
        self.calibDirs = ['rf_bkgd', 'esr']
        ### This defines which sub dir under calib dir to replicate
        self.rawDataDir = 'raw'
        self.acceptableFileSuffix = [
            '-esr.json', '.json', '.dpt', '.root', '.Setup'
        ]

        return S_OK()
Beispiel #7
0
 def setupCatalogClient( self, catalogs ):
   """ Init FileCatalog client
       Ideally we would like to use only FileCatalog but it doesn't work for setMetadata
       because the returned value has a wrong format. So use of FileCatalogClient instead
   """
   self.fc = FileCatalog( catalogs )
   self.fcc = FileCatalogClient()
Beispiel #8
0
  def __init__( self, cpuTime = 432000 ):
    """ Constructor
    
    Keyword arguments:
    cpuTime -- max cpu time allowed for the job
    """
    Job.__init__( self )
    self.setCPUTime( cpuTime )
    # defaults
    self.setName( 'Mars_Analysis' )
    self.package = 'chimp'
    self.version = 'prod3_xxx'
    self.PixelRequiredPhes = '-100.'
    self.outdir = './'
    self.MuonMode = '0'
    self.StarOutput = '-staroutput'
#    self.training_type = 'point-like'
    self.training_type = 'diffuse'
    #self.training_type = 'point-like diffuse'
    self.basepath = '/vo.cta.in2p3.fr/MC/PROD3/'
    #self.outputpattern = './stereo_All/*.root' # not used
    self.outputpattern = './data/*.tar.gz' # used for ctastereo
    #self.outputpattern = './*_I.root' # used for chimp
    self.fcc = FileCatalogClient()
    self.metadata = collections.OrderedDict()
    self.filemetadata = {}
    self.jobGroupID = 1
Beispiel #9
0
class TestUserMetadataBasicTestCase(unittest.TestCase):
    def setUp(self):
        self.dirac = Dirac()
        csAPI = CSAPI()

        self.lfn5 = os.path.join(DESTINATION_PATH, 'test_file_10MB_v5.bin')
        self.dir5 = os.path.dirname(self.lfn5)
        # local file, for now:
        self.fname = os.path.basename(self.lfn5)
        random_dd(self.fname, 10)
        self.diracSE = 'SE-1'
        try:
            self.fc = FileCatalogClient("DataManagement/MultiVOFileCatalog")
        except Exception:
            self.fail(
                " FileCatalog(['MultiVOFileCatalog']) raised Exception unexpectedly!\n"
                + traceback.format_exc())
            return
        # add a replica
        self.fileadded = self.dirac.addFile(self.lfn5, self.fname,
                                            self.diracSE)
        self.assertTrue(self.fileadded['OK'])

    def tearDown(self):
        # meta index -r
        result = self.fc.deleteMetadataField('MetaInt6')
        self.assertTrue(result['OK'])
        result = self.fc.deleteMetadataField('TestDirectory6')
        self.assertTrue(result['OK'])
        # remove the MultiVOFileCatalog
        self.fc.removeCatalog('MultiVOFileCatalog')
        # delete a sole replica: dirac-dms-remove-files
        result = self.dirac.removeFile(self.lfn5)
        self.assertTrue(result['OK'])
        os.remove(self.fname)
Beispiel #10
0
 def __init__(self):
   super(OverlayInput, self).__init__()
   self.enable = True
   self.STEP_NUMBER = ''
   self.log = gLogger.getSubLogger( "OverlayInput" )
   self.applicationName = 'OverlayInput'
   self.curdir = os.getcwd()
   self.applicationLog = ''
   self.printoutflag = ''
   self.prodid = 0
   self.detector = '' ##needed for backward compatibility
   self.detectormodel = ""
   self.energytouse = ''
   self.energy = 0
   self.nbofeventsperfile = 100
   self.lfns = []
   self.nbfilestoget = 0
   self.BkgEvtType = 'gghad'
   self.metaEventType = self.BkgEvtType
   self.BXOverlay = 0
   self.ggtohadint = 3.2
   self.nbsigeventsperfile = 0
   self.nbinputsigfile = 1
   self.NbSigEvtsPerJob = 0
   self.datMan = DataManager()
   self.fcc = FileCatalogClient()
   self.site = DIRAC.siteName()
   self.useEnergyForFileLookup = True
   self.machine = 'clic_cdr'
   self.pathToOverlayFiles = ''
   self.processorName = ''
def getFileInfo(lfn):
  """ Retrieve the file info
  """
  from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
  from DIRAC.Core.Utilities import DEncode
  from DIRAC import gLogger
  fc = FileCatalogClient()
  lumi = 0
  nbevts = 0
  res  = fc.getFileUserMetadata(lfn)
  if not res['OK']:
    gLogger.error("Failed to get metadata of %s" % lfn)
    return (0,0,{})
  if res['Value'].has_key('Luminosity'):   
    lumi += float(res['Value']['Luminosity'])
  addinfo = {}
  if 'AdditionalInfo' in res['Value']:
    addinfo = res['Value']['AdditionalInfo']
    if addinfo.count("{"):
      addinfo = eval(addinfo)
    else:
      addinfo = DEncode.decode(addinfo)[0]
  if "NumberOfEvents" in res['Value'].keys():
    nbevts += int(res['Value']['NumberOfEvents'])
  return (float(lumi),int(nbevts),addinfo)
Beispiel #12
0
    def __init__(self, script=None):
        super(ProductionJob, self).__init__(script)
        self.prodVersion = __RCSID__
        self.dryrun = False
        self.created = False
        self.checked = False
        self.call_finalization = False
        self.finalsdict = {}
        self.transfid = 0
        self.type = 'Production'
        self.csSection = '/Production/Defaults'
        self.ops = Operations()
        self.fc = FileCatalogClient()
        self.trc = TransformationClient()
        self.defaultProdID = '12345'
        self.defaultProdJobID = '12345'
        self.jobFileGroupSize = 1
        self.nbtasks = 1
        self.slicesize = 0
        self.basename = ''
        self.basepath = self.ops.getValue('/Production/CLIC/BasePath',
                                          '/ilc/prod/clic/')
        self.evttype = ''
        self.datatype = ''
        self.energycat = ''
        self.detector = ''
        self.currtrans = None
        self.description = ''

        self.finalpaths = []
        self.finalMetaDict = defaultdict(dict)
        self.prodMetaDict = {}
        self.finalMetaDictNonSearch = {}
        self.metadict_external = {}
        self.outputStorage = ''

        self.proxyinfo = getProxyInfo()

        self.inputdataquery = False
        self.inputBKSelection = {}
        self.plugin = 'Standard'
        self.prodGroup = ''

        self.prodTypes = [
            'MCGeneration', 'MCSimulation', 'Test', 'MCReconstruction',
            'MCReconstruction_Overlay', 'Merge', 'Split', 'MCGeneration_ILD',
            'MCSimulation_ILD', 'MCReconstruction_ILD',
            'MCReconstruction_Overlay_ILD', 'Split_ILD'
        ]
        self.prodparameters = {}
        self.prodparameters['NbInputFiles'] = 1
        self.prodparameters['nbevts'] = 0
        #self.prodparameters["SWPackages"] = ''
        self._addParameter(self.workflow, "IS_PROD", 'JDL', True,
                           "This job is a production job")
        if not script:
            self.__setDefaults()

        self._recBasePaths = {}
Beispiel #13
0
    def initialize(self):
        """ agent initialisation

    reading and setting confing opts

    :param self: self reference
    """
        # # shifter proxy
        self.am_setOption('shifterProxy', 'DataManager')
        # # transformations types
        self.dataProcTTypes = Operations().getValue(
            'Transformations/DataProcessing', self.dataProcTTypes)
        self.dataManipTTypes = Operations().getValue(
            'Transformations/DataManipulation', self.dataManipTTypes)
        agentTSTypes = self.am_getOption('TransformationTypes', [])
        if agentTSTypes:
            self.transformationTypes = sorted(agentTSTypes)
        else:
            self.transformationTypes = sorted(self.dataProcTTypes +
                                              self.dataManipTTypes)
        self.log.info("Will consider the following transformation types: %s" %
                      str(self.transformationTypes))
        # # directory locations
        self.directoryLocations = sorted(
            self.am_getOption('DirectoryLocations',
                              ['TransformationDB', 'MetadataCatalog']))
        self.log.info(
            "Will search for directories in the following locations: %s" %
            str(self.directoryLocations))
        # # transformation metadata
        self.transfidmeta = self.am_getOption('TransfIDMeta',
                                              "TransformationID")
        self.log.info("Will use %s as metadata tag name for TransformationID" %
                      self.transfidmeta)
        # # archive periof in days
        self.archiveAfter = self.am_getOption('ArchiveAfter', 7)  # days
        self.log.info("Will archive Completed transformations after %d days" %
                      self.archiveAfter)
        # # active SEs
        self.activeStorages = sorted(self.am_getOption('ActiveSEs', []))
        self.log.info("Will check the following storage elements: %s" %
                      str(self.activeStorages))
        # # transformation log SEs
        self.logSE = Operations().getValue('/LogStorage/LogSE', 'LogSE')
        self.log.info("Will remove logs found on storage element: %s" %
                      self.logSE)
        # # enable/disable execution, should be using CS option Status?? with default value as 'Active'??
        self.enableFlag = self.am_getOption('EnableFlag', 'True')

        # # transformation client
        self.transClient = TransformationClient()
        # # wms client
        self.wmsClient = WMSClient()
        # # request client
        self.reqClient = ReqClient()
        # # file catalog client
        self.metadataClient = FileCatalogClient()

        return S_OK()
Beispiel #14
0
 def __init__(self, fcClient=False):
     """Internal initialization of Badger API.
     """
     if not fcClient:
         _fcType = 'DataManagement/FileCatalog'
         self.client = FileCatalogClient(_fcType)
     else:
         self.client = fcClient
Beispiel #15
0
def check_file_existence(directory, file, dontPromptMe=False):
    myLFN = 'LFN:' + STORAGE_USER_PATH + directory + '/' + file
    fcc = FileCatalogClient()
    result = fcc.exists(myLFN)
    if myLFN in result.get('Value', {}).get('Successful', {}) and result.get(
            'Value', {}).get('Successful', {})[myLFN]:
        return True
    return False
Beispiel #16
0
def registerInputData(filepath, size, prefix='/cepc/lustre-ro'):
    infoDict = {}
    infoDict['PFN'] = ''
    infoDict['Size'] = size
    infoDict['SE'] = 'IHEP-STORM'
    infoDict['GUID'] = commands.getoutput('uuidgen')
    infoDict['Checksum'] = ''
    fileDict = {}
    lfn = prefix + filepath
    fileDict[lfn] = infoDict
    fcc = FileCatalogClient('DataManagement/FileCatalog')
    rm = ReplicaManager()
    result = {}
    result['lfn'] = lfn
    result['is_registered'] = False

    #查询
    for repeatTimes in range(10):
        is_registered = fcc.isFile(lfn)
        if (is_registered['OK']
                and is_registered['Value']['Successful'].has_key(lfn)):
            break
#         else:
#             continue
    if not is_registered['OK']:  #查询失败
        result['is_registered'] = 'querry error. unkown'
        print 'Failed to query %s in DFC. Error message is %s' % (
            lfn, is_registered['Message'])

    if is_registered['Value']['Successful'][lfn]:  #已注册
        result['is_registered'] = True
        for repeatTimes in range(10):
            is_removed = rm.removeCatalogFile(lfn)  #删除
            if (is_removed['OK']
                    and is_removed['Value']['Successful'][lfn]['FileCatalog']):
                result['is_removed'] = True
                break
#             else:
#                 continue
        if not is_removed['OK']:  #删除失败
            result['is_removed'] = 'remove error'
            print 'Failed to remove %s from DFC.' % lfn
    #add
    for repeatTimes in range(10):
        is_added = fcc.addFile(fileDict)  #add/register
        if (is_added['OK'] and is_added['Value']['Successful'][lfn]):
            result['OK'] = True
            return result
#         else:
#             continue
    if not is_added['OK']:  #add unsuccessfully
        result['OK'] = False
        result['Message'] = is_added['Message']
    elif is_added['Value']['Failed']:
        result['OK'] = False
        result['Message'] = 'Failed to add file' + lfn
    return result
 def __init__(self):
   super(DBDGenRegisterOutputData).__init__()
   self.version = "DBDGenRegisterOutputData v1"
   self.log = gLogger.getSubLogger( "DBDGenRegisterOutputData" )
   self.commandTimeOut = 10 * 60
   self.enable = True
   self.fcc = FileCatalogClient()
   self.nbofevents = 0
   self.prodOutputLFNs = []
def _setMetaData(directory, meta):
    fcc = FileCatalogClient()
    res = fcc.createDirectory(directory)
    if not res['OK']:
        raise Exception('DFC createDirectory for "{0}" error: {1}'.format(
            directory, res['Message']))
    res = fcc.setMetadata(directory, meta)
    if not res['OK']:
        raise Exception('DFC setMetadata for "{0}" error: {1}'.format(
            directory, res['Message']))
Beispiel #19
0
 def __init__(self):
     super(ILDRegisterOutputData, self).__init__()
     self.version = "ILDRegisterOutputData v1"
     self.log = gLogger.getSubLogger("ILDRegisterOutputData")
     self.commandTimeOut = 10 * 60
     self.enable = True
     self.prodOutputLFNs = []
     self.nbofevents = 0
     self.filecatalog = FileCatalogClient()
     self.ildconfig = ''
     self.swpackages = []
Beispiel #20
0
def registerInputData(filepath, size, prefix = '/cepc/lustre-ro'):
    infoDict = {}
    infoDict['PFN'] = ''
    infoDict['Size'] = size
    infoDict['SE'] = 'IHEP-STORM'
    infoDict['GUID'] = commands.getoutput('uuidgen')
    infoDict['Checksum'] = ''
    fileDict = {}
    lfn =  prefix + filepath
    fileDict[lfn] = infoDict
    fcc = FileCatalogClient('DataManagement/FileCatalog')
    rm = ReplicaManager()
    result = {}
    result['lfn'] = lfn
    result['is_registered'] = False
    
    #查询
    for repeatTimes in range(10):
        is_registered = fcc.isFile(lfn)
        if (is_registered['OK'] and is_registered['Value']['Successful'].has_key(lfn)):
            break
#         else:
#             continue
    if not is_registered['OK']:#查询失败
        result['is_registered'] = 'querry error. unkown'
        print 'Failed to query %s in DFC. Error message is %s' %(lfn, is_registered['Message'])
        
    if is_registered['Value']['Successful'][lfn]:#已注册
        result['is_registered'] = True
        for repeatTimes in range(10):
            is_removed = rm.removeCatalogFile(lfn)#删除
            if (is_removed['OK'] and is_removed['Value']['Successful'][lfn]['FileCatalog']):
                result['is_removed'] = True
                break
#             else:
#                 continue
        if not is_removed['OK']:#删除失败
            result['is_removed'] = 'remove error'
            print 'Failed to remove %s from DFC.' %lfn
    #add       
    for repeatTimes in range(10):
        is_added = fcc.addFile(fileDict)#add/register
        if (is_added['OK'] and is_added['Value']['Successful'][lfn]):
            result['OK'] = True
            return result
#         else:
#             continue
    if not is_added['OK']:#add unsuccessfully
        result['OK'] = False
        result['Message'] = is_added['Message']
    elif is_added['Value']['Failed']:
        result['OK'] = False
        result['Message'] = 'Failed to add file' + lfn
    return result
Beispiel #21
0
def get_dataset_MQ(dataset_name):
    """ Return the Meta Query associated with a given dataset
    """
    fc = FileCatalogClient()
    result = returnSingleResult(fc.getDatasetParameters(dataset_name))
    if not result['OK']:
        DIRAC.gLogger.error("Failed to retrieved dataset:", result['Message'])
        DIRAC.exit(-1)
    else:
        DIRAC.gLogger.info("Successfully retrieved dataset: ", dataset_name)
    return result['Value']['MetaQuery']
def get_list_of_datasets(tag=''):
    fc = FileCatalogClient()
    dataset_tag = '*%s*' % tag
    result = fc.getDatasets(dataset_tag)
    if not result['OK']:
        print("ERROR: failed to get datasets")
        DIRAC.exit(-1)
    dataset_dict = result['Value']
    dataset_list = list()
    for dataset_name in dataset_dict['Successful'][dataset_tag].keys():
        dataset_list.append(dataset_name)
    return dataset_list
def get_dataset_info(dataset_name):
    fc = FileCatalogClient()
    result = fc.getDatasets(dataset_name)
    if not result['OK']:
        print("ERROR: failed to get datasets")
        DIRAC.exit(-1)
    dataset_dict = result['Value']
    res = dataset_dict['Successful'][dataset_name][dataset_name]
    number_of_files = res['NumberOfFiles']
    meta_query = res['MetaQuery']
    total_size = res['TotalSize']
    return (dataset_name, number_of_files, total_size, meta_query)
Beispiel #24
0
def _findInFC():
    """Find something in the FileCatalog"""
    from DIRAC import exit as dexit
    clip = _Params()
    clip.registerSwitches()
    Script.parseCommandLine()

    args = Script.getPositionalArgs()
    if len(args) < 2:
        Script.showHelp('ERROR: Not enough arguments')
        gLogger.error("Run %s --help" % SCRIPTNAME)
        dexit(1)

    path = args[0]
    if path == '.':
        path = '/'

    ## Check that the first argument is not a MetaQuery
    if any(op in path for op in OPLIST):
        gLogger.error(
            "ERROR: Path '%s' is not a valid path! The first argument must be a path"
            % path)
        gLogger.error("Run %s --help" % SCRIPTNAME)
        dexit(1)

    gLogger.verbose("Path:", path)
    metaQuery = args[1:]
    metaDataDict = _createQueryDict(metaQuery)
    gLogger.verbose("Query:", str(metaDataDict))
    if not metaDataDict:
        gLogger.info("No query")
        dexit(1)

    fc = FileCatalogClient()
    res = fc.findFilesByMetadata(metaDataDict, path)
    if not res['OK']:
        gLogger.error(res['Message'])
        dexit(1)
    if not res['Value']:
        gLogger.notice("No files found")

    listToPrint = None

    if clip.printOnlyDirectories:
        listToPrint = set("/".join(fullpath.split("/")[:-1])
                          for fullpath in res['Value'])
    else:
        listToPrint = res['Value']

    for entry in listToPrint:
        print entry

    dexit(0)
Beispiel #25
0
def eventTypeCheck(eventType):
    entries = []

    client = FileCatalogClient()
    dir = '/BES3/EventTypeList'
    result = client.listDirectory(dir)
    if result['OK']:
        for i, v in enumerate(result['Value']['Successful'][dir]['SubDirs']):
            if eventType == v.split('/')[2]:
                return True

    return False
Beispiel #26
0
 def initialize(self):
     """ Make the necessary initializations """
     self.fileLog = {}
     self.timeLog = {}
     self.fullTimeLog = {}
     self.pollingTime = self.am_getOption('PollingTime', 120)
     self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod', 86400)
     gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME,
                               "Loops/min", gMonitor.OP_SUM)
     self.transClient = TransformationClient('TransformationDB')
     self.metadataClient = FileCatalogClient()
     return S_OK()
Beispiel #27
0
  def __retrieveAllRemoteAttributes(self, lfnList):
    fc = FileCatalogClient('DataManagement/FileCatalog')
    result = fc.getFileMetadata(lfnList)
    if not result['OK']:
      raise Exception('getFileMetadata failed: %s' % result['Message'])

    attributes = {}
    for lfn in lfnList:
      if lfn in result['Value']['Successful']:
        attributes[lfn] = self.__parseMetadata(result['Value']['Successful'][lfn])

    return attributes
Beispiel #28
0
def eventTypeCheck(eventType):
    entries = []
    
    client = FileCatalogClient()
    dir = '/BES3/EventTypeList'
    result = client.listDirectory(dir)
    if result['OK']:
        for i,v in enumerate(result['Value']['Successful'][dir]['SubDirs']):
            if eventType == v.split('/')[2]:   
                return True

    return False 
 def __init__(self):
     super(RegisterOutputData, self).__init__()
     self.version = "RegisterOutputData v1"
     self.log = gLogger.getSubLogger("RegisterOutputData")
     self.commandTimeOut = 10 * 60
     self.enable = True
     self.prodOutputLFNs = []
     self.nbofevents = 0
     self.luminosity = 0
     self.sel_eff = 0
     self.cut_eff = 0
     self.add_info = ''
     self.filecatalog = FileCatalogClient()
     self.filemeta = {}
Beispiel #30
0
def getFiles():
    """ Get the lfns: This is not the point of this example, so keep it out of the main
  """
    fc = FileCatalogClient()

    meta = {}
    meta['ProdID'] = 1543
    meta["Datatype"] = "DST"

    result = fc.findFilesByMetadata(meta, "/ilc/prod/clic")
    if not result["OK"]:
        gLogger.error(result["Message"])
        dexit(1)
    return result['Value']
def _findInFC():
  """Find something in the FileCatalog"""
  from DIRAC import exit as dexit
  clip = _Params()
  clip.registerSwitches()
  Script.parseCommandLine()

  args = Script.getPositionalArgs()
  if len(args)<2:
    Script.showHelp('ERROR: Not enough arguments')
    gLogger.error("Run %s --help" % SCRIPTNAME )
    dexit(1)
    
  path = args[0]
  if path == '.':
    path = '/'

  ## Check that the first argument is not a MetaQuery
  if any( op in path for op in OPLIST ):
    gLogger.error("ERROR: Path '%s' is not a valid path! The first argument must be a path" % path)
    gLogger.error("Run %s --help" % SCRIPTNAME )
    dexit(1)

  gLogger.verbose("Path:", path)
  metaQuery = args[1:]
  metaDataDict = _createQueryDict(metaQuery)
  gLogger.verbose("Query:",str(metaDataDict))
  if not metaDataDict:
    gLogger.info("No query")
    dexit(1)
  
  fc = FileCatalogClient()
  res = fc.findFilesByMetadata(metaDataDict, path)
  if not res['OK']:
    gLogger.error(res['Message'])
    dexit(1)
  if not res['Value']:
    gLogger.notice("No files found")

  listToPrint = None

  if clip.printOnlyDirectories:
    listToPrint = set( "/".join(fullpath.split("/")[:-1]) for fullpath in res['Value'] )
  else:
    listToPrint = res['Value']

  for entry in listToPrint:
    print entry

  dexit(0)
def getFiles():
  """ Get the lfns: This is not the point of this example, so keep it out of the main
  """
  fc = FileCatalogClient()
  
  meta = {}
  meta['ProdID'] = 1543
  meta["Datatype"] = "DST"
  
  result = fc.findFilesByMetadata(meta, "/ilc/prod/clic")
  if not result["OK"]:
    gLogger.error(result["Message"])
    dexit(1)
  return result['Value']
Beispiel #33
0
def addDataFiles(args):

    fcc = FileCatalogClient()
    dm = DataManager(['FileCatalog', 'TSCatalog'])

    outputPath = args[0]
    outputPattern = args[1]
    outputSE = args[2]
    metadata = args[3]
    metadata = json.loads(metadata)

    ## Create path
    res = fcc.createDirectory(outputPath)
    if not res['OK']:
        return res

    ##  Set metadata
    for key, value in metadata.items():
        res = fcc.setMetadata(outputPath, {key: value})
        # terrible hack
        # if not res['OK']:
        #  return res

    ## Upload data files
    all_files = glob.glob(outputPattern)

    ## Check that data files exist
    if len(all_files) == 0:
        return DIRAC.S_ERROR('No data files found')

    for one_file in all_files:
        lfn = os.path.join(outputPath, one_file)
        msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % (
            one_file, lfn, outputSE)
        DIRAC.gLogger.notice(msg)
        res = dm.putAndRegister(lfn, one_file, outputSE)
        ##  Check if failed
        if not res['OK']:
            DIRAC.gLogger.error(
                'Failed to putAndRegister %s \nto %s \nwith message: %s' %
                (lfn, outputSE, res['Message']))
            return res
        elif res['Value']['Failed'].has_key(lfn):
            DIRAC.gLogger.error('Failed to putAndRegister %s to %s' %
                                (lfn, outputSE))
            return res

    return DIRAC.S_OK()
  def initialize( self ):
    """Sets defaults """
    self.replicaManager = ReplicaManager()
    self.transClient = TransformationClient()
    self.wmsClient = WMSClient()
    self.requestClient = RequestClient()
    self.metadataClient = FileCatalogClient()
    self.storageUsageClient = StorageUsageClient()

    # This sets the Default Proxy to used as that defined under 
    # /Operations/Shifter/DataManager
    # the shifterProxy option in the Configuration can be used to change this default.
    self.am_setOption( 'shifterProxy', 'DataManager' )

    self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge', 'Replication'] ) )
    gLogger.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) )
    self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'StorageUsage', 'MetadataCatalog'] ) )
    gLogger.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) )
    self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
    gLogger.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta )
    self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days
    gLogger.info( "Will archive Completed transformations after %d days" % self.archiveAfter )
    self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) )
    gLogger.info( "Will check the following storage elements: %s" % str( self.activeStorages ) )
    self.logSE = self.am_getOption( 'TransformationLogSE', 'LogSE' )
    gLogger.info( "Will remove logs found on storage element: %s" % self.logSE )
    return S_OK()
Beispiel #35
0
  def __init__(self, script = None):
    super(ProductionJob, self).__init__( script )
    self.prodVersion = __RCSID__
    self.dryrun = False
    self.created = False
    self.checked = False
    self.call_finalization = False
    self.finalsdict = {}
    self.transfid = 0
    self.type = 'Production'
    self.csSection = '/Production/Defaults'
    self.ops = Operations()
    self.fc = FileCatalogClient()
    self.trc = TransformationClient()
    self.defaultProdID = '12345'
    self.defaultProdJobID = '12345'
    self.jobFileGroupSize = 1
    self.nbtasks = 1
    self.slicesize =0
    self.basename = ''
    self.basepath = self.ops.getValue('/Production/CLIC/BasePath','/ilc/prod/clic/')
    self.evttype = ''
    self.datatype = ''
    self.energycat = ''
    self.detector = ''
    self.currtrans = None
    self.description = ''

    self.finalpaths = []
    self.finalMetaDict = defaultdict( dict )
    self.prodMetaDict = {}
    self.finalMetaDictNonSearch = {}
    self.metadict_external = {}
    self.outputStorage = ''

    self.proxyinfo = getProxyInfo()

    self.inputdataquery = False
    self.inputBKSelection = {}
    self.plugin = 'Standard'
    self.prodGroup = ''

    self.prodTypes = ['MCGeneration', 'MCSimulation', 'Test', 'MCReconstruction',
                      'MCReconstruction_Overlay', 'Merge', 'Split',
                      'MCGeneration_ILD',
                      'MCSimulation_ILD',
                      'MCReconstruction_ILD',
                      'MCReconstruction_Overlay_ILD',
                      'Split_ILD'
                     ]
    self.prodparameters = {}
    self.prodparameters['NbInputFiles'] = 1
    self.prodparameters['nbevts']  = 0 
    #self.prodparameters["SWPackages"] = ''
    self._addParameter(self.workflow, "IS_PROD", 'JDL', True, "This job is a production job")
    if not script:
      self.__setDefaults()

    self._recBasePaths = {}
    self.maxFCFoldersToCheck = 100000
    def __init__(self, cpuTime=432000):
        """ Constructor

    Keyword arguments:
    cpuTime -- max cpu time allowed for the job
    """
        Job.__init__(self)
        self.setCPUTime(cpuTime)
        # defaults
        self.setName('Evndisplay_CalibReco')
        self.package = 'evndisplay'
        self.program_category = 'calibimgreco'
        self.version = 'prod3b_d20170602'  # or later
        self.configuration_id = 0
        self.output_data_level = 1
        self.prefix = 'CTA.prod3Nb'
        self.layout = 'Baseline'
        self.calibration_file = 'gamma_20deg_180deg_run3___cta-prod3-lapalma3-2147m-LaPalma.ped.root'
        self.reconstructionparameter = 'EVNDISP.prod3.reconstruction.runparameter.NN'
        self.NNcleaninginputcard = 'EVNDISP.NNcleaning.dat'
        self.basepath = '/vo.cta.in2p3.fr/MC/PROD3/'
        self.fcc = FileCatalogClient()
        self.metadata = collections.OrderedDict()
        self.filemetadata = {}
        self.catalogs = json.dumps(['DIRACFileCatalog', 'TSCatalog2'])
    def __init__(self, cpuTime=432000):
        """ Constructor

    Keyword arguments:
    cpuTime -- max cpu time allowed for the job
    """
        Job.__init__(self)
        self.setCPUTime(cpuTime)
        # defaults
        self.setName('Evndisplay_Reco')
        self.package = 'evndisplay'
        self.program_category = 'reconstruction'
        self.version = 'prod3b_d20170602'  # or later
        self.configuration_id = 0
        self.output_data_level = 2
        self.prefix = 'CTA.prod3Nb'
        self.layout = 'Baseline'
        self.pointing = '180'
        self.table_file = 'tables_CTA-prod3b-LaPalma-NNq05-NN-ID0_0deg-d20160925m4-Nb.3AL4-BN15.root'
        self.disp_subdir_name = 'BDTdisp.Nb.3AL4-BN15.T1'
        self.recid = '0,1,2'  # 0 = all teltescopes, 1 = LST only, 2 = MST only
        self.basepath = '/vo.cta.in2p3.fr/MC/PROD3/'
        self.fcc = FileCatalogClient()
        self.metadata = collections.OrderedDict()
        self.filemetadata = {}
        self.catalogs = json.dumps(['DIRACFileCatalog', 'TSCatalog'])
Beispiel #38
0
    def __init__(self, cpuTime=432000):
        """ Constructor

        Keyword arguments:
        cpuTime -- max cpu time allowed for the job
        """
        Job.__init__(self)
        self.setCPUTime(cpuTime)
        # defaults
        self.setName('Evndisplay_CalibReco')
        self.package = 'evndisplay'
        self.program_category = 'calibimgreco'
        self.version = 'prod4_d20181110'
        self.configuration_id = 4
        self.output_data_level = DATA_LEVEL_METADATA_ID['DL1']
        self.N_output_files = 1
        self.prefix = 'CTA.prod4S'
        self.layout = '3HB9-SST'
        self.calibration_file = 'prod4b-SST-IPR.root'
        self.reconstructionparameter = 'EVNDISP.prod4.reconstruction.runparameter.NN.noLL'
        self.base_path = '/vo.cta.in2p3.fr/MC/PROD4/'
        self.fcc = FileCatalogClient()
        self.metadata = collections.OrderedDict()
        self.filemetadata = {}
        self.catalogs = json.dumps(['DIRACFileCatalog', 'TSCatalog'])
        self.ts_task_id = 0
  def __init__( self, *args, **kwargs ):
    ''' c'tor
    '''
    AgentModule.__init__( self, *args, **kwargs )
    # # replica manager
    self.replicaManager = ReplicaManager()
    # # transformation client
    self.transClient = TransformationClient()
    # # wms client
    self.wmsClient = WMSClient()
    # # request client
    self.requestClient = RequestClient()
    # # file catalog clinet
    self.metadataClient = FileCatalogClient()

    # # placeholders for CS options

    # # transformations types
    self.transformationTypes = None
    # # directory locations
    self.directoryLocations = None
    # # transformation metadata
    self.transfidmeta = None
    # # archive periof in days
    self.archiveAfter = None
    # # active SEs
    self.activeStorages = None
    # # transformation log SEs
    self.logSE = None
    # # enable/disable execution
    self.enableFlag = None
Beispiel #40
0
  def __init__(self):
    super(OverlayInput, self).__init__()
    self.enable = True
    self.STEP_NUMBER = ''
    self.log = gLogger.getSubLogger( "OverlayInput" )
    self.applicationName = 'OverlayInput'
    self.curdir = os.getcwd()
    self.applicationLog = ''
    self.printoutflag = ''
    self.prodid = 0
    self.detector = '' ##needed for backward compatibility
    self.detectormodel = ""
    self.energytouse = ''
    self.energy = 0
    self.nbofeventsperfile = 100
    self.lfns = []
    self.nbfilestoget = 0
    self.BkgEvtType = 'gghad'
    self.BXOverlay = 0
    self.ggtohadint = 3.2
    self.nbsigeventsperfile = 0
    self.nbinputsigfile = 1
    self.NbSigEvtsPerJob = 0
    self.rm = ReplicaManager()
    self.fc = FileCatalogClient()
    self.site = DIRAC.siteName()

    self.machine = 'clic_cdr'
 def __init__(self):
   super(DBDGenRegisterOutputData).__init__()
   self.version = "DBDGenRegisterOutputData v1"
   self.commandTimeOut = 10 * 60
   self.enable = True
   self.fcc = FileCatalogClient()
   self.nbofevents = 0
   self.prodOutputLFNs = []
Beispiel #42
0
 def __init__(self, fcClient=False):
     """Internal initialization of Badger API.
     """
     if not fcClient:
         _fcType = "DataManagement/FileCatalog"
         self.client = FileCatalogClient(_fcType)
     else:
         self.client = fcClient
def runOne ():
    fc = FileCatalogClient()
    for mokkaJobNumber in range(39,45):
        for eventType in ['Z_uds','Photon','Kaon0L','Muon']:
            energies = []
            if eventType == 'Z_uds':
                energies = [91,200,360,500,750,1000,2000,3000]
            elif eventType == 'Photon':
                energies = [10]
            elif eventType == 'Muon':
                energies = [10]
            elif eventType == 'Kaon0L':
                energies = [1,2,3,4,5,6,7,8,9,10,15,20,25,30,35,40,45,50]
            for energy in energies:
                path = '/ilc/user/s/sgreen/0804/HCalStudiessg568/GJN' + str(mokkaJobNumber) + '/' + eventType + '/' + str(energy) + 'GeV'
                pathdict = {'path':path, 'meta':{'MokkaJobNumber':mokkaJobNumber, 'Type':eventType, 'Energy':energy}}
                res = fc.setMetadata(pathdict['path'], pathdict['meta'])
    return
Beispiel #44
0
def compare(attributes,input):
    error_list = {}
    inDFC = {}
    
    #print "input keys:",input.keys()
    keys = sorted(input.keys())
    #print "keys after being sorted:",keys
    
    expNum = _get_exp_num(attributes["expNum"])
    eventType = _get_event_type(attributes["eventType"])
    #dir = "/BES3/File/"+attributes["resonance"]+"/"+attributes["bossVer"]

    dir = "/BES3/File/"+attributes["resonance"]+"/"+attributes["bossVer"]

    if attributes["streamId"]=="stream0":
        dir = dir+ "/data"+"/"+eventType + "/"+expNum+"/"+attributes["LFN"]
    else:
        dir = dir+"/mc"+"/"+eventType+"/"+expNum+"/"+attributes["streamId"]+"/"+attributes["LFN"]

    client=FileCatalogClient()
    
    result = client.getFileMetadata(dir)

    file_exist = len(result['Value']['Successful'])
    if file_exist == 0:
        print "this file does't exist in DFC",attributes['LFN']
    else:    
        result = client.getDirectoryMetadata(dir)

        if result["OK"]:
            inDFC["resonance"] = result["Value"]["resonance"]
            inDFC["streamId"] = result["Value"]["streamId"]
            inDFC["eventType"] = result["Value"]["eventType"]
            inDFC["bossVer"] = result["Value"]["bossVer"]
            inDFC["expNum"] = result["Value"]["expNum"]
        
        for key in keys:
            if input[key] != inDFC[key]:
                error_list[key] = inDFC[key]


        if error_list is not None:
            return error_list
 def __init__(self):
   super(ILDRegisterOutputData, self).__init__()
   self.version = "ILDRegisterOutputData v1"
   self.commandTimeOut = 10 * 60
   self.enable = True
   self.prodOutputLFNs = []
   self.nbofevents = 0
   self.filecatalog = FileCatalogClient()
   self.ildconfig = ''
   self.swpackages = []
Beispiel #46
0
 def initialize(self):
   """ Make the necessary initializations """
   self.fileLog = {}
   self.timeLog = {}
   self.fullTimeLog = {}
   self.pollingTime = self.am_getOption('PollingTime',120)
   self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod',86400)
   gMonitor.registerActivity("Iteration","Agent Loops",AGENT_NAME,"Loops/min",gMonitor.OP_SUM)
   self.transClient = TransformationClient('TransformationDB')
   self.metadataClient = FileCatalogClient()
   return S_OK()
 def __init__(self):
   super(SIDRegisterOutputData, self).__init__()
   self.version = "SIDRegisterOutputData v1"
   self.log = gLogger.getSubLogger( "SIDRegisterOutputData" )
   self.commandTimeOut = 10 * 60
   self.enable = True
   self.prodOutputLFNs = []
   self.swpackages = []
   self.nbofevents = 0
   self.luminosity = 0
   self.filecatalog = FileCatalogClient()
Beispiel #48
0
 def __init__(self):
   super(RegisterOutputData, self).__init__()
   self.version = "RegisterOutputData v1"
   self.commandTimeOut = 10*60
   self.enable = True
   self.prodOutputLFNs = []
   self.nbofevents = 0
   self.luminosity = 0
   self.sel_eff = 0
   self.cut_eff = 0
   self.add_info = ''
   self.filecatalog = FileCatalogClient()
   self.filemeta = {}
Beispiel #49
0
def insert(attributes):

    dir = "/BES3/File/"+attributes["resonance"]+"/"+attributes["bossVer"]

    #get real eventType and expNum in catalog in amga
    eventType = _get_event_type(attributes["eventType"])
    expNum = _get_exp_num(attributes["expNum"])

    if attributes["streamId"]=="stream0":
        dir=dir+"/data"
    else:
        dir=dir+"/mc"

    client = FileCatalogClient()
    
    # make sure insertion directory exists in DFC
    insertDir = createCatalog(client,dir,eventType,expNum,attributes)

    #do we need to explicitly add LFN attr in DFC?
    # we do need to get the PFN, size, se for addFile (below)
    #TODO pfn = ?
    # need to add PFN attribute to the list, probably in control.py
    pfn = attributes["PFN"]
    lfn=insertDir+"/"+attributes["LFN"]
    attributes["LFN"] = lfn
    #TODO se = ?  temporarily use a made-up SE name
    se = 'BES-TEST' 

    # register files in DFC
    infoDict = {}
    #infoDict['PFN'] = attributes['PFN']
    infoDict['PFN'] = pfn
    infoDict['Size'] = attributes['fileSize']
    infoDict['SE'] = se
   # infoDict['guid'] = attributes['guid']
    infoDict['Checksum'] = ''
    result = client.addFile({lfn:infoDict})
    print result
  def initialize( self ):
    """ agent initialisation

    reading and setting confing opts

    :param self: self reference
    """
    # # shifter proxy
    self.am_setOption( 'shifterProxy', 'DataManager' )
    # # transformations types
    self.dataProcTTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
    self.dataManipTTypes = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
    agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
    if agentTSTypes:
      self.transformationTypes = sorted( agentTSTypes )
    else:
      self.transformationTypes = sorted( self.dataProcTTypes + self.dataManipTTypes )
    self.log.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) )
    # # directory locations
    self.directoryLocations = sorted( self.am_getOption( 'DirectoryLocations', [ 'TransformationDB',
                                                                                   'MetadataCatalog' ] ) )
    self.log.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) )
    # # transformation metadata
    self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
    self.log.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta )
    # # archive periof in days
    self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 )  # days
    self.log.info( "Will archive Completed transformations after %d days" % self.archiveAfter )
    # # active SEs
    self.activeStorages = sorted( self.am_getOption( 'ActiveSEs', [] ) )
    self.log.info( "Will check the following storage elements: %s" % str( self.activeStorages ) )
    # # transformation log SEs
    self.logSE = Operations().getValue( '/LogStorage/LogSE', 'LogSE' )
    self.log.info( "Will remove logs found on storage element: %s" % self.logSE )
    # # enable/disable execution, should be using CS option Status?? with default value as 'Active'??
    self.enableFlag = self.am_getOption( 'EnableFlag', 'True' )

    # # data manager
#     self.dm = DataManager()
    # # transformation client
    self.transClient = TransformationClient()
    # # wms client
    self.wmsClient = WMSClient()
    # # request client
    self.reqClient = ReqClient()
    # # file catalog client
    self.metadataClient = FileCatalogClient()

    return S_OK()
  def initialize(self):
    """ agent initialisation

    reading and setting confing opts

    :param self: self reference
    """
    # # shifter proxy
    # See cleanCatalogContents method: this proxy will be used ALSO when the file catalog used
    # is the DIRAC File Catalog (DFC).
    # This is possible because of unset of the "UseServerCertificate" option
    self.shifterProxy = self.am_getOption('shifterProxy', None)

    # # transformations types
    self.dataProcTTypes = Operations().getValue('Transformations/DataProcessing', self.dataProcTTypes)
    self.dataManipTTypes = Operations().getValue('Transformations/DataManipulation', self.dataManipTTypes)
    agentTSTypes = self.am_getOption('TransformationTypes', [])
    if agentTSTypes:
      self.transformationTypes = sorted(agentTSTypes)
    else:
      self.transformationTypes = sorted(self.dataProcTTypes + self.dataManipTTypes)
    self.log.info("Will consider the following transformation types: %s" % str(self.transformationTypes))
    # # directory locations
    self.directoryLocations = sorted(self.am_getOption('DirectoryLocations', self.directoryLocations))
    self.log.info("Will search for directories in the following locations: %s" % str(self.directoryLocations))
    # # transformation metadata
    self.transfidmeta = self.am_getOption('TransfIDMeta', self.transfidmeta)
    self.log.info("Will use %s as metadata tag name for TransformationID" % self.transfidmeta)
    # # archive periof in days
    self.archiveAfter = self.am_getOption('ArchiveAfter', self.archiveAfter)  # days
    self.log.info("Will archive Completed transformations after %d days" % self.archiveAfter)
    # # active SEs
    self.activeStorages = sorted(self.am_getOption('ActiveSEs', self.activeStorages))
    if self.activeStorages:
      self.log.info("Will check the following storage elements: %s" % str(self.activeStorages))
    # # transformation log SEs
    self.logSE = Operations().getValue('/LogStorage/LogSE', self.logSE)
    self.log.info("Will remove logs found on storage element: %s" % self.logSE)

    # # transformation client
    self.transClient = TransformationClient()
    # # wms client
    self.wmsClient = WMSClient()
    # # request client
    self.reqClient = ReqClient()
    # # file catalog client
    self.metadataClient = FileCatalogClient()

    return S_OK()
class DBDGenRegisterOutputData(ModuleBase):
  """ Normally, was supposed to be used to produce the DBD gen level files. Dropped in the end.
  """
  def __init__(self):
    super(DBDGenRegisterOutputData).__init__()
    self.version = "DBDGenRegisterOutputData v1"
    self.log = gLogger.getSubLogger( "DBDGenRegisterOutputData" )
    self.commandTimeOut = 10 * 60
    self.enable = True
    self.fc = FileCatalogClient()
    self.nbofevents = 0
    self.prodOutputLFNs = []
    
  def applicationSpecificInputs(self):
    if self.workflow_commons.has_key('ProductionOutputData'):
      self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
      
    if self.workflow_commons.has_key('NbOfEvts'):
      self.nbofevents = self.workflow_commons[ 'NbOfEvts']          
    return S_OK("Parameters resolved")
      
  def execute(self):
    self.log.info('Initializing %s' % self.version)
    result = self.resolveInputVariables()
    if not result['OK']:
      self.log.error(result['Message'])
      return result
    
    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('No registration of output data metadata attempted')

    if not len(self.prodOutputLFNs):
      self.log.info('No production data found, so no metadata registration to be done')  
      return S_OK("No files' metadata to be registered")
    
    self.log.verbose("Will try to set the metadata for the following files: \n %s" % '\n'.join(self.prodOutputLFNs))
    
    for files in self.prodOutputLFNs:
      metadict = {}
      metadict['NumberOfEvents'] = self.nbofevents
      path = files
      
      res = self.fc.setMetadata(files, metadict)
      if not res['OK']:
        self.log.error("Could not register %s for %s"%(metadict, path))
        return res
    
    return S_OK()
Beispiel #53
0
  def __init__( self, *args, **kwargs ):
    ''' c'tor
    '''
    AgentModule.__init__( self, *args, **kwargs )

    self.fileLog = {}
    self.timeLog = {}
    self.fullTimeLog = {}

    self.pollingTime = self.am_getOption( 'PollingTime', 120 )
    self.fullUpdatePeriod = self.am_getOption( 'FullUpdatePeriod', 86400 )
    self.refreshonly = self.am_getOption( 'RefreshOnly', False )
    self.dateKey = self.am_getOption( 'DateKey', None )

    self.transClient = TransformationClient()
    self.metadataClient = FileCatalogClient()
Beispiel #54
0
  def __init__( self, *args, **kwargs ):
    """ c'tor
    """
    AgentModule.__init__( self, *args, **kwargs )

    self.integrityClient = DataIntegrityClient()
    self.replicaManager = ReplicaManager()
    self.transClient = TransformationClient()
    self.fileCatalogClient = FileCatalogClient()

    agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
    if agentTSTypes:
      self.transformationTypes = agentTSTypes
    else:
      self.transformationTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )

    self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'MetadataCatalog'] ) )
    self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) )
    self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
  def __init__(self, *args, **kwargs):
    AgentModule.__init__(self, *args, **kwargs)
    self.name = 'FileStatusTransformationAgent'
    self.enabled = False
    self.shifterProxy = 'DataManager'
    self.transformationTypes = ["Replication"]
    self.transformationStatuses = ["Active"]
    self.transformationFileStatuses = ["Assigned", "Problematic", "Processed", "Unused"]

    self.addressTo = ["*****@*****.**"]
    self.addressFrom = "*****@*****.**"
    self.emailSubject = "FileStatusTransformationAgent"

    self.accounting = defaultdict(list)
    self.errors = []

    self.fcClient = FileCatalogClient()
    self.tClient = TransformationClient()
    self.reqClient = ReqClient()
    self.nClient = NotificationClient()
  def __init__(self, *args, **kwargs):
    """ c'tor
    """
    AgentModule.__init__(self, *args, **kwargs)

    self.consistencyInspector = ConsistencyInspector()
    self.integrityClient = DataIntegrityClient()
    self.fc = FileCatalog()
    self.transClient = TransformationClient()
    self.fileCatalogClient = FileCatalogClient()

    agentTSTypes = self.am_getOption('TransformationTypes', [])
    if agentTSTypes:
      self.transformationTypes = agentTSTypes
    else:
      self.transformationTypes = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge'])

    self.directoryLocations = sorted(self.am_getOption('DirectoryLocations', ['TransformationDB',
                                                                              'MetadataCatalog']))
    self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID")
    self.enableFlag = True
Beispiel #57
0
  def initialize( self ):
    """Sets defaults
    """
    self.integrityClient = DataIntegrityClient()
    self.replicaManager = ReplicaManager()
    self.transClient = TransformationClient()
    self.fileCatalogClient = FileCatalogClient()

    # This sets the Default Proxy to used as that defined under 
    # /Operations/Shifter/DataManager
    # the shifterProxy option in the Configuration can be used to change this default.
    self.am_setOption( 'shifterProxy', 'DataManager' )

    self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge'] ) )
    gLogger.info( "Will treat the following transformation types: %s" % str( self.transformationTypes ) )
    self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'MetadataCatalog'] ) )
    gLogger.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) )
    self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) )
    gLogger.info( "Will check the following storage elements: %s" % str( self.activeStorages ) )
    self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
    gLogger.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta )
    return S_OK()
  def __init__( self, agentName, baseAgentName = False,	properties = dict() ):
    """ c'tor

    :param self: self reference
    :param str agentName: name of agent
    :param bool baseAgentName: whatever  
    :param dict properties: whatever else
    """
    AgentModule.__init__( self, agentName, baseAgentName, properties )
    ## replica manager
    self.replicaManager = ReplicaManager()
    ## transformation client
    self.transClient = TransformationClient()
    ## wms client
    self.wmsClient = WMSClient()
    ## request client
    self.requestClient = RequestClient()
    ## file catalog clinet
    self.metadataClient = FileCatalogClient()
    ## storage usage agent
    self.storageUsageClient = StorageUsageClient()

    ## placeholders for CS options

    ## transformations types
    self.transformationTypes = None 
    ## directory locations
    self.directoryLocations = None 
    ## transformation metadata
    self.transfidmeta = None 
    ## archive periof in days
    self.archiveAfter = None 
    ## active SEs
    self.activeStorages = None 
    ## transformation log SEs
    self.logSE = None 
    ## enable/disable execution
    self.enableFlag = None 
    def __init__(self, *args, **kwargs):
        """ c'tor
    """
        AgentModule.__init__(self, *args, **kwargs)

        self.integrityClient = DataIntegrityClient()
        self.replicaManager = ReplicaManager()
        self.transClient = TransformationClient()
        self.fileCatalogClient = FileCatalogClient()

        agentTSTypes = self.am_getOption("TransformationTypes", [])
        if agentTSTypes:
            self.transformationTypes = agentTSTypes
        else:
            self.transformationTypes = Operations().getValue(
                "Transformations/DataProcessing", ["MCSimulation", "Merge"]
            )

        self.directoryLocations = sortList(
            self.am_getOption("DirectoryLocations", ["TransformationDB", "MetadataCatalog"])
        )
        self.activeStorages = sortList(self.am_getOption("ActiveSEs", []))
        self.transfidmeta = self.am_getOption("TransfIDMeta", "TransformationID")
        self.enableFlag = True